{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5"},"cell_type":"markdown","source":"<h1>Protonet</h1>     \n**As discussed in this [thread](https://www.kaggle.com/c/humpback-whale-identification/discussion/81085#478051), I converted [this](https://github.com/daisukelab/protonet-fine-grained-clf) repository into kernel. My computer can not handle this much computation therefore I used this kernel to play with protonets. I thought it would be helpful for someone who want to do experiment with protonets. \nThe code here does not belongs to me but the kernel is in running form without no errors. **"},{"metadata":{"_uuid":"baac26184352984a60692ed00ae1f8daca585e9d","trusted":false},"cell_type":"code","source":"import cv2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"625100ad82a22f69ce4a6cdcb0ea908cce479238","trusted":false},"cell_type":"code","source":"import os\n\n\nPATH = os.path.dirname(os.path.realpath('../input/'))\n\nDATA_PATH = '../input/'\n\nEPSILON = 1e-8\n\nif DATA_PATH is None:\n    raise Exception('Configure your data folder location in config.py before continuing!')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0170f42f5b282c6e0db26cdd706c8f1f34c95ba0","trusted":false},"cell_type":"code","source":"#few_shot.metric 1\n\nimport torch\n\n\ndef categorical_accuracy(y, y_pred):\n    \"\"\"Calculates categorical accuracy.\n    # Arguments:\n        y_pred: Prediction probabilities or logits of shape [batch_size, num_categories]\n        y: Ground truth categories. Must have shape [batch_size,]\n    \"\"\"\n    return torch.eq(y_pred.argmax(dim=-1), y).sum().item() / y_pred.shape[0]\n\n\nNAMED_METRICS = {\n    'categorical_accuracy': categorical_accuracy\n}","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"#few_shot.utils.py 2\nimport torch\nimport os\nimport shutil\nfrom typing import Tuple, List\n\n#from config import EPSILON, PATH\n\n\ndef mkdir(dir):\n    \"\"\"Create a directory, ignoring exceptions\n\n    # Arguments:\n        dir: Path of directory to create\n    \"\"\"\n    try:\n        os.mkdir(dir)\n    except:\n        pass\n\n\ndef rmdir(dir):\n    \"\"\"Recursively remove a directory and contents, ignoring exceptions\n\n   # Arguments:\n       dir: Path of directory to recursively remove\n   \"\"\"\n    try:\n        shutil.rmtree(dir)\n    except:\n        pass\n\n\ndef setup_dirs():\n    \"\"\"Creates directories for this project.\"\"\"\n    mkdir(PATH + '/logs/')\n    mkdir(PATH + '/logs/proto_nets')\n    mkdir(PATH + '/logs/matching_nets')\n    mkdir(PATH + '/models/')\n    mkdir(PATH + '/models/proto_nets')\n    mkdir(PATH + '/models/matching_nets')\n\n\ndef pairwise_distances(x: torch.Tensor,\n                       y: torch.Tensor,\n                       matching_fn: str) -> torch.Tensor:\n    \"\"\"Efficiently calculate pairwise distances (or other similarity scores) between\n    two sets of samples.\n\n    # Arguments\n        x: Query samples. A tensor of shape (n_x, d) where d is the embedding dimension\n        y: Class prototypes. A tensor of shape (n_y, d) where d is the embedding dimension\n        matching_fn: Distance metric/similarity score to compute between samples\n    \"\"\"\n    n_x = x.shape[0]\n    n_y = y.shape[0]\n\n    if matching_fn == 'l2':\n        distances = (\n                x.unsqueeze(1).expand(n_x, n_y, -1) -\n                y.unsqueeze(0).expand(n_x, n_y, -1)\n        ).pow(2).sum(dim=2)\n        return distances\n    elif matching_fn == 'cosine':\n        normalised_x = x / (x.pow(2).sum(dim=1, keepdim=True).sqrt() + EPSILON)\n        normalised_y = y / (y.pow(2).sum(dim=1, keepdim=True).sqrt() + EPSILON)\n\n        expanded_x = normalised_x.unsqueeze(1).expand(n_x, n_y, -1)\n        expanded_y = normalised_y.unsqueeze(0).expand(n_x, n_y, -1)\n\n        cosine_similarities = (expanded_x * expanded_y).sum(dim=2)\n        return 1 - cosine_similarities\n    elif matching_fn == 'dot':\n        expanded_x = x.unsqueeze(1).expand(n_x, n_y, -1)\n        expanded_y = y.unsqueeze(0).expand(n_x, n_y, -1)\n\n        return -(expanded_x * expanded_y).sum(dim=2)\n    else:\n        raise(ValueError('Unsupported similarity function'))\n\n\ndef copy_weights(from_model: torch.nn.Module, to_model: torch.nn.Module):\n    \"\"\"Copies the weights from one model to another model.\n\n    # Arguments:\n        from_model: Model from which to source weights\n        to_model: Model which will receive weights\n    \"\"\"\n    if not from_model.__class__ == to_model.__class__:\n        raise(ValueError(\"Models don't have the same architecture!\"))\n\n    for m_from, m_to in zip(from_model.modules(), to_model.modules()):\n        is_linear = isinstance(m_to, torch.nn.Linear)\n        is_conv = isinstance(m_to, torch.nn.Conv2d)\n        is_bn = isinstance(m_to, torch.nn.BatchNorm2d)\n        if is_linear or is_conv or is_bn:\n            m_to.weight.data = m_from.weight.data.clone()\n            if m_to.bias is not None:\n                m_to.bias.data = m_from.bias.data.clone()\n\n\ndef autograd_graph(tensor: torch.Tensor) -> Tuple[\n            List[torch.autograd.Function],\n            List[Tuple[torch.autograd.Function, torch.autograd.Function]]\n        ]:\n    \"\"\"Recursively retrieves the autograd graph for a particular tensor.\n\n    # Arguments\n        tensor: The Tensor to retrieve the autograd graph for\n\n    # Returns\n        nodes: List of torch.autograd.Functions that are the nodes of the autograd graph\n        edges: List of (Function, Function) tuples that are the edges between the nodes of the autograd graph\n    \"\"\"\n    nodes, edges = list(), list()\n\n    def _add_nodes(tensor):\n        if tensor not in nodes:\n            nodes.append(tensor)\n\n            if hasattr(tensor, 'next_functions'):\n                for f in tensor.next_functions:\n                    if f[0] is not None:\n                        edges.append((f[0], tensor))\n                        _add_nodes(f[0])\n\n            if hasattr(tensor, 'saved_tensors'):\n                for t in tensor.saved_tensors:\n                    edges.append((t, tensor))\n                    _add_nodes(t)\n\n    _add_nodes(tensor.grad_fn)\n\n    return nodes, edges\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"63d02eab87c9d9ea37bd8606e4a00f3848b0c27c","trusted":false},"cell_type":"code","source":"#few_shot.eval 3\n\nimport torch\nfrom torch.nn import Module\nfrom torch.utils.data import DataLoader\nfrom typing import Callable, List, Union\n\n#from few_shot.metrics import NAMED_METRICS\n\n\ndef evaluate(model: Module, dataloader: DataLoader, prepare_batch: Callable, metrics: List[Union[str, Callable]],\n             loss_fn: Callable = None, prefix: str = 'val_', suffix: str = ''):\n    \"\"\"Evaluate a model on one or more metrics on a particular dataset\n    # Arguments\n        model: Model to evaluate\n        dataloader: Instance of torch.utils.data.DataLoader representing the dataset\n        prepare_batch: Callable to perform any desired preprocessing\n        metrics: List of metrics to evaluate the model with. Metrics must either be a named metric (see `metrics.py`) or\n            a Callable that takes predictions and ground truth labels and returns a scalar value\n        loss_fn: Loss function to calculate over the dataset\n        prefix: Prefix to prepend to the name of each metric - used to identify the dataset. Defaults to 'val_' as\n            it is typical to evaluate on a held-out validation dataset\n        suffix: Suffix to append to the name of each metric.\n    \"\"\"\n    logs = {}\n    seen = 0\n    totals = {m: 0 for m in metrics}\n    if loss_fn is not None:\n        totals['loss'] = 0\n    model.eval()\n    with torch.no_grad():\n        for batch in dataloader:\n            x, y = prepare_batch(batch)\n            y_pred = model(x)\n\n            seen += x.shape[0]\n\n            if loss_fn is not None:\n                totals['loss'] += loss_fn(y_pred, y).item() * x.shape[0]\n\n            for m in metrics:\n                if isinstance(m, str):\n                    v = NAMED_METRICS[m](y, y_pred)\n                else:\n                    # Assume metric is a callable function\n                    v = m(y, y_pred)\n\n                totals[m] += v * x.shape[0]\n\n    for m in ['loss'] + metrics:\n        logs[prefix + m + suffix] = totals[m] / seen\n    return logs","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e9e957fe7f35681e1fffa67cbe3c454a3aacd86a","trusted":false},"cell_type":"code","source":"!pip install albumentations","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6b41d3420df6bb1b4f5b2a890e063d5b819b4030","trusted":false},"cell_type":"code","source":"#few_shot.callbacks 4\n\n\"\"\"\nPorts of Callback classes from the Keras library.\n\"\"\"\nfrom tqdm import tqdm\nimport numpy as np\nimport torch\nfrom collections import OrderedDict, Iterable\nimport warnings\nimport os\nimport csv\nimport io\n\n#from few_shot.eval import evaluate\n\n\nclass CallbackList(object):\n    \"\"\"Container abstracting a list of callbacks.\n    # Arguments\n        callbacks: List of `Callback` instances.\n    \"\"\"\n    def __init__(self, callbacks):\n        self.callbacks = [c for c in callbacks]\n\n    def set_params(self, params):\n        for callback in self.callbacks:\n            callback.set_params(params)\n\n    def set_model(self, model):\n        for callback in self.callbacks:\n            callback.set_model(model)\n\n    def on_epoch_begin(self, epoch, logs=None):\n        \"\"\"Called at the start of an epoch.\n        # Arguments\n            epoch: integer, index of epoch.\n            logs: dictionary of logs.\n        \"\"\"\n        logs = logs or {}\n        for callback in self.callbacks:\n            callback.on_epoch_begin(epoch, logs)\n\n    def on_epoch_end(self, epoch, logs=None):\n        \"\"\"Called at the end of an epoch.\n        # Arguments\n            epoch: integer, index of epoch.\n            logs: dictionary of logs.\n        \"\"\"\n        logs = logs or {}\n        for callback in self.callbacks:\n            callback.on_epoch_end(epoch, logs)\n\n    def on_batch_begin(self, batch, logs=None):\n        \"\"\"Called right before processing a batch.\n        # Arguments\n            batch: integer, index of batch within the current epoch.\n            logs: dictionary of logs.\n        \"\"\"\n        logs = logs or {}\n        for callback in self.callbacks:\n            callback.on_batch_begin(batch, logs)\n\n    def on_batch_end(self, batch, logs=None):\n        \"\"\"Called at the end of a batch.\n        # Arguments\n            batch: integer, index of batch within the current epoch.\n            logs: dictionary of logs.\n        \"\"\"\n        logs = logs or {}\n        for callback in self.callbacks:\n            callback.on_batch_end(batch, logs)\n\n    def on_train_begin(self, logs=None):\n        \"\"\"Called at the beginning of training.\n        # Arguments\n            logs: dictionary of logs.\n        \"\"\"\n        logs = logs or {}\n        for callback in self.callbacks:\n            callback.on_train_begin(logs)\n\n    def on_train_end(self, logs=None):\n        \"\"\"Called at the end of training.\n        # Arguments\n            logs: dictionary of logs.\n        \"\"\"\n        logs = logs or {}\n        for callback in self.callbacks:\n            callback.on_train_end(logs)\n\n\nclass Callback(object):\n    def __init__(self):\n        self.model = None\n\n    def set_params(self, params):\n        self.params = params\n\n    def set_model(self, model):\n        self.model = model\n\n    def on_epoch_begin(self, epoch, logs=None):\n        pass\n\n    def on_epoch_end(self, epoch, logs=None):\n        pass\n\n    def on_batch_begin(self, batch, logs=None):\n        pass\n\n    def on_batch_end(self, batch, logs=None):\n        pass\n\n    def on_train_begin(self, logs=None):\n        pass\n\n    def on_train_end(self, logs=None):\n        pass\n\n\nclass DefaultCallback(Callback):\n    \"\"\"Records metrics over epochs by averaging over each batch.\n    NB The metrics are calculated with a moving model\n    \"\"\"\n    def on_epoch_begin(self, batch, logs=None):\n        self.seen = 0\n        self.totals = {}\n        self.metrics = ['loss'] + self.params['metrics']\n\n    def on_batch_end(self, batch, logs=None):\n        logs = logs or {}\n        batch_size = logs.get('size', 1) or 1\n        self.seen += batch_size\n\n        for k, v in logs.items():\n            if k in self.totals:\n                self.totals[k] += v * batch_size\n            else:\n                self.totals[k] = v * batch_size\n\n    def on_epoch_end(self, epoch, logs=None):\n        if logs is not None:\n            for k in self.metrics:\n                if k in self.totals:\n                    # Make value available to next callbacks.\n                    logs[k] = self.totals[k] / self.seen\n\n\nclass ProgressBarLogger(Callback):\n    \"\"\"TQDM progress bar that displays the running average of loss and other metrics.\"\"\"\n    def __init__(self):\n        super(ProgressBarLogger, self).__init__()\n\n    def on_train_begin(self, logs=None):\n        self.num_batches = self.params['num_batches']\n        self.verbose = self.params['verbose']\n        self.metrics = ['loss'] + self.params['metrics']\n        self.epoch_metrics = self.params['epoch_metrics']\n\n    def on_epoch_begin(self, epoch, logs=None):\n        self.target = self.num_batches\n        self.pbar = tqdm(total=self.target, desc='Epoch {}'.format(epoch))\n        self.seen = 0\n\n    def on_batch_begin(self, batch, logs=None):\n        self.log_values = {}\n\n    def on_batch_end(self, batch, logs=None):\n        logs = logs or {}\n        self.seen += 1\n\n        for k in self.metrics:\n            if k in logs:\n                self.log_values[k] = logs[k]\n\n        # Skip progbar update for the last batch;\n        # will be handled by on_epoch_end.\n        if self.verbose and self.seen < self.target:\n            self.pbar.update(1)\n            self.pbar.set_postfix(self.log_values)\n\n    def on_epoch_end(self, epoch, logs=None):\n        # Update log values\n        self.log_values = {}\n        for k in self.metrics + self.epoch_metrics:\n            if k in logs:\n                self.log_values[k] = logs[k]\n\n        if self.verbose:\n            self.pbar.update(1)\n            self.pbar.set_postfix(self.log_values)\n\n        self.pbar.close()\n\n\nclass CSVLogger(Callback):\n    \"\"\"Callback that streams epoch results to a csv file.\n    Supports all values that can be represented as a string,\n    including 1D iterables such as np.ndarray.\n    # Arguments\n        filename: filename of the csv file, e.g. 'run/log.csv'.\n        separator: string used to separate elements in the csv file.\n        append: True: append if file exists (useful for continuing\n            training). False: overwrite existing file,\n    \"\"\"\n\n    def __init__(self, filename, separator=',', append=False):\n        self.sep = separator\n        self.filename = filename\n        self.append = append\n        self.writer = None\n        self.keys = None\n        self.append_header = True\n        self.file_flags = ''\n        self._open_args = {'newline': '\\n'}\n        super(CSVLogger, self).__init__()\n\n    def on_train_begin(self, logs=None):\n        if self.append:\n            if os.path.exists(self.filename):\n                with open(self.filename, 'r' + self.file_flags) as f:\n                    self.append_header = not bool(len(f.readline()))\n            mode = 'a'\n        else:\n            mode = 'w'\n\n        self.csv_file = open(self.filename,mode + self.file_flags,**self._open_args)\n\n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n\n        def handle_value(k):\n            is_zero_dim_ndarray = isinstance(k, np.ndarray) and k.ndim == 0\n            if isinstance(k, str):\n                return k\n            elif isinstance(k, Iterable) and not is_zero_dim_ndarray:\n                return '\"[%s]\"' % (', '.join(map(str, k)))\n            else:\n                return k\n\n        if self.keys is None:\n            self.keys = sorted(logs.keys())\n\n        if not self.writer:\n            class CustomDialect(csv.excel):\n                delimiter = self.sep\n            fieldnames = ['epoch'] + self.keys\n            self.writer = csv.DictWriter(self.csv_file,\n                                         fieldnames=fieldnames,\n                                         dialect=CustomDialect)\n            if self.append_header:\n                self.writer.writeheader()\n\n        row_dict = OrderedDict({'epoch': epoch})\n        row_dict.update((key, handle_value(logs[key])) for key in self.keys)\n        self.writer.writerow(row_dict)\n        self.csv_file.flush()\n\n    def on_train_end(self, logs=None):\n        self.csv_file.close()\n        self.writer = None\n\n\nclass EvaluateMetrics(Callback):\n    \"\"\"Evaluates metrics on a dataset after every epoch.\n    # Argments\n        dataloader: torch.DataLoader of the dataset on which the model will be evaluated\n        prefix: Prefix to prepend to the names of the metrics when they is logged. Defaults to 'val_' but can be changed\n        if the model is to be evaluated on many datasets separately.\n        suffix: Suffix to append to the names of the metrics when they is logged.\n    \"\"\"\n    def __init__(self, dataloader, prefix='val_', suffix=''):\n        super(EvaluateMetrics, self).__init__()\n        self.dataloader = dataloader\n        self.prefix = prefix\n        self.suffix = suffix\n\n    def on_train_begin(self, logs=None):\n        self.metrics = self.params['metrics']\n        self.prepare_batch = self.params['prepare_batch']\n        self.loss_fn = self.params['loss_fn']\n\n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        logs.update(\n            evaluate(self.model, self.dataloader, self.prepare_batch, self.metrics, self.loss_fn, self.prefix, self.suffix)\n        )\n\n\nclass ReduceLROnPlateau(Callback):\n    \"\"\"Reduce learning rate when a metric has stopped improving.\n    Models often benefit from reducing the learning rate by a factor\n    of 2-10 once learning stagnates. This callback monitors a\n    quantity and if no improvement is seen for a 'patience' number\n    of epochs, the learning rate is reduced.\n    # Arguments\n        monitor: quantity to be monitored.\n        factor: factor by which the learning rate will\n            be reduced. new_lr = lr * factor\n        patience: number of epochs with no improvement\n            after which learning rate will be reduced.\n        verbose: int. 0: quiet, 1: update messages.\n        mode: one of {auto, min, max}. In `min` mode,\n            lr will be reduced when the quantity\n            monitored has stopped decreasing; in `max`\n            mode it will be reduced when the quantity\n            monitored has stopped increasing; in `auto`\n            mode, the direction is automatically inferred\n            from the name of the monitored quantity.\n        min_delta: threshold for measuring the new optimum,\n            to only focus on significant changes.\n        cooldown: number of epochs to wait before resuming\n            normal operation after lr has been reduced.\n        min_lr: lower bound on the learning rate.\n    \"\"\"\n\n    def __init__(self, monitor='val_loss', factor=0.1, patience=10,\n                 verbose=0, mode='auto', min_delta=1e-4, cooldown=0, min_lr=0,\n                 **kwargs):\n        super(ReduceLROnPlateau, self).__init__()\n\n        self.monitor = monitor\n        if factor >= 1.0:\n            raise ValueError('ReduceLROnPlateau does not support a factor >= 1.0.')\n        self.factor = factor\n        self.min_lr = min_lr\n        self.min_delta = min_delta\n        self.patience = patience\n        self.verbose = verbose\n        self.cooldown = cooldown\n        self.cooldown_counter = 0  # Cooldown counter.\n        self.wait = 0\n        self.best = 0\n        if mode not in ['auto', 'min', 'max']:\n            raise ValueError('Mode must be one of (auto, min, max).')\n        self.mode = mode\n        self.monitor_op = None\n\n        self._reset()\n\n    def _reset(self):\n        \"\"\"Resets wait counter and cooldown counter.\n        \"\"\"\n        if (self.mode == 'min' or\n                (self.mode == 'auto' and 'acc' not in self.monitor)):\n            self.monitor_op = lambda a, b: np.less(a, b - self.min_delta)\n            self.best = np.Inf\n        else:\n            self.monitor_op = lambda a, b: np.greater(a, b + self.min_delta)\n            self.best = -np.Inf\n        self.cooldown_counter = 0\n        self.wait = 0\n\n    def on_train_begin(self, logs=None):\n        self.optimiser = self.params['optimiser']\n        self.min_lrs = [self.min_lr] * len(self.optimiser.param_groups)\n        self._reset()\n\n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        if len(self.optimiser.param_groups) == 1:\n            logs['lr'] = self.optimiser.param_groups[0]['lr']\n        else:\n            for i, param_group in enumerate(self.optimiser.param_groups):\n                logs['lr_{}'.format(i)] = param_group['lr']\n\n        current = logs.get(self.monitor)\n\n        if self.in_cooldown():\n            self.cooldown_counter -= 1\n            self.wait = 0\n\n        if self.monitor_op(current, self.best):\n            self.best = current\n            self.wait = 0\n        elif not self.in_cooldown():\n            self.wait += 1\n            if self.wait >= self.patience:\n                self._reduce_lr(epoch)\n                self.cooldown_counter = self.cooldown\n                self.wait = 0\n\n    def _reduce_lr(self, epoch):\n        for i, param_group in enumerate(self.optimiser.param_groups):\n            old_lr = float(param_group['lr'])\n            new_lr = max(old_lr * self.factor, self.min_lrs[i])\n            if old_lr - new_lr > self.min_delta:\n                param_group['lr'] = new_lr\n                if self.verbose:\n                    print('Epoch {:5d}: reducing learning rate'\n                          ' of group {} to {:.4e}.'.format(epoch, i, new_lr))\n\n    def in_cooldown(self):\n        return self.cooldown_counter > 0\n\n\nclass ModelCheckpoint(Callback):\n    \"\"\"Save the model after every epoch.\n    `filepath` can contain named formatting options, which will be filled the value of `epoch` and keys in `logs`\n    (passed in `on_epoch_end`).\n    For example: if `filepath` is `weights.{epoch:02d}-{val_loss:.2f}.hdf5`, then the model checkpoints will be saved\n    with the epoch number and the validation loss in the filename.\n    # Arguments\n        filepath: string, path to save the model file.\n        monitor: quantity to monitor.\n        verbose: verbosity mode, 0 or 1.\n        save_best_only: if `save_best_only=True`,\n            the latest best model according to\n            the quantity monitored will not be overwritten.\n        mode: one of {auto, min, max}.\n            If `save_best_only=True`, the decision\n            to overwrite the current save file is made\n            based on either the maximization or the\n            minimization of the monitored quantity. For `val_acc`,\n            this should be `max`, for `val_loss` this should\n            be `min`, etc. In `auto` mode, the direction is\n            automatically inferred from the name of the monitored quantity.\n        save_weights_only: if True, then only the model's weights will be\n            saved (`model.save_weights(filepath)`), else the full model\n            is saved (`model.save(filepath)`).\n        period: Interval (number of epochs) between checkpoints.\n    \"\"\"\n\n    def __init__(self, filepath, monitor='val_loss', verbose=0, save_best_only=False, mode='auto', period=1):\n        super(ModelCheckpoint, self).__init__()\n        self.monitor = monitor\n        self.verbose = verbose\n        self.filepath = filepath\n        self.save_best_only = save_best_only\n        self.period = period\n        self.epochs_since_last_save = 0\n\n        if mode not in ['auto', 'min', 'max']:\n            raise ValueError('Mode must be one of (auto, min, max).')\n\n        if mode == 'min':\n            self.monitor_op = np.less\n            self.best = np.Inf\n        elif mode == 'max':\n            self.monitor_op = np.greater\n            self.best = -np.Inf\n        else:\n            if 'acc' in self.monitor or self.monitor.startswith('fmeasure'):\n                self.monitor_op = np.greater\n                self.best = -np.Inf\n            else:\n                self.monitor_op = np.less\n\n        self.best = np.Inf\n\n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        self.epochs_since_last_save += 1\n        if self.epochs_since_last_save >= self.period:\n            self.epochs_since_last_save = 0\n            filepath = self.filepath.format(epoch=epoch + 1, **logs)\n            if self.save_best_only:\n                current = logs.get(self.monitor)\n                if current is None:\n                    warnings.warn('Can save best model only with %s available, '\n                                  'skipping.' % (self.monitor), RuntimeWarning)\n                else:\n                    if self.monitor_op(current, self.best):\n                        if self.verbose > 0:\n                            print('\\nEpoch %05d: %s improved from %0.5f to %0.5f,'\n                                  ' saving model to %s'\n                                  % (epoch + 1, self.monitor, self.best,\n                                     current, filepath))\n                        self.best = current\n                        torch.save(self.model.state_dict(), filepath)\n                    else:\n                        if self.verbose > 0:\n                            print('\\nEpoch %05d: %s did not improve from %0.5f' %\n                                  (epoch + 1, self.monitor, self.best))\n            else:\n                if self.verbose > 0:\n                    print('\\nEpoch %05d: saving model to %s' % (epoch + 1, filepath))\n                torch.save(self.model.state_dict(), filepath)\n\n\nclass LearningRateScheduler(Callback):\n    \"\"\"Learning rate scheduler.\n    # Arguments\n        schedule: a function that takes an epoch index as input\n            (integer, indexed from 0) and current learning rate\n            and returns a new learning rate as output (float).\n        verbose: int. 0: quiet, 1: update messages.\n    \"\"\"\n\n    def __init__(self, schedule, verbose=0):\n        super(LearningRateScheduler, self).__init__()\n        self.schedule = schedule\n        self.verbose = verbose\n\n    def on_train_begin(self, logs=None):\n        self.optimiser = self.params['optimiser']\n\n    def on_epoch_begin(self, epoch, logs=None):\n        lrs = [self.schedule(epoch, param_group['lr']) for param_group in self.optimiser.param_groups]\n\n        if not all(isinstance(lr, (float, np.float32, np.float64)) for lr in lrs):\n            raise ValueError('The output of the \"schedule\" function '\n                             'should be float.')\n        self.set_lr(epoch, lrs)\n\n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        if len(self.optimiser.param_groups) == 1:\n            logs['lr'] = self.optimiser.param_groups[0]['lr']\n        else:\n            for i, param_group in enumerate(self.optimiser.param_groups):\n                logs['lr_{}'.format(i)] = param_group['lr']\n\n    def set_lr(self, epoch, lrs):\n        for i, param_group in enumerate(self.optimiser.param_groups):\n            new_lr = lrs[i]\n            param_group['lr'] = new_lr\n            if self.verbose:\n                print('Epoch {:5d}: setting learning rate'\n' of group {} to {:.4e}.'.format(epoch, i, new_lr))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2d9ed7956554da76aef5cf4485d4d7e6813f185f","trusted":false},"cell_type":"code","source":"!pip install easydict","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b07c9a95afefd59a9818a5cdefb77769802a18fc","trusted":false},"cell_type":"code","source":"#few_shot.core.py 5\n\n\nfrom torch.utils.data import Sampler\nfrom typing import List, Iterable, Callable, Tuple\nimport numpy as np\nimport torch\n\n#from few_shot.metrics import categorical_accuracy\n#from few_shot.callbacks import Callback\n\n\nclass NShotTaskSampler(Sampler):\n    def __init__(self,\n                 dataset: torch.utils.data.Dataset,\n                 episodes_per_epoch: int = None,\n                 n: int = None,\n                 k: int = None,\n                 q: int = None,\n                 num_tasks: int = 1,\n                 fixed_tasks: List[Iterable[int]] = None):\n        \"\"\"PyTorch Sampler subclass that generates batches of n-shot, k-way, q-query tasks.\n        Each n-shot task contains a \"support set\" of `k` sets of `n` samples and a \"query set\" of `k` sets\n        of `q` samples. The support set and the query set are all grouped into one Tensor such that the first n * k\n        samples are from the support set while the remaining q * k samples are from the query set.\n        The support and query sets are sampled such that they are disjoint i.e. do not contain overlapping samples.\n        # Arguments\n            dataset: Instance of torch.utils.data.Dataset from which to draw samples\n            episodes_per_epoch: Arbitrary number of batches of n-shot tasks to generate in one epoch\n            n_shot: int. Number of samples for each class in the n-shot classification tasks.\n            k_way: int. Number of classes in the n-shot classification tasks.\n            q_queries: int. Number query samples for each class in the n-shot classification tasks.\n            num_tasks: Number of n-shot tasks to group into a single batch\n            fixed_tasks: If this argument is specified this Sampler will always generate tasks from\n                the specified classes\n        \"\"\"\n        super(NShotTaskSampler, self).__init__(dataset)\n        self.episodes_per_epoch = episodes_per_epoch\n        self.dataset = dataset\n        if num_tasks < 1:\n            raise ValueError('num_tasks must be > 1.')\n\n        self.num_tasks = num_tasks\n        # TODO: Raise errors if initialise badly\n        self.k = k\n        self.n = n\n        self.q = q\n        self.fixed_tasks = fixed_tasks\n\n        self.i_task = 0\n\n    def __len__(self):\n        return self.episodes_per_epoch\n\n    def __iter__(self):\n        for _ in range(self.episodes_per_epoch):\n            batch = []\n\n            for task in range(self.num_tasks):\n                if self.fixed_tasks is None:\n                    # Get random classes\n                    episode_classes = np.random.choice(self.dataset.df['class_id'].unique(), size=self.k, replace=False)\n                else:\n                    # Loop through classes in fixed_tasks\n                    episode_classes = self.fixed_tasks[self.i_task % len(self.fixed_tasks)]\n                    self.i_task += 1\n\n                df = self.dataset.df[self.dataset.df['class_id'].isin(episode_classes)]\n\n                support_k = {k: None for k in episode_classes}\n                for k in episode_classes:\n                    # Select support examples\n                    support = df[df['class_id'] == k].sample(self.n)\n                    support_k[k] = support\n\n                    for i, s in support.iterrows():\n                        batch.append(s['id'])\n\n                for k in episode_classes:\n                    query = df[(df['class_id'] == k) & (~df['id'].isin(support_k[k]['id']))].sample(self.q)\n                    for i, q in query.iterrows():\n                        batch.append(q['id'])\n\n            yield np.stack(batch)\n\n\nclass EvaluateFewShot(Callback):\n    \"\"\"Evaluate a network on  an n-shot, k-way classification tasks after every epoch.\n    # Arguments\n        eval_fn: Callable to perform few-shot classification. Examples include `proto_net_episode`,\n            `matching_net_episode` and `meta_gradient_step` (MAML).\n        num_tasks: int. Number of n-shot classification tasks to evaluate the model with.\n        n_shot: int. Number of samples for each class in the n-shot classification tasks.\n        k_way: int. Number of classes in the n-shot classification tasks.\n        q_queries: int. Number query samples for each class in the n-shot classification tasks.\n        task_loader: Instance of NShotWrapper class\n        prepare_batch: function. The preprocessing function to apply to samples from the dataset.\n        prefix: str. Prefix to identify dataset.\n    \"\"\"\n\n    def __init__(self,\n                 eval_fn: Callable,\n                 num_tasks: int,\n                 n_shot: int,\n                 k_way: int,\n                 q_queries: int,\n                 taskloader: torch.utils.data.DataLoader,\n                 prepare_batch: Callable,\n                 prefix: str = 'val_',\n                 **kwargs):\n        super(EvaluateFewShot, self).__init__()\n        self.eval_fn = eval_fn\n        self.num_tasks = num_tasks\n        self.n_shot = n_shot\n        self.k_way = k_way\n        self.q_queries = q_queries\n        self.taskloader = taskloader\n        self.prepare_batch = prepare_batch\n        self.prefix = prefix\n        self.kwargs = kwargs\n        self.metric_name = f'{self.prefix}{self.n_shot}-shot_{self.k_way}-way_acc'\n\n    def on_train_begin(self, logs=None):\n        self.loss_fn = self.params['loss_fn']\n        self.optimiser = self.params['optimiser']\n\n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        seen = 0\n        totals = {'loss': 0, self.metric_name: 0}\n        for batch_index, batch in enumerate(self.taskloader):\n            x, y = self.prepare_batch(batch)\n\n            loss, y_pred = self.eval_fn(\n                self.model,\n                self.optimiser,\n                self.loss_fn,\n                x,\n                y,\n                n_shot=self.n_shot,\n                k_way=self.k_way,\n                q_queries=self.q_queries,\n                train=False,\n                **self.kwargs\n            )\n\n            seen += y_pred.shape[0]\n\n            totals['loss'] += loss.item() * y_pred.shape[0]\n            totals[self.metric_name] += categorical_accuracy(y, y_pred) * y_pred.shape[0]\n\n        logs[self.prefix + 'loss'] = totals['loss'] / seen\n        logs[self.metric_name] = totals[self.metric_name] / seen\n\n\ndef prepare_nshot_task(n: int, k: int, q: int) -> Callable:\n    \"\"\"Typical n-shot task preprocessing.\n    # Arguments\n        n: Number of samples for each class in the n-shot classification task\n        k: Number of classes in the n-shot classification task\n        q: Number of query samples for each class in the n-shot classification task\n    # Returns\n        prepare_nshot_task_: A Callable that processes a few shot tasks with specified n, k and q\n    \"\"\"\n    def prepare_nshot_task_(batch: Tuple[torch.Tensor, torch.Tensor]) -> Tuple[torch.Tensor, torch.Tensor]:\n        \"\"\"Create 0-k label and move to GPU.\n        TODO: Move to arbitrary device\n        \"\"\"\n        x, y = batch\n        x = x.float().cuda()\n        # Create dummy 0-(num_classes - 1) label\n        y = create_nshot_task_label(k, q).cuda()\n        return x, y\n\n    return prepare_nshot_task_\n\n\ndef create_nshot_task_label(k: int, q: int) -> torch.Tensor:\n    \"\"\"Creates an n-shot task label.\n    Label has the structure:\n        [0]*q + [1]*q + ... + [k-1]*q\n    # TODO: Test this\n    # Arguments\n        k: Number of classes in the n-shot classification task\n        q: Number of query samples for each class in the n-shot classification task\n    # Returns\n        y: Label vector for n-shot task of shape [q * k, ]\n    \"\"\"\n    y = torch.arange(0, k, 1 / q).long()\n    return y","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fb5c4c8dcdc79527a2c9a854c9b5e9fd4662ef85","trusted":false},"cell_type":"code","source":"#few_shot.train.py\n\n\n\"\"\"\nThe `fit` function in this file implements a slightly modified version\nof the Keras `model.fit()` API.\n\"\"\"\nimport torch\nfrom torch.optim import Optimizer\nfrom torch.nn import Module\nfrom torch.utils.data import DataLoader\nfrom typing import Callable, List, Union\n\n#from few_shot.callbacks import DefaultCallback, ProgressBarLogger, CallbackList, Callback\n#from few_shot.metrics import NAMED_METRICS\n\n\ndef gradient_step(model: Module, optimiser: Optimizer, loss_fn: Callable, x: torch.Tensor, y: torch.Tensor, **kwargs):\n    \"\"\"Takes a single gradient step.\n    # Arguments\n        model: Model to be fitted\n        optimiser: Optimiser to calculate gradient step from loss\n        loss_fn: Loss function to calculate between predictions and outputs\n        x: Input samples\n        y: Input targets\n    \"\"\"\n    model.train()\n    optimiser.zero_grad()\n    y_pred = model(x)\n    loss = loss_fn(y_pred, y)\n    loss.backward()\n    optimiser.step()\n\n    return loss, y_pred\n\n\ndef batch_metrics(model: Module, y_pred: torch.Tensor, y: torch.Tensor, metrics: List[Union[str, Callable]],\n                  batch_logs: dict):\n    \"\"\"Calculates metrics for the current training batch\n    # Arguments\n        model: Model being fit\n        y_pred: predictions for a particular batch\n        y: labels for a particular batch\n        batch_logs: Dictionary of logs for the current batch\n    \"\"\"\n    model.eval()\n    for m in metrics:\n        if isinstance(m, str):\n            batch_logs[m] = NAMED_METRICS[m](y, y_pred)\n        else:\n            # Assume metric is a callable function\n            batch_logs = m(y, y_pred)\n\n    return batch_logs\n\n\ndef fit(model: Module, optimiser: Optimizer, loss_fn: Callable, epochs: int, dataloader: DataLoader,\n        prepare_batch: Callable, metrics: List[Union[str, Callable]] = None,\n        epoch_metrics: List[str] = None, callbacks: List[Callback] = None,\n        verbose: bool =True, fit_function: Callable = gradient_step, fit_function_kwargs: dict = {}):\n    \"\"\"Function to abstract away training loop.\n    The benefit of this function is that allows training scripts to be much more readable and allows for easy re-use of\n    common training functionality provided they are written as a subclass of voicemap.Callback (following the\n    Keras API).\n    # Arguments\n        model: Model to be fitted.\n        optimiser: Optimiser to calculate gradient step from loss\n        loss_fn: Loss function to calculate between predictions and outputs\n        epochs: Number of epochs of fitting to be performed\n        dataloader: `torch.DataLoader` instance to fit the model to\n        prepare_batch: Callable to perform any desired preprocessing\n        metrics: Optional list of metrics to evaluate the model with\n        epoch_metrics: Optional list of metrics on top of metrics at the end of epoch\n        callbacks: Additional functionality to incorporate into training such as logging metrics to csv, model\n            checkpointing, learning rate scheduling etc... See voicemap.callbacks for more.\n        verbose: All print output is muted if this argument is `False`\n        fit_function: Function for calculating gradients. Leave as default for simple supervised training on labelled\n            batches. For more complex training procedures (meta-learning etc...) you will need to write your own\n            fit_function\n        fit_function_kwargs: Keyword arguments to pass to `fit_function`\n    \"\"\"\n    # Determine number of samples:\n    num_batches = len(dataloader)\n    batch_size = dataloader.batch_size\n\n    callbacks = CallbackList([DefaultCallback(), ] + (callbacks or []) + [ProgressBarLogger(), ])\n    callbacks.set_model(model)\n    callbacks.set_params({\n        'num_batches': num_batches,\n        'batch_size': batch_size,\n        'verbose': verbose,\n        'metrics': (metrics or []),\n        'epoch_metrics': (epoch_metrics or []),\n        'prepare_batch': prepare_batch,\n        'loss_fn': loss_fn,\n        'optimiser': optimiser\n    })\n\n    if verbose:\n        print('Begin training...')\n\n    callbacks.on_train_begin()\n\n    for epoch in range(1, epochs+1):\n        callbacks.on_epoch_begin(epoch)\n\n        epoch_logs = {}\n        for batch_index, batch in enumerate(dataloader):\n            batch_logs = dict(batch=batch_index, size=(batch_size or 1))\n\n            callbacks.on_batch_begin(batch_index, batch_logs)\n\n            x, y = prepare_batch(batch)\n\n            loss, y_pred = fit_function(model, optimiser, loss_fn, x, y, **fit_function_kwargs)\n            batch_logs['loss'] = loss.item()\n\n            # Loops through all metrics\n            batch_logs = batch_metrics(model, y_pred, y, metrics, batch_logs)\n\n            callbacks.on_batch_end(batch_index, batch_logs)\n\n        # Run on epoch end\n        callbacks.on_epoch_end(epoch, epoch_logs)\n\n    # Run on train end\n    if verbose:\n        print('Finished.')\n    callbacks.on_train_end()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ad94aea42893b2f8fb7526a0c226e19b45176470","trusted":false},"cell_type":"code","source":"#few_shot.proto.py\n\nimport torch\nfrom torch.optim import Optimizer\nfrom torch.nn import Module\nfrom typing import Callable\n\n#from few_shot.utils import pairwise_distances\n\n\ndef proto_net_episode(model: Module,\n                      optimiser: Optimizer,\n                      loss_fn: Callable,\n                      x: torch.Tensor,\n                      y: torch.Tensor,\n                      n_shot: int,\n                      k_way: int,\n                      q_queries: int,\n                      distance: str,\n                      train: bool):\n    \"\"\"Performs a single training episode for a Prototypical Network.\n    # Arguments\n        model: Prototypical Network to be trained.\n        optimiser: Optimiser to calculate gradient step\n        loss_fn: Loss function to calculate between predictions and outputs. Should be cross-entropy\n        x: Input samples of few shot classification task\n        y: Input labels of few shot classification task\n        n_shot: Number of examples per class in the support set\n        k_way: Number of classes in the few shot classification task\n        q_queries: Number of examples per class in the query set\n        distance: Distance metric to use when calculating distance between class prototypes and queries\n        train: Whether (True) or not (False) to perform a parameter update\n    # Returns\n        loss: Loss of the Prototypical Network on this task\n        y_pred: Predicted class probabilities for the query set on this task\n    \"\"\"\n    if train:\n        # Zero gradients\n        model.train()\n        optimiser.zero_grad()\n    else:\n        model.eval()\n\n    # Embed all samples\n    embeddings = model(x)\n\n    # Samples are ordered by the NShotWrapper class as follows:\n    # k lots of n support samples from a particular class\n    # k lots of q query samples from those classes\n    support = embeddings[:n_shot*k_way]\n    queries = embeddings[n_shot*k_way:]\n    prototypes = compute_prototypes(support, k_way, n_shot)\n\n    # Calculate squared distances between all queries and all prototypes\n    # Output should have shape (q_queries * k_way, k_way) = (num_queries, k_way)\n    distances = pairwise_distances(queries, prototypes, distance)\n\n    # Calculate log p_{phi} (y = k | x)\n    log_p_y = (-distances).log_softmax(dim=1)\n    loss = loss_fn(log_p_y, y)\n\n    # Prediction probabilities are softmax over distances\n    y_pred = (-distances).softmax(dim=1)\n\n    if train:\n        # Take gradient step\n        loss.backward()\n        optimiser.step()\n    else:\n        pass\n\n    return loss, y_pred\n\n\ndef compute_prototypes(support: torch.Tensor, k: int, n: int) -> torch.Tensor:\n    \"\"\"Compute class prototypes from support samples.\n    # Arguments\n        support: torch.Tensor. Tensor of shape (n * k, d) where d is the embedding\n            dimension.\n        k: int. \"k-way\" i.e. number of classes in the classification task\n        n: int. \"n-shot\" of the classification task\n    # Returns\n        class_prototypes: Prototypes aka mean embeddings for each class\n    \"\"\"\n    # Reshape so the first dimension indexes by class then take the mean\n    # along that dimension to generate the \"prototypes\" for each class\n    class_prototypes = support.reshape(k, n, -1).mean(dim=1)\n    return class_prototypes","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e7717cf4d7ffd61d3d47a138628ae4af3bb8b6f0","trusted":false},"cell_type":"code","source":"#few_shot.models\n\nfrom torch import nn\nimport numpy as np\nimport torch.nn.functional as F\nimport torch\nfrom typing import Dict\n\n\n##########\n# Layers #\n##########\nclass Flatten(nn.Module):\n    \"\"\"Converts N-dimensional Tensor of shape [batch_size, d1, d2, ..., dn] to 2-dimensional Tensor\n    of shape [batch_size, d1*d2*...*dn].\n    # Arguments\n        input: Input tensor\n    \"\"\"\n    def forward(self, input):\n        return input.view(input.size(0), -1)\n\n\nclass GlobalMaxPool1d(nn.Module):\n    \"\"\"Performs global max pooling over the entire length of a batched 1D tensor\n    # Arguments\n        input: Input tensor\n    \"\"\"\n    def forward(self, input):\n        return nn.functional.max_pool1d(input, kernel_size=input.size()[2:]).view(-1, input.size(1))\n\n\nclass GlobalAvgPool2d(nn.Module):\n    \"\"\"Performs global average pooling over the entire height and width of a batched 2D tensor\n    # Arguments\n        input: Input tensor\n    \"\"\"\n    def forward(self, input):\n        return nn.functional.avg_pool2d(input, kernel_size=input.size()[2:]).view(-1, input.size(1))\n\n\ndef conv_block(in_channels: int, out_channels: int) -> nn.Module:\n    \"\"\"Returns a Module that performs 3x3 convolution, ReLu activation, 2x2 max pooling.\n    # Arguments\n        in_channels:\n        out_channels:\n    \"\"\"\n    return nn.Sequential(\n        nn.Conv2d(in_channels, out_channels, 3, padding=1),\n        nn.BatchNorm2d(out_channels),\n        nn.ReLU(),\n        nn.MaxPool2d(kernel_size=2, stride=2)\n    )\n\n\ndef functional_conv_block(x: torch.Tensor, weights: torch.Tensor, biases: torch.Tensor,\n                          bn_weights, bn_biases) -> torch.Tensor:\n    \"\"\"Performs 3x3 convolution, ReLu activation, 2x2 max pooling in a functional fashion.\n    # Arguments:\n        x: Input Tensor for the conv block\n        weights: Weights for the convolutional block\n        biases: Biases for the convolutional block\n        bn_weights:\n        bn_biases:\n    \"\"\"\n    x = F.conv2d(x, weights, biases, padding=1)\n    x = F.batch_norm(x, running_mean=None, running_var=None, weight=bn_weights, bias=bn_biases, training=True)\n    x = F.relu(x)\n    x = F.max_pool2d(x, kernel_size=2, stride=2)\n    return x\n\n\n##########\n# Models #\n##########\ndef get_few_shot_encoder(num_input_channels=1) -> nn.Module:\n    \"\"\"Creates a few shot encoder as used in Matching and Prototypical Networks\n    # Arguments:\n        num_input_channels: Number of color channels the model expects input data to contain. Omniglot = 1,\n            miniImageNet = 3\n    \"\"\"\n    return nn.Sequential(\n        conv_block(num_input_channels, 64),\n        conv_block(64, 64),\n        conv_block(64, 64),\n        conv_block(64, 64),\n        Flatten(),\n    )\n\n\nclass FewShotClassifier(nn.Module):\n    def __init__(self, num_input_channels: int, k_way: int, final_layer_size: int = 64):\n        \"\"\"Creates a few shot classifier as used in MAML.\n        This network should be identical to the one created by `get_few_shot_encoder` but with a\n        classification layer on top.\n        # Arguments:\n            num_input_channels: Number of color channels the model expects input data to contain. Omniglot = 1,\n                miniImageNet = 3\n            k_way: Number of classes the model will discriminate between\n            final_layer_size: 64 for Omniglot, 1600 for miniImageNet\n        \"\"\"\n        super(FewShotClassifier, self).__init__()\n        self.conv1 = conv_block(num_input_channels, 64)\n        self.conv2 = conv_block(64, 64)\n        self.conv3 = conv_block(64, 64)\n        self.conv4 = conv_block(64, 64)\n\n        self.logits = nn.Linear(final_layer_size, k_way)\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.conv2(x)\n        x = self.conv3(x)\n        x = self.conv4(x)\n\n        x = x.view(x.size(0), -1)\n\n        return self.logits(x)\n\n    def functional_forward(self, x, weights):\n        \"\"\"Applies the same forward pass using PyTorch functional operators using a specified set of weights.\"\"\"\n\n        for block in [1, 2, 3, 4]:\n            x = functional_conv_block(x, weights[f'conv{block}.0.weight'], weights[f'conv{block}.0.bias'],\n                                      weights.get(f'conv{block}.1.weight'), weights.get(f'conv{block}.1.bias'))\n\n        x = x.view(x.size(0), -1)\n\n        x = F.linear(x, weights['logits.weight'], weights['logits.bias'])\n\n        return x\n\n\nclass MatchingNetwork(nn.Module):\n    def __init__(self, n: int, k: int, q: int, fce: bool, num_input_channels: int,\n                 lstm_layers: int, lstm_input_size: int, unrolling_steps: int, device: torch.device):\n        \"\"\"Creates a Matching Network as described in Vinyals et al.\n        # Arguments:\n            n: Number of examples per class in the support set\n            k: Number of classes in the few shot classification task\n            q: Number of examples per class in the query set\n            fce: Whether or not to us fully conditional embeddings\n            num_input_channels: Number of color channels the model expects input data to contain. Omniglot = 1,\n                miniImageNet = 3\n            lstm_layers: Number of LSTM layers in the bidrectional LSTM g that embeds the support set (fce = True)\n            lstm_input_size: Input size for the bidirectional and Attention LSTM. This is determined by the embedding\n                dimension of the few shot encoder which is in turn determined by the size of the input data. Hence we\n                have Omniglot -> 64, miniImageNet -> 1600.\n            unrolling_steps: Number of unrolling steps to run the Attention LSTM\n            device: Device on which to run computation\n        \"\"\"\n        super(MatchingNetwork, self).__init__()\n        self.n = n\n        self.k = k\n        self.q = q\n        self.fce = fce\n        self.num_input_channels = num_input_channels\n        self.encoder = get_few_shot_encoder(self.num_input_channels)\n        if self.fce:\n            self.g = BidrectionalLSTM(lstm_input_size, lstm_layers).to(device, dtype=torch.float)\n            self.f = AttentionLSTM(lstm_input_size, unrolling_steps=unrolling_steps).to(device, dtype=torch.float)\n\n    def forward(self, inputs):\n        pass\n\n\nclass BidrectionalLSTM(nn.Module):\n    def __init__(self, size: int, layers: int):\n        \"\"\"Bidirectional LSTM used to generate fully conditional embeddings (FCE) of the support set as described\n        in the Matching Networks paper.\n        # Arguments\n            size: Size of input and hidden layers. These are constrained to be the same in order to implement the skip\n                connection described in Appendix A.2\n            layers: Number of LSTM layers\n        \"\"\"\n        super(BidrectionalLSTM, self).__init__()\n        self.num_layers = layers\n        self.batch_size = 1\n        # Force input size and hidden size to be the same in order to implement\n        # the skip connection as described in Appendix A.1 and A.2 of Matching Networks\n        self.lstm = nn.LSTM(input_size=size,\n                            num_layers=layers,\n                            hidden_size=size,\n                            bidirectional=True)\n\n    def forward(self, inputs):\n        # Give None as initial state and Pytorch LSTM creates initial hidden states\n        output, (hn, cn) = self.lstm(inputs, None)\n\n        forward_output = output[:, :, :self.lstm.hidden_size]\n        backward_output = output[:, :, self.lstm.hidden_size:]\n\n        # g(x_i, S) = h_forward_i + h_backward_i + g'(x_i) as written in Appendix A.2\n        # AKA A skip connection between inputs and outputs is used\n        output = forward_output + backward_output + inputs\n        return output, hn, cn\n\n\nclass AttentionLSTM(nn.Module):\n    def __init__(self, size: int, unrolling_steps: int):\n        \"\"\"Attentional LSTM used to generate fully conditional embeddings (FCE) of the query set as described\n        in the Matching Networks paper.\n        # Arguments\n            size: Size of input and hidden layers. These are constrained to be the same in order to implement the skip\n                connection described in Appendix A.2\n            unrolling_steps: Number of steps of attention over the support set to compute. Analogous to number of\n                layers in a regular LSTM\n        \"\"\"\n        super(AttentionLSTM, self).__init__()\n        self.unrolling_steps = unrolling_steps\n        self.lstm_cell = nn.LSTMCell(input_size=size,\n                                     hidden_size=size)\n\n    def forward(self, support, queries):\n        # Get embedding dimension, d\n        if support.shape[-1] != queries.shape[-1]:\n            raise(ValueError(\"Support and query set have different embedding dimension!\"))\n\n        batch_size = queries.shape[0]\n        embedding_dim = queries.shape[1]\n\n        h_hat = torch.zeros_like(queries).cuda().float()\n        c = torch.zeros(batch_size, embedding_dim).cuda().float()\n\n        for k in range(self.unrolling_steps):\n            # Calculate hidden state cf. equation (4) of appendix A.2\n            h = h_hat + queries\n\n            # Calculate softmax attentions between hidden states and support set embeddings\n            # cf. equation (6) of appendix A.2\n            attentions = torch.mm(h, support.t())\n            attentions = attentions.softmax(dim=1)\n\n            # Calculate readouts from support set embeddings cf. equation (5)\n            readout = torch.mm(attentions, support)\n\n            # Run LSTM cell cf. equation (3)\n            # h_hat, c = self.lstm_cell(queries, (torch.cat([h, readout], dim=1), c))\n            h_hat, c = self.lstm_cell(queries, (h + readout, c))\n\n        h = h_hat + queries\n        return h","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a6a3c0fde96ef0004e4059ef232cba1bf4bfc4ba","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3800edf13ba994169d4a774786dcc99dd3d02c94","trusted":false},"cell_type":"code","source":"#few_shot.matching\n\nimport torch\nfrom torch.nn.utils import clip_grad_norm_\nfrom torch.optim import Optimizer\nfrom torch.nn import Module\nfrom torch.nn.modules.loss import _Loss as Loss\n\n#from config import EPSILON\n#from few_shot.core import create_nshot_task_label\n#from few_shot.utils import pairwise_distances\n\n\ndef matching_net_episode(model: Module,\n                         optimiser: Optimizer,\n                         loss_fn: Loss,\n                         x: torch.Tensor,\n                         y: torch.Tensor,\n                         n_shot: int,\n                         k_way: int,\n                         q_queries: int,\n                         distance: str,\n                         fce: bool,\n                         train: bool):\n    \"\"\"Performs a single training episode for a Matching Network.\n    # Arguments\n        model: Matching Network to be trained.\n        optimiser: Optimiser to calculate gradient step from loss\n        loss_fn: Loss function to calculate between predictions and outputs\n        x: Input samples of few shot classification task\n        y: Input labels of few shot classification task\n        n_shot: Number of examples per class in the support set\n        k_way: Number of classes in the few shot classification task\n        q_queries: Number of examples per class in the query set\n        distance: Distance metric to use when calculating distance between support and query set samples\n        fce: Whether or not to us fully conditional embeddings\n        train: Whether (True) or not (False) to perform a parameter update\n    # Returns\n        loss: Loss of the Matching Network on this task\n        y_pred: Predicted class probabilities for the query set on this task\n    \"\"\"\n    if train:\n        # Zero gradients\n        model.train()\n        optimiser.zero_grad()\n    else:\n        model.eval()\n\n    # Embed all samples\n    embeddings = model.encoder(x)\n\n    # Samples are ordered by the NShotWrapper class as follows:\n    # k lots of n support samples from a particular class\n    # k lots of q query samples from those classes\n    support = embeddings[:n_shot * k_way]\n    queries = embeddings[n_shot * k_way:]\n\n    # Optionally apply full context embeddings\n    if fce:\n        # LSTM requires input of shape (seq_len, batch, input_size). `support` is of\n        # shape (k_way * n_shot, embedding_dim) and we want the LSTM to treat the\n        # support set as a sequence so add a single dimension to transform support set\n        # to the shape (k_way * n_shot, 1, embedding_dim) and then remove the batch dimension\n        # afterwards\n\n        # Calculate the fully conditional embedding, g, for support set samples as described\n        # in appendix A.2 of the paper. g takes the form of a bidirectional LSTM with a\n        # skip connection from inputs to outputs\n        support, _, _ = model.g(support.unsqueeze(1))\n        support = support.squeeze(1)\n\n        # Calculate the fully conditional embedding, f, for the query set samples as described\n        # in appendix A.1 of the paper.\n        queries = model.f(support, queries)\n\n    # Efficiently calculate distance between all queries and all prototypes\n    # Output should have shape (q_queries * k_way, k_way) = (num_queries, k_way)\n    distances = pairwise_distances(queries, support, distance)\n\n    # Calculate \"attention\" as softmax over support-query distances\n    attention = (-distances).softmax(dim=1)\n\n    # Calculate predictions as in equation (1) from Matching Networks\n    # y_hat = \\sum_{i=1}^{k} a(x_hat, x_i) y_i\n    y_pred = matching_net_predictions(attention, n_shot, k_way, q_queries)\n\n    # Calculated loss with negative log likelihood\n    # Clip predictions for numerical stability\n    clipped_y_pred = y_pred.clamp(EPSILON, 1 - EPSILON)\n    loss = loss_fn(clipped_y_pred.log(), y)\n\n    if train:\n        # Backpropagate gradients\n        loss.backward()\n        # I found training to be quite unstable so I clip the norm\n        # of the gradient to be at most 1\n        clip_grad_norm_(model.parameters(), 1)\n        # Take gradient step\n        optimiser.step()\n\n    return loss, y_pred\n\n\ndef matching_net_predictions(attention: torch.Tensor, n: int, k: int, q: int) -> torch.Tensor:\n    \"\"\"Calculates Matching Network predictions based on equation (1) of the paper.\n    The predictions are the weighted sum of the labels of the support set where the\n    weights are the \"attentions\" (i.e. softmax over query-support distances) pointing\n    from the query set samples to the support set samples.\n    # Arguments\n        attention: torch.Tensor containing softmax over query-support distances.\n            Should be of shape (q * k, k * n)\n        n: Number of support set samples per class, n-shot\n        k: Number of classes in the episode, k-way\n        q: Number of query samples per-class\n    # Returns\n        y_pred: Predicted class probabilities\n    \"\"\"\n    if attention.shape != (q * k, k * n):\n        raise(ValueError(f'Expecting attention Tensor to have shape (q * k, k * n) = ({q * k, k * n})'))\n\n    # Create one hot label vector for the support set\n    y_onehot = torch.zeros(k * n, k)\n\n    # Unsqueeze to force y to be of shape (K*n, 1) as this\n    # is needed for .scatter()\n    y = create_nshot_task_label(k, n).unsqueeze(-1)\n    y_onehot = y_onehot.scatter(1, y, 1)\n\n    y_pred = torch.mm(attention, y_onehot.cuda().float())\n    return y_pred","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eb4248cd63721bf8c9cbee33a180a21515507ada","trusted":false},"cell_type":"code","source":"#few_shot.maml\n\nimport torch\nfrom collections import OrderedDict\nfrom torch.optim import Optimizer\nfrom torch.nn import Module\nfrom typing import Dict, List, Callable, Union\n\n#from few_shot.core import create_nshot_task_label\n\n\ndef replace_grad(parameter_gradients, parameter_name):\n    def replace_grad_(module):\n        return parameter_gradients[parameter_name]\n\n    return replace_grad_\n\n\ndef meta_gradient_step(model: Module,\n                       optimiser: Optimizer,\n                       loss_fn: Callable,\n                       x: torch.Tensor,\n                       y: torch.Tensor,\n                       n_shot: int,\n                       k_way: int,\n                       q_queries: int,\n                       order: int,\n                       inner_train_steps: int,\n                       inner_lr: float,\n                       train: bool,\n                       device: Union[str, torch.device]):\n    \"\"\"\n    Perform a gradient step on a meta-learner.\n    # Arguments\n        model: Base model of the meta-learner being trained\n        optimiser: Optimiser to calculate gradient step from loss\n        loss_fn: Loss function to calculate between predictions and outputs\n        x: Input samples for all few shot tasks\n        y: Input labels of all few shot tasks\n        n_shot: Number of examples per class in the support set of each task\n        k_way: Number of classes in the few shot classification task of each task\n        q_queries: Number of examples per class in the query set of each task. The query set is used to calculate\n            meta-gradients after applying the update to\n        order: Whether to use 1st order MAML (update meta-learner weights with gradients of the updated weights on the\n            query set) or 2nd order MAML (use 2nd order updates by differentiating through the gradients of the updated\n            weights on the query with respect to the original weights).\n        inner_train_steps: Number of gradient steps to fit the fast weights during each inner update\n        inner_lr: Learning rate used to update the fast weights on the inner update\n        train: Whether to update the meta-learner weights at the end of the episode.\n        device: Device on which to run computation\n    \"\"\"\n    data_shape = x.shape[2:]\n    create_graph = (True if order == 2 else False) and train\n\n    task_gradients = []\n    task_losses = []\n    task_predictions = []\n    for meta_batch in x:\n        # By construction x is a 5D tensor of shape: (meta_batch_size, n*k + q*k, channels, width, height)\n        # Hence when we iterate over the first  dimension we are iterating through the meta batches\n        x_task_train = meta_batch[:n_shot * k_way]\n        x_task_val = meta_batch[n_shot * k_way:]\n\n        # Create a fast model using the current meta model weights\n        fast_weights = OrderedDict(model.named_parameters())\n\n        # Train the model for `inner_train_steps` iterations\n        for inner_batch in range(inner_train_steps):\n            # Perform update of model weights\n            y = create_nshot_task_label(k_way, n_shot).to(device)\n            logits = model.functional_forward(x_task_train, fast_weights)\n            loss = loss_fn(logits, y)\n            gradients = torch.autograd.grad(loss, fast_weights.values(), create_graph=create_graph)\n\n            # Update weights manually\n            fast_weights = OrderedDict(\n                (name, param - inner_lr * grad)\n                for ((name, param), grad) in zip(fast_weights.items(), gradients)\n            )\n\n        # Do a pass of the model on the validation data from the current task\n        y = create_nshot_task_label(k_way, q_queries).to(device)\n        logits = model.functional_forward(x_task_val, fast_weights)\n        loss = loss_fn(logits, y)\n        loss.backward(retain_graph=True)\n\n        # Get post-update accuracies\n        y_pred = logits.softmax(dim=1)\n        task_predictions.append(y_pred)\n\n        # Accumulate losses and gradients\n        task_losses.append(loss)\n        gradients = torch.autograd.grad(loss, fast_weights.values(), create_graph=create_graph)\n        named_grads = {name: g for ((name, _), g) in zip(fast_weights.items(), gradients)}\n        task_gradients.append(named_grads)\n\n    if order == 1:\n        if train:\n            sum_task_gradients = {k: torch.stack([grad[k] for grad in task_gradients]).mean(dim=0)\n                                  for k in task_gradients[0].keys()}\n            hooks = []\n            for name, param in model.named_parameters():\n                hooks.append(\n                    param.register_hook(replace_grad(sum_task_gradients, name))\n                )\n\n            model.train()\n            optimiser.zero_grad()\n            # Dummy pass in order to create `loss` variable\n            # Replace dummy gradients with mean task gradients using hooks\n            logits = model(torch.zeros((k_way, ) + data_shape).to(device, dtype=torch.float))\n            loss = loss_fn(logits, create_nshot_task_label(k_way, 1).to(device))\n            loss.backward()\n            optimiser.step()\n\n            for h in hooks:\n                h.remove()\n\n        return torch.stack(task_losses).mean(), torch.cat(task_predictions)\n\n    elif order == 2:\n        model.train()\n        optimiser.zero_grad()\n        meta_batch_loss = torch.stack(task_losses).mean()\n\n        if train:\n            meta_batch_loss.backward()\n            optimiser.step()\n\n        return meta_batch_loss, torch.cat(task_predictions)\n    else:\n        raise ValueError('Order must be either 1 or 2.')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"29f3ffd5e6a48be3bde9feaee61dcbc4fc83da01","trusted":false},"cell_type":"code","source":"#dlclihe.util\n\nimport os\nimport sys\nfrom pathlib import Path\nimport numpy as np\nimport warnings\nwarnings.simplefilter('ignore')\nimport numpy as np\nnp.warnings.filterwarnings('ignore')\n\nimport IPython\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport pandas as pd\nfrom easydict import EasyDict\nfrom tqdm import tqdm_notebook\nimport shutil\nimport datetime\n\n## File utilities\n\ndef ensure_folder(folder):\n    \"\"\"Make sure a folder exists.\"\"\"\n    Path(folder).mkdir(exist_ok=True, parents=True)\n\ndef ensure_delete(folder_or_file):\n    anything = Path(folder_or_file)\n    if anything.is_dir():\n        shutil.rmtree(str(folder_or_file))\n    elif anything.exists():\n        anything.unlink()\n\ndef copy_file(src, dst):\n    \"\"\"Copy source file to destination file.\"\"\"\n    assert Path(src).is_file()\n    shutil.copy(str(src), str(dst))\n\ndef _copy_any(src, dst, symlinks):\n    if Path(src).is_dir():\n        if Path(dst).is_dir():\n            dst = Path(dst)/Path(src).name\n        assert not Path(dst).exists()\n        shutil.copytree(src, dst, symlinks=symlinks)\n    else:\n        copy_file(src, dst)\n\ndef copy_any(src, dst, symlinks=True):\n    \"\"\"Copy any file or folder recursively.\n    Source file can be list/array of files.\n    \"\"\"\n    do_list_item(_copy_any, src, dst, symlinks)\n\ndef do_list_item(func, src, *prms):\n    if isinstance(src, (list, tuple, np.ndarray)):\n        result = True\n        for element in src:\n            result = do_list_item(func, element, *prms) and result\n        return result\n    else:\n        return func(src, *prms)\n\ndef _move_file(src, dst):\n    shutil.move(str(src), str(dst))\n\ndef move_file(src, dst):\n    \"\"\"Move source file to destination file/folder.\n    Source file can be list/array of files.\n    \"\"\"\n    do_list_item(_move_file, src, dst)\n\ndef symlink_file(fromfile, tofile):\n    \"\"\"Make fromfile's symlink as tofile.\"\"\"\n    Path(tofile).symlink_to(fromfile)\n\ndef make_copy_to(dest_folder, files, n_sample=None, operation=copy_file):\n    \"\"\"Do file copy like operation from files to dest_folder.\n    \n    If n_sample is set, it creates symlinks up to number of n_sample files.\n    If n_sample is greater than len(files), symlinks are repeated twice or more until it reaches to n_sample.\n    If n_sample is less than len(files), n_sample symlinks are created for the top n_sample samples in files.\"\"\"\n    dest_folder.mkdir(exist_ok=True, parents=True)\n    if n_sample is None:\n        n_sample = len(files)\n\n    _done = False\n    _dup = 0\n    _count = 0\n    while not _done: # yet\n        for f in files:\n            f = Path(f)\n            name = f.stem+('_%d'%_dup)+f.suffix if 0 < _dup else f.name\n            to_file = dest_folder / name\n            operation(f, to_file)\n            _count += 1\n            _done = n_sample <= _count\n            if _done: break\n        _dup += 1\n    print('Now', dest_folder, 'has', len(list(dest_folder.glob('*'))), 'files.')\n\n## Log utilities\n\nimport logging\n_loggers = {}\ndef get_logger(name=None, level=logging.DEBUG, format=None, print=True, output_file=None):\n    \"\"\"One liner to get logger.\n    See test_log.py for example.\n    \"\"\"\n    name = name or __name__\n    if _loggers.get(name):\n        return _loggers.get(name)\n    else:\n        log = logging.getLogger(name)\n    formatter = logging.Formatter(format or '%(asctime)s %(name)s %(funcName)s [%(levelname)s]: %(message)s')\n    def add_handler(handler):\n        handler.setFormatter(formatter)\n        handler.setLevel(level)\n        log.addHandler(handler)\n    if print:\n        add_handler(logging.StreamHandler())\n    if output_file:\n        ensure_folder(Path(output_file).parent)\n        add_handler(logging.FileHandler(output_file))\n    log.setLevel(level)\n    log.propagate = False\n    _loggers[name] = log\n    return log\n\n## Multi process utilities\n\ndef caller_func_name(level=2):\n    \"\"\"Return caller function name.\"\"\"\n    return sys._getframe(level).f_code.co_name\n\ndef _file_mutex_filename(filename):\n    return filename or '/tmp/'+Path(caller_func_name(level=3)).stem+'.lock'\n\ndef lock_file_mutex(filename=None):\n    \"\"\"Lock file mutex (usually placed under /tmp).\n    Note that filename will be created based on caller function name.\n    \"\"\"\n    filename = _file_mutex_filename(filename)\n    with open(filename, 'w') as f:\n        f.write('locked at {}'.format(datetime.datetime.now()))\ndef release_file_mutex(filename=None):\n    \"\"\"Release file mutex.\"\"\"\n    filename = _file_mutex_filename(filename)\n    ensure_delete(filename)\n\ndef is_file_mutex_locked(filename=None):\n    \"\"\"Check if file mutex is locked or not.\"\"\"\n    filename = _file_mutex_filename(filename)\n    return Path(filename).exists()\n\n## Date utilities\n\ndef str_to_date(text):\n    if '/' in text:\n        temp_dt = datetime.datetime.strptime(text, '%Y/%m/%d')\n    else:\n        temp_dt = datetime.datetime.strptime(text, '%Y-%m-%d')\n    return datetime.date(temp_dt.year, temp_dt.month, temp_dt.day)\n\ndef get_week_start_end_dates(week_no:int, year=None) -> [datetime.datetime, datetime.datetime]:\n    \"\"\"Get start and end date of an ISO calendar week.\n    ISO week starts on Monday, and ends on Sunday.\n    \n    Arguments:\n        week_no: ISO calendar week number\n        year: Year to calculate, None will set this year\n    Returns:\n        [start_date:datetime, end_date:datetime]\n    \"\"\"\n    if not year:\n        year, this_week, this_day = datetime.datetime.today().isocalendar()\n    start_date = datetime.datetime.strptime(f'{year}-W{week_no:02d}-1', \"%G-W%V-%u\").date()\n    end_date = datetime.datetime.strptime(f'{year}-W{week_no:02d}-7', \"%G-W%V-%u\").date()\n    return [start_date, end_date]\n\ndef get_this_week_no():\n    \"\"\"Get ISO calendar week no of today.\"\"\"\n    return datetime.datetime.today().isocalendar()[1]\n\n## List utilities\n\ndef write_text_list(textfile, a_list):\n    \"\"\"Write list of str to a file with new lines.\"\"\"\n    with open(textfile, 'w') as f:\n        f.write('\\n'.join(a_list)+'\\n')\n\ndef read_text_list(filename) -> list:\n    \"\"\"Read text file splitted as list of texts, stripped.\"\"\"\n    with open(filename) as f:\n        lines = f.read().splitlines()\n        return [l.strip() for l in lines]\n\nfrom itertools import chain\ndef flatten_list(lists):\n    return list(chain.from_iterable(lists))\n\n# Thanks to https://stackoverflow.com/questions/3844801/check-if-all-elements-in-a-list-are-identical\ndef all_elements_are_identical(iterator):\n    \"\"\"Check all elements in iterable like list are identical.\"\"\"\n    iterator = iter(iterator)\n    try:\n        first = next(iterator)\n    except StopIteration:\n        return True\n    return all(first == rest for rest in iterator)\n\n## Text utilities\n\n# Thanks to https://github.com/dsindex/blog/wiki/%5Bpython%5D-difflib,-show-differences-between-two-strings\nimport difflib\ndef show_text_diff(text, n_text):\n    \"\"\"\n    http://stackoverflow.com/a/788780\n    Unify operations between two compared strings seqm is a difflib.\n    SequenceMatcher instance whose a & b are strings\n    \"\"\"\n    seqm = difflib.SequenceMatcher(None, text, n_text)\n    output= []\n    for opcode, a0, a1, b0, b1 in seqm.get_opcodes():\n        if opcode == 'equal':\n            pass # output.append(seqm.a[a0:a1])\n        elif opcode == 'insert':\n            output.append(\"<INS>\" + seqm.b[b0:b1] + \"</INS>\")\n        elif opcode == 'delete':\n            output.append(\"<DEL>\" + seqm.a[a0:a1] + \"</DEL>\")\n        elif opcode == 'replace':\n            # seqm.a[a0:a1] -> seqm.b[b0:b1]\n            output.append(\"<REPL>\" + seqm.b[b0:b1] + \"</REPL>\")\n        else:\n            raise RuntimeError\n    return ''.join(output)\n\nimport unicodedata\ndef unicode_visible_width(unistr):\n    \"\"\"Returns the number of printed characters in a Unicode string.\"\"\"\n    return sum([1 if unicodedata.east_asian_width(char) in ['N', 'Na'] else 2 for char in unistr])\n\n## Pandas utilities\n\ndef df_to_csv_excel_friendly(df, filename, **args):\n    \"\"\"df.to_csv() to be excel friendly UTF-8 handling.\"\"\"\n    df.to_csv(filename, encoding='utf_8_sig', **args)\n\ndef df_merge_update(df_list_or_org_file, opt_joining_file=None):\n    \"\"\"Merge data frames while update duplicated index with following (joining) row.\n    \n    Usages:\n        - df_merge_update([df1, df2, ...]) merges dfs in list.\n        - df_merge_update(df1, df2) merges df1 and df2.\n    \"\"\"\n    if opt_joining_file is not None:\n        df_list = [df_list_or_org_file, opt_joining_file]\n    else:\n        df_list = df_list_or_org_file\n\n    master = df_list[0]\n    for df in df_list[1:]:\n        tmp_df = pd.concat([master, df])\n        master = tmp_df[~tmp_df.index.duplicated(keep='last')].sort_index()\n    return master\n\ndef df_select_by_keyword(source_df, keyword, search_columns=None, as_mask=False):\n    \"\"\"Select data frame rows by a search keyword.\n    Any row will be selected if any of its search columns contain the keyword.\n    \n    Returns:\n        New data frame where rows have the keyword,\n        or mask if as_mask is True.\n    \"\"\"\n    search_columns = search_columns or source_df.columns\n    masks = np.column_stack([source_df[col].str.contains(keyword, na=False) for col in search_columns])\n    mask = masks.any(axis=1)\n    if as_mask:\n        return mask\n    return source_df.loc[mask]\n\ndef df_select_by_keywords(source_df, keys_cols, and_or='or', as_mask=False):\n    \"\"\"Multi keyword version of df_select_by_keyword.\n    Arguments:\n        key_cols: dict defined as `{'keyword1': [search columns] or None, ...}`\n    \"\"\"\n    masks = []\n    for keyword in keys_cols:\n        columns = keys_cols[keyword]\n        mask = df_select_by_keyword(source_df, keyword, search_columns=columns, as_mask=True)\n        masks.append(mask)\n    mask = np.column_stack(masks).any(axis=1) if and_or == 'or' else \\\n           np.column_stack(masks).all(axis=1)\n    if as_mask:\n        return mask\n    return source_df.loc[mask]\n\ndef df_str_replace(df, from_strs, to_str):\n    \"\"\"Apply str.replace to entire DataFrame inplace.\"\"\"\n    for i, row in df.iterrows():\n        df.ix[i] = df.ix[i].str.replace(from_strs, to_str)\n\ndef df_cell_str_replace(df, from_str, to_str):\n    \"\"\"Replace cell string with new string if entire string matches.\"\"\"\n    for i, row in df.iterrows():\n        for c in df.columns:\n            df.at[i, c] = to_str if str(df.at[i, c]) == from_str else df.at[i, c]\n\n_EXCEL_LIKE = ['.csv', '.xls', '.xlsx', '.xlsm']\ndef is_excel_file(filename):\n    # not accepted if suffix == '.csv': return True\n    return Path(filename).suffix.lower() in _EXCEL_LIKE\n\ndef is_csv_file(filename):\n    return Path(filename).suffix.lower() == '.csv'\n\ndef pd_read_excel_keep_dtype(io, **args):\n    \"\"\"pd.read_excel() wrapper to do as described in pandas document:\n    '... preserve data as stored in Excel and not interpret dtype'\n    Details:\n        - String '1' might be loaded as int 1 by pd.read_excel(file).\n        - By setting `dtype=object` it will preserve it as string '1'.\n    \"\"\"\n    return pd.read_excel(io, dtype=object, **args)\n\ndef pd_read_csv_as_str(filename, **args):\n    \"\"\"pd.read_csv() wrapper to preserve data type = str\"\"\"\n    return pd.read_csv(filename, dtype=object, **args)\n\ndef df_load_excel_like(filename, preserve_dtype=True, **args):\n    \"\"\"Load Excel like files. (csv, xlsx, ...)\"\"\"\n    if is_csv_file(filename):\n        if preserve_dtype:\n            return pd_read_csv_as_str(filename, **args)\n        return pd.read_csv(filename, **args)\n    if preserve_dtype:\n        return pd_read_excel_keep_dtype(filename, **args)\n    return pd.read_excel(filename, **args)\n\nimport codecs\ndef df_read_sjis_csv(filename, **args):\n    \"\"\"Read shift jis Japanese csv file.\n    Thanks to https://qiita.com/niwaringo/items/d2a30e04e08da8eaa643\n    \"\"\"\n    with codecs.open(filename, 'r', 'Shift-JIS', 'ignore') as file:\n        return pd.read_table(file, delimiter=',', **args)\n\n## Dataset utilities\n\nfrom imblearn.over_sampling import RandomOverSampler\nfrom imblearn.under_sampling import RandomUnderSampler\n\ndef flatten_y_if_onehot(y):\n    \"\"\"De-one-hot y, i.e. [0,1,0,0,...] to 1 for all y.\"\"\"\n    return y if len(np.array(y).shape) == 1 else np.argmax(y, axis = -1)\n\ndef get_class_distribution(y):\n    \"\"\"Calculate number of samples per class.\"\"\"\n    # y_cls can be one of [OH label, index of class, class label name string]\n    # convert OH to index of class\n    y_cls = flatten_y_if_onehot(y)\n    # y_cls can be one of [index of class, class label name]\n    classset = sorted(list(set(y_cls)))\n    sample_distribution = {cur_cls:len([one for one in y_cls if one == cur_cls]) for cur_cls in classset}\n    return sample_distribution\n\ndef get_class_distribution_list(y, num_classes):\n    \"\"\"Calculate number of samples per class as list\"\"\"\n    dist = get_class_distribution(y)\n    assert(y[0].__class__ != str) # class index or class OH label only\n    list_dist = np.zeros((num_classes))\n    for i in range(num_classes):\n        if i in dist:\n            list_dist[i] = dist[i]\n    return list_dist\n\ndef _balance_class(X, y, min_or_max, sampler_class, random_state):\n    \"\"\"Balance class distribution with sampler_class.\"\"\"\n    y_cls = flatten_y_if_onehot(y)\n    distribution = get_class_distribution(y_cls)\n    classes = list(distribution.keys())\n    counts  = list(distribution.values())\n    nsamples = np.max(counts) if min_or_max == 'max' \\\n          else np.min(counts)\n    flat_ratio = {cls:nsamples for cls in classes}\n    Xidx = [[xidx] for xidx in range(len(X))]\n    sampler_instance = sampler_class(ratio=flat_ratio, random_state=random_state)\n    Xidx_resampled, y_cls_resampled = sampler_instance.fit_sample(Xidx, y_cls)\n    sampled_index = [idx[0] for idx in Xidx_resampled]\n    return np.array([X[idx] for idx in sampled_index]), np.array([y[idx] for idx in sampled_index])\n\ndef balance_class_by_over_sampling(X, y, random_state=42):\n    \"\"\"Balance class distribution with imbalanced-learn RandomOverSampler.\"\"\"\n    return  _balance_class(X, y, 'max', RandomOverSampler, random_state)\n\ndef balance_class_by_under_sampling(X, y, random_state=42):\n    \"\"\"Balance class distribution with imbalanced-learn RandomUnderSampler.\"\"\"\n    return  _balance_class(X, y, 'min', RandomUnderSampler, random_state)\n\ndef df_balance_class_by_over_sampling(df, label_column, random_state=42):\n    \"\"\"Balance class distribution in DataFrame with imbalanced-learn RandomOverSampler.\"\"\"\n    X, y = list(range(len(df))), list(df[label_column])\n    X, _ = balance_class_by_over_sampling(X, y, random_state=random_state)\n    return df.iloc[X].sort_index()\n\ndef df_balance_class_by_under_sampling(df, label_column, random_state=42):\n    \"\"\"Balance class distribution in DataFrame with imbalanced-learn RandomUnderSampler.\"\"\"\n    X, y = list(range(len(df))), list(df[label_column])\n    X, _ = balance_class_by_under_sampling(X, y, random_state=random_state)\n    return df.iloc[X].sort_index()\n\n## Visualization utilities\n\ndef _expand_labels_from_y(y, labels):\n    \"\"\"Make sure y is index of label set.\"\"\"\n    if labels is None:\n        labels = sorted(list(set(y)))\n        y = [labels.index(_y) for _y in y]\n    return y, labels\n\ndef visualize_class_balance(title, y, labels=None, sorted=False):\n    y, labels = _expand_labels_from_y(y, labels)\n    sample_dist_list = get_class_distribution_list(y, len(labels))\n    if sorted:\n        items = list(zip(labels, sample_dist_list))\n        items.sort(key=lambda x:x[1], reverse=True)\n        sample_dist_list = [x[1] for x in items]\n        labels = [x[0] for x in items]\n    index = range(len(labels))\n    fig, ax = plt.subplots(1, 1, figsize = (16, 5))\n    ax.bar(index, sample_dist_list)\n    ax.set_xlabel('Label')\n    ax.set_xticks(index)\n    ax.set_xticklabels(labels, rotation='vertical')\n    ax.set_ylabel('Number of Samples')\n    ax.set_title(title)\n    fig.show()\n\nfrom collections import OrderedDict\ndef print_class_balance(title, y, labels=None, sorted=False):\n    y, labels = _expand_labels_from_y(y, labels)\n    distributions = get_class_distribution(y)\n    dist_dic = {labels[cls]:distributions[cls] for cls in distributions}\n    if sorted:\n        items = list(dist_dic.items())\n        items.sort(key=lambda x:x[1], reverse=True)\n        dist_dic = OrderedDict(items) # sorted(dist_dic.items(), key=...) didn't work for some reason...\n    print(title, '=', dist_dic)\n    zeroclasses = [label for i, label in enumerate(labels) if i not in distributions.keys()]\n    if 0 < len(zeroclasses):\n        print(' 0 sample classes:', zeroclasses)\n\nfrom sklearn.metrics import f1_score, precision_score, recall_score, accuracy_score\n\ndef calculate_clf_metrics(y_true, y_pred, average='weighted'):\n    \"\"\"Calculate metrics: f1/recall/precision/accuracy.\n    # Arguments\n        y_true: GT, an index of label or one-hot encoding format.\n        y_pred: Prediction output, index or one-hot.\n        average: `average` parameter passed to sklearn.metrics functions.\n    # Returns\n        Four metrics: f1, recall, precision, accuracy.\n    \"\"\"\n    y_true = flatten_y_if_onehot(y_true)\n    y_pred = flatten_y_if_onehot(y_pred)\n    if np.max(y_true) < 2 and np.max(y_pred) < 2:\n        average = 'binary'\n\n    f1 = f1_score(y_true, y_pred, average=average)\n    recall = recall_score(y_true, y_pred, average=average)\n    precision = precision_score(y_true, y_pred, average=average)\n    accuracy = accuracy_score(y_true, y_pred)\n    return f1, recall, precision, accuracy\n\ndef skew_bin_clf_preds(y_pred, binary_bias=None, logger=None):\n    \"\"\"Apply bias to prediction results for binary classification.\n    Calculated as follows.\n        p(y=1) := p(y=1) ^ binary_bias\n        p(y=0) := 1 - p(y=0)\n    0 < binary_bias < 1 will be optimistic with result=1.\n    Inversely, 1 < binary_bias will make results pesimistic.\n    \"\"\"\n    _preds = np.array(y_pred.copy())\n    if binary_bias is not None:\n        ps = np.power(_preds[:, 1], binary_bias)\n        _preds[:, 1] = ps\n        _preds[:, 0] = 1 - ps\n        logger = get_logger() if logger is None else logger\n        logger.info(f' @skew{\"+\" if binary_bias >= 0 else \"\"}{binary_bias}')\n    return _preds\n\ndef print_clf_metrics(y_true, y_pred, average='weighted', binary_bias=None, title_prefix='', logger=None):\n    \"\"\"Calculate and print metrics: f1/recall/precision/accuracy.\n    See calculate_clf_metrics() and skew_bin_clf_preds() for more detail.\n    \"\"\"\n    # Add bias if binary_bias is set\n    _preds = skew_bin_clf_preds(y_pred, binary_bias, logger=logger)\n    # Calculate metrics\n    f1, recall, precision, acc = calculate_clf_metrics(y_true, _preds, average=average)\n    logger = get_logger() if logger is None else logger\n    logger.info('{0:s}F1/Recall/Precision/Accuracy = {1:.4f}/{2:.4f}/{3:.4f}/{4:.4f}' \\\n          .format(title_prefix, f1, recall, precision, acc))\n\n# Thanks to https://qiita.com/knknkn1162/items/be87cba14e38e2c0f656\ndef plt_japanese_font_ready():\n    \"\"\"Set font family with Japanese fonts.\n    \n    # How to install fonts:\n        wget https://ipafont.ipa.go.jp/old/ipafont/IPAfont00303.php\n        mv IPAfont00303.php IPAfont00303.zip\n        unzip -q IPAfont00303.zip\n        sudo cp IPAfont00303/*.ttf /usr/share/fonts/truetype/\n    \"\"\"\n    plt.rcParams['font.family'] = 'IPAPGothic'\n\ndef plt_looks_good():\n    \"\"\"Plots will be looks good (at least to me).\"\"\"\n    plt.rcParams[\"figure.figsize\"] = [16, 10]\n    plt.rcParams['font.size'] = 14\n    plt.rcParams['xtick.labelsize'] = 10\n    plt.rcParams['ytick.labelsize'] = 10\n\n# Thanks to http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html#sphx-glr-auto-examples-model-selection-plot-confusion-matrix-py\nimport itertools\nfrom sklearn.metrics import confusion_matrix\n\ndef plot_confusion_matrix(y_test, y_pred, classes,\n                          normalize=True,\n                          title=None,\n                          cmap=plt.cm.Blues):\n    \"\"\"Plot confusion matrix.\"\"\"\n    po = np.get_printoptions()\n    np.set_printoptions(precision=2)\n\n    y_test = flatten_y_if_onehot(y_test)\n    y_pred = flatten_y_if_onehot(y_pred)\n    cm = confusion_matrix(y_test, y_pred)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        if title is None: title = 'Normalized confusion matrix'\n    else:\n        if title is None: title = 'Confusion matrix (not normalized)'\n\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    fmt = '.2f' if normalize else 'd'\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], fmt),\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.tight_layout()\n    np.set_printoptions(**po)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"85c3ab736b84d39900f6dd2f7443eebd3f532b33","trusted":false},"cell_type":"code","source":"#dlcliche.math\n\nimport math\nimport numpy as np\nimport pandas as pd\n\n\ndef roundup(x, n=10):\n    \"\"\"Round up x to multiple of n.\"\"\"\n    return int(math.ceil(x / n)) * n\n\n\ndef running_mean(x, N):\n    \"\"\"Calculate running/rolling mean or moving average.\n    Thanks to https://stackoverflow.com/a/27681394/6528729\n    \"\"\"\n    cumsum = np.cumsum(np.insert(x, 0, 0)) \n    return (cumsum[N:] - cumsum[:-N]) / float(N)\n\n\ndef np_describe(arr):\n    \"\"\"Describe numpy array statistics.\n    Thanks to https://qiita.com/AnchorBlues/items/051dc69e81705b52adad\n    \"\"\"\n    return pd.DataFrame(pd.Series(arr.ravel()).describe()).transpose()\n\n\ndef np_softmax(z):\n    \"\"\"Numpy version softmax.\n    Thanjs to https://stackoverflow.com/a/39558290/6528729\n    \"\"\"\n    assert len(z.shape) == 2\n    s = np.max(z, axis=1)\n    s = s[:, np.newaxis] # necessary step to do broadcasting\n    e_x = np.exp(z - s)\n    div = np.sum(e_x, axis=1)\n    div = div[:, np.newaxis] # dito\n    return e_x / div\n\n\nclass OnlineStats:\n    \"\"\"Calculate mean/variance of a vector online\n    Thanks to https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance\n    \"\"\"\n\n    def __init__(self, length):\n        self.K = np.zeros((length))\n        self.Ex = np.zeros((length))\n        self.Ex2 = np.zeros((length))\n        self.n = 0\n\n    def put(self, x):\n        if self.n == 0:\n            self.K = x\n        self.n += 1\n        d = x - self.K\n        self.Ex += d\n        self.Ex2 += d * d\n\n    def undo(self, x):\n        self.n -= 1\n        d = x - self.K\n        self.Ex -= d\n        self.Ex2 -= d * d\n\n    def mean(self):\n        if self.n == 0:\n            return np.zeros_like(self.K)\n        return self.K + self.Ex / self.n\n\n    def variance(self):\n        if self.n < 2:\n            return np.zeros_like(self.K)\n        return (self.Ex2 - (self.Ex * self.Ex) / self.n) / (self.n - 1)\n\n    def sigma(self):\n        if self.n < 2:\n            return np.zeros_like(self.K)\n        return np.sqrt(self.variance())\n\n    def count(self):\n        return self.n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f1f92c40a36aed2445f3c3dadda770e7825d3f0b","trusted":false},"cell_type":"code","source":"#dlcliche.image\n\n# Thanks to https://github.com/ipython/ipython/issues/9732\nfrom IPython import get_ipython\nipython = get_ipython()\n\n# Determine if this is running in Jupyter notebook or not\nif ipython:\n    running_in_notebook = ipython.has_trait('kernel')\n\n    if running_in_notebook:\n        ipython.magic('reload_ext autoreload')\n        ipython.magic('autoreload 2')\n        ipython.magic('matplotlib inline')\nelse:\n    # cannot even get ipython object...\n    running_in_notebook = False\n\n\ndef fit_notebook_to_window():\n    \"\"\"Fit notebook width to width of browser window.\n    Thanks to https://stackoverflow.com/questions/21971449/how-do-i-increase-the-cell-width-of-the-jupyter-ipython-notebook-in-my-browser\n    \"\"\"\n    from IPython.core.display import display, HTML\n    display(HTML(\"<style>.container { width:100% !important; }</style>\"))\n\n\nimport cv2\nimport tqdm\nimport math\nfrom PIL import Image\nfrom multiprocessing import Pool\n\ndef resize_image(dest_folder, filename, shape):\n    \"\"\"Resize and save copy of image file to destination folder.\"\"\"\n    img = cv2.imread(str(filename))\n    if shape is not None:\n        img = cv2.resize(img, shape)\n    outfile = str(Path(dest_folder)/Path(filename).name)\n    cv2.imwrite(outfile, img)\n    return outfile, (img.shape[1], img.shape[0]) # original size\n\ndef _resize_image_worker(args):\n    return resize_image(args[0], args[1], args[2])\n\ndef resize_image_files(dest_folder, source_files, shape=(224, 224), num_threads=8, skip_if_any_there=False):\n    \"\"\"Make resized copy of listed images in parallel processes.\n    Arguments:\n        dest_folder: Destination folder to make copies.\n        source_files: Source image files.\n        shape: (Width, Depth) shape of copies. None will NOT resize and makes dead copy.\n        num_threads: Number of parallel workers.\n        skip_if_any_there: If True, skip processing processing if any file have already been done.\n    Returns:\n        List of image info (filename, original size) tuples, or None if skipped.\n        ex)\n        ```python\n        [('tmp/8d6ed7c786dcbc93.jpg', (1024, 508)),\n         ('tmp/8d6ee9921e4aeb18.jpg', (891, 1024)),\n         ('tmp/8d6f00feedb09efa.jpg', (1024, 683))]\n        ```\n    \"\"\"\n    if skip_if_any_there:\n        if (Path(dest_folder)/Path(source_files[0]).name).exists():\n            return None\n    # Create destination folder if needed\n    ensure_folder(dest_folder)\n    # Do resize\n    if running_in_notebook:  # Workaround: not using pool on notebook\n        returns = []\n        for f in tqdm.tqdm(source_files, total=len(source_files)):\n            returns.append(resize_image(dest_folder, f, shape))\n    else:\n        with Pool(num_threads) as p:\n            args = [[dest_folder, f, shape] for f in source_files]\n            returns = list(tqdm.tqdm(p.imap(_resize_image_worker, args), total=len(args)))\n    return returns\n\ndef _get_shape_worker(filename):\n    return Image.open(filename).size # Image.open() is much faster than cv2.imread()\n\ndef read_file_shapes(files, num_threads=8):\n    \"\"\"Read shape of files in parallel.\"\"\"\n    if running_in_notebook:  # Workaround: not using pool on notebook\n        shapes = []\n        for f in tqdm.tqdm(files, total=len(files)):\n            shapes.append(_get_shape_worker(f))\n    else:\n        with Pool(num_threads) as p:\n            shapes = list(tqdm.tqdm(p.imap(_get_shape_worker, files), total=len(files)))\n    return np.array(shapes)\n\ndef load_rgb_image(filename):\n    \"\"\"Load image file and make sure that format is RGB.\"\"\"\n    img = cv2.imread(str(filename))\n    if img is None:\n        raise ValueError(f'Failed to load {filename}.')\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    return img\n\ndef convert_mono_to_jpg(fromfile, tofile):\n    \"\"\"Convert monochrome image to color jpeg format.\n    Linear copy to RGB channels. \n    https://docs.opencv.org/3.1.0/de/d25/imgproc_color_conversions.html\n    Args:\n        fromfile: float png mono image\n        tofile: RGB color jpeg image.\n    \"\"\"\n    img = np.array(Image.open(fromfile)) # TODO Fix this to cv2.imread\n    img = img - np.min(img)\n    img = img / (np.max(img) + 1e-4)\n    img = (img * 255).astype(np.uint8) # [0,1) float to [0,255] uint8\n    img = np.repeat(img[..., np.newaxis], 3, axis=-1) # mono to RGB color\n    img = Image.fromarray(img)\n    tofile = Path(tofile)\n    img.save(tofile.with_suffix('.jpg'), 'JPEG', quality=100)\n\n# Borrowing from fast.ai course notebook\nfrom matplotlib import patches, patheffects\ndef subplot_matrix(rows, columns, figsize=(12, 12)):\n    \"\"\"Subplot utility for drawing matrix of images.\n    # Usage\n    Following will show images in 2x3 matrix.\n    ```python\n    axes = subplot_matrix(2, 3)\n    for img, ax in zip(images, axes):\n        show_image(img, ax=ax)\n    ```\n    \"\"\"\n    fig, axes = plt.subplots(rows, columns, figsize=figsize)\n    return list(axes.flat) if 1 < rows*columns else [axes]\n\n\ndef show_np_image(img, figsize=None, ax=None):\n    \"\"\"Show numpy object image with figsize on axes of subplot.\n    Using this with subplot_matrix() will make it easy to plot matrix of images.\n    # Returns\n        Axes of subplot created, or given.\"\"\"\n    if not ax: fig,ax = plt.subplots(figsize=figsize)\n    ax.imshow(img)\n    ax.get_xaxis().set_visible(False)\n    ax.get_yaxis().set_visible(False)\n    return ax\ndef _draw_outline(o, lw):\n    o.set_path_effects([patheffects.Stroke(\n        linewidth=lw, foreground='black'), patheffects.Normal()])\ndef ax_draw_rect(ax, b):\n    patch = ax.add_patch(patches.Rectangle(b[:2], *b[-2:], fill=False, edgecolor='white', lw=2))\n    _draw_outline(patch, 4)\ndef ax_draw_text(ax, xy, txt, sz=14):\n    text = ax.text(*xy, txt,\n        verticalalignment='top', color='white', fontsize=sz, weight='bold')\n    _draw_outline(text, 1)\ndef ax_draw_bbox(ax, bbox, class_name):\n    \"\"\"Object Detection Helper: Draw single bounding box with class name on top of image.\"\"\"\n    ax_draw_rect(ax, bbox)\n    ax_draw_text(ax, bbox[:2], class_name)\n\ndef show_np_od_data(image, bboxes, labels, class_names=None, figsize=None):\n    \"\"\"Object Detection Helper: Show numpy object detector data (set of an image, bboxes and labels).\"\"\"\n    ax = show_np_image(image, figsize=figsize)\n    for bbox, label in zip(bboxes, labels):\n        if class_names is not None:\n            label = class_names[label]\n        ax_draw_bbox(ax, bbox, label)\n    plt.show()\n\ndef union_of_bboxes(height, width, bboxes, erosion_rate=0.0, to_int=False):\n    \"\"\"Calculate union bounding box of boxes.\n    # Arguments\n        height (float): Height of image or space.\n        width (float): Width of image or space.\n        bboxes (list): List like bounding boxes. Format is `[x_min, y_min, x_max, y_max]`.\n        erosion_rate (float): How much each bounding box can be shrinked, useful for erosive cropping.\n            Set this in range [0, 1]. 0 will not be erosive at all, 1.0 can make any bbox to lose its volume.\n        to_int (bool): Returns as int if True.\n    \"\"\"\n    x1, y1 = width, height\n    x2, y2 = 0, 0\n    for b in bboxes:\n        w, h = b[2]-b[0], b[3]-b[1]\n        lim_x1, lim_y1 = b[0] + erosion_rate*w, b[1] + erosion_rate*h\n        lim_x2, lim_y2 = b[2] - erosion_rate*w, b[3] - erosion_rate*h\n        x1, y1 = np.min([x1, lim_x1]), np.min([y1, lim_y1])\n        x2, y2 = np.max([x2, lim_x2]), np.max([y2, lim_y2])\n        #print(b, [lim_x1, lim_y1, lim_x2, lim_y2], [x1, y1, x2, y2])\n    if to_int:\n        x1, y1 = int(math.floor(x1)), int(math.floor(y1))\n        x2, y2 = int(np.min([width, math.ceil(x2)])), int(np.min([height, math.ceil(y2)]))\n    return x1, y1, x2, y2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"771e7f1d7f6500c433e6316b5b15a25b54606cd9","trusted":false},"cell_type":"code","source":"#few_shot.extmodel_proto_net_clf.py\n\n\n\"\"\"\nFor testing what if we use ImageNet pretrained model as ProtoNet??\n\"\"\"\n#from dlcliche.utils import *\n#from dlcliche.math import *\n#from dlcliche.image import show_np_image, subplot_matrix\n\nfrom torchvision import models\nfrom torch import nn\nimport torch\nfrom tqdm import tqdm\n\n# TODO: Support cpu environment\n\nclass BasePretrainedModel(nn.Module):\n    def __init__(self, base_model=models.resnet18, n_embs=512, print_shape=False):\n        super(BasePretrainedModel, self).__init__()\n        resnet = base_model(pretrained=True)\n        self.body = nn.Sequential(*list(resnet.children())[:-1])\n        self.n_embs = n_embs\n        self.print_shape = print_shape\n\n    def forward(self, x):\n        x = self.body(x)\n        if self.print_shape:\n            print(x.shape)\n        return x.view(-1, self.n_embs)\n\n\nclass ExtModelProtoNetClf(object):\n    \"\"\"ProtoNet as conventional classifier using external model.\n    Created for testing what if we use ImageNet pretrained model for getting embeddings.\n    TODO Fix bad design for member-call-order dependency...\n    \"\"\"\n\n    def __init__(self, model, classes, device):\n        model.to(device)\n        model.eval()\n        self.model = model\n        self.classes = classes\n        self.device = device\n        self.prototypes = None\n        self.n_embeddings = None  # First get_embeddings() will set this\n        self.n_classes = len(classes)\n        self.log = get_logger()\n\n    \n    def _make_null_prototypes(self):\n        self.prototypes = [OnlineStats(self.n_embeddings)for _ in range(self.n_classes)]\n    \n    def get_embeddings(self, dl, visualize=False):\n        \"\"\"Get embeddings for all samples available in dataloader.\"\"\"\n        gts, cur = [], 0\n        with torch.no_grad():\n            for batch_index, (X, y_gt) in tqdm(enumerate(dl), total=len(dl)):\n                dev_X, y_gt = X.to(self.device), list(y_gt)\n                this_embs = self.model(dev_X).cpu().detach().numpy()\n                if cur == 0:\n                    self.n_embeddings = this_embs.shape[-1]\n                    embs = np.zeros((len(dl.dataset), self.n_embeddings))\n\n                if visualize:\n                    for i, ax in enumerate(subplot_matrix(columns=4, rows=2, figsize=(16, 8))):\n                        if len(dl) <= batch_index * 8 + i: break\n                        show_np_image(np.transpose(X[i].cpu().detach().numpy(), [1, 2, 0]), ax=ax)\n                    plt.show()\n\n                for i in range(len(this_embs)):\n                    embs[cur] = this_embs[i]\n                    gts.append(y_gt[i])\n                    cur += 1\n        return np.array(embs), gts\n\n    def make_prototypes(self, support_set_dl, repeat=1, update=False, visualize=False):\n        \"\"\"Calculate prototypes by accumulating embeddings of all samples in given support set.\n        Args:\n             support_set_dl: support set dataloader.\n             repeat: test parameter for what if we get prototype with augmented samples.\n             update: set True if you don't want to update prototypes with new samples from dataloader.\n        \"\"\"\n        # Get embeddings of support set samples\n        embs, gts = self.get_embeddings(support_set_dl, visualize=visualize)\n        # Make prototypes if not there\n        \n        if update:\n            self.log.info('Using current prototypes.')\n        else:\n            self.log.info('Making new prototypes.')\n            self._make_null_prototypes()\n        \n        # Update prototypes (just by feeding to online stat class)\n        \n        for i in range(repeat):\n            for emb, cls in zip(embs, gts):\n                if not isinstance(cls, int):\n                    cls = self.classes.index(cls)\n                self.prototypes[cls].put(emb)\n            if i < repeat - 1:\n                embs, gts = self.get_embeddings(support_set_dl)  # no visualization\n        \n    def predict_embeddings(self, X_embs, softmax=True):\n        preds = np.zeros((len(X_embs), self.n_classes))\n        proto_embs = [p.mean() for p in self.prototypes]\n        for idx_sample, x in tqdm(enumerate(X_embs), total=len(X_embs)):\n            for idx_class, proto in enumerate(proto_embs):\n                preds[idx_sample, idx_class] = -np.log(np.sum((x - proto)**2) + 1e-300) # preventing log(0)\n            if softmax:\n                preds[idx_sample, :] = np_softmax(preds[idx_sample])\n        return preds\n    def predict(self, data_loader):\n        embs, y_gts = self.get_embeddings(data_loader)\n        return self.predict_embeddings(embs), y_gts\n\n    def evaluate(self, data_loader):\n        y_hat, y_gts = self.predict(data_loader)\n        return calculate_clf_metrics(y_gts, y_hat)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7a39beffe3f76ac9a05e0338c2d7271d7d803305","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"edd4f7d7f729147152c8666cc08184e8e8275664","trusted":false},"cell_type":"code","source":"#few_shot.datasets.py\n\n\nfrom torch.utils.data import Dataset\nimport torch\nfrom PIL import Image\nfrom torchvision import transforms\nfrom skimage import io\nfrom tqdm import tqdm\nimport pandas as pd\nimport numpy as np\nimport os\n\n#from config import DATA_PATH\n\n\nclass OmniglotDataset(Dataset):\n    def __init__(self, subset):\n        \"\"\"Dataset class representing Omniglot dataset\n        # Arguments:\n            subset: Whether the dataset represents the background or evaluation set\n        \"\"\"\n        if subset not in ('background', 'evaluation'):\n            raise(ValueError, 'subset must be one of (background, evaluation)')\n        self.subset = subset\n\n        self.df = pd.DataFrame(self.index_subset(self.subset))\n\n        # Index of dataframe has direct correspondence to item in dataset\n        self.df = self.df.assign(id=self.df.index.values)\n\n        # Convert arbitrary class names of dataset to ordered 0-(num_speakers - 1) integers\n        self.unique_characters = sorted(self.df['class_name'].unique())\n        self.class_name_to_id = {self.unique_characters[i]: i for i in range(self.num_classes())}\n        self.df = self.df.assign(class_id=self.df['class_name'].apply(lambda c: self.class_name_to_id[c]))\n\n        # Create dicts\n        self.datasetid_to_filepath = self.df.to_dict()['filepath']\n        self.datasetid_to_class_id = self.df.to_dict()['class_id']\n\n    def __getitem__(self, item):\n        instance = io.imread(self.datasetid_to_filepath[item])\n        # Reindex to channels first format as supported by pytorch\n        instance = instance[np.newaxis, :, :]\n\n        # Normalise to 0-1\n        instance = (instance - instance.min()) / (instance.max() - instance.min())\n\n        label = self.datasetid_to_class_id[item]\n\n        return torch.from_numpy(instance), label\n\n    def __len__(self):\n        return len(self.df)\n\n    def num_classes(self):\n        return len(self.df['class_name'].unique())\n\n    @staticmethod\n    def index_subset(subset):\n        \"\"\"Index a subset by looping through all of its files and recording relevant information.\n        # Arguments\n            subset: Name of the subset\n        # Returns\n            A list of dicts containing information about all the image files in a particular subset of the\n            Omniglot dataset dataset\n        \"\"\"\n        images = []\n        print('Indexing {}...'.format(subset))\n        # Quick first pass to find total for tqdm bar\n        subset_len = 0\n        for root, folders, files in os.walk(DATA_PATH + '/Omniglot/images_{}/'.format(subset)):\n            subset_len += len([f for f in files if f.endswith('.png')])\n\n        progress_bar = tqdm(total=subset_len)\n        for root, folders, files in os.walk(DATA_PATH + '/Omniglot/images_{}/'.format(subset)):\n            if len(files) == 0:\n                continue\n\n            alphabet = root.split('/')[-2]\n            class_name = '{}.{}'.format(alphabet, root.split('/')[-1])\n\n            for f in files:\n                progress_bar.update(1)\n                images.append({\n                    'subset': subset,\n                    'alphabet': alphabet,\n                    'class_name': class_name,\n                    'filepath': os.path.join(root, f)\n                })\n\n        progress_bar.close()\n        return images\n\n\nclass MiniImageNet(Dataset):\n    def __init__(self, subset):\n        \"\"\"Dataset class representing miniImageNet dataset\n        # Arguments:\n            subset: Whether the dataset represents the background or evaluation set\n        \"\"\"\n        if subset not in ('background', 'evaluation'):\n            raise(ValueError, 'subset must be one of (background, evaluation)')\n        self.subset = subset\n\n        self.df = pd.DataFrame(self.index_subset(self.subset))\n\n        # Index of dataframe has direct correspondence to item in dataset\n        self.df = self.df.assign(id=self.df.index.values)\n\n        # Convert arbitrary class names of dataset to ordered 0-(num_speakers - 1) integers\n        self.unique_characters = sorted(self.df['class_name'].unique())\n        self.class_name_to_id = {self.unique_characters[i]: i for i in range(self.num_classes())}\n        self.df = self.df.assign(class_id=self.df['class_name'].apply(lambda c: self.class_name_to_id[c]))\n\n        # Create dicts\n        self.datasetid_to_filepath = self.df.to_dict()['filepath']\n        self.datasetid_to_class_id = self.df.to_dict()['class_id']\n\n        # Setup transforms\n        self.transform = transforms.Compose([\n            transforms.CenterCrop(224),\n            transforms.Resize(84),\n            transforms.ToTensor(),\n            transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                 std=[0.229, 0.224, 0.225])\n        ])\n\n    def __getitem__(self, item):\n        instance = Image.open(self.datasetid_to_filepath[item])\n        instance = self.transform(instance)\n        label = self.datasetid_to_class_id[item]\n        return instance, label\n\n    def __len__(self):\n        return len(self.df)\n\n    def num_classes(self):\n        return len(self.df['class_name'].unique())\n\n    @staticmethod\n    def index_subset(subset):\n        \"\"\"Index a subset by looping through all of its files and recording relevant information.\n        # Arguments\n            subset: Name of the subset\n        # Returns\n            A list of dicts containing information about all the image files in a particular subset of the\n            miniImageNet dataset\n        \"\"\"\n        images = []\n        print('Indexing {}...'.format(subset))\n        # Quick first pass to find total for tqdm bar\n        subset_len = 0\n        for root, folders, files in os.walk(DATA_PATH + '/miniImageNet/images_{}/'.format(subset)):\n            subset_len += len([f for f in files if f.endswith('.png')])\n\n        progress_bar = tqdm(total=subset_len)\n        for root, folders, files in os.walk(DATA_PATH + '/miniImageNet/images_{}/'.format(subset)):\n            if len(files) == 0:\n                continue\n\n            class_name = root.split('/')[-1]\n\n            for f in files:\n                progress_bar.update(1)\n                images.append({\n                    'subset': subset,\n                    'class_name': class_name,\n                    'filepath': os.path.join(root, f)\n                })\n\n        progress_bar.close()\n        return images\n\n\nclass DummyDataset(Dataset):\n    def __init__(self, samples_per_class=10, n_classes=10, n_features=1):\n        \"\"\"Dummy dataset for debugging/testing purposes\n        A sample from the DummyDataset has (n_features + 1) features. The first feature is the index of the sample\n        in the data and the remaining features are the class index.\n        # Arguments\n            samples_per_class: Number of samples per class in the dataset\n            n_classes: Number of distinct classes in the dataset\n            n_features: Number of extra features each sample should have.\n        \"\"\"\n        self.samples_per_class = samples_per_class\n        self.n_classes = n_classes\n        self.n_features = n_features\n\n        # Create a dataframe to be consistent with other Datasets\n        self.df = pd.DataFrame({\n            'class_id': [i % self.n_classes for i in range(len(self))]\n        })\n        self.df = self.df.assign(id=self.df.index.values)\n\n    def __len__(self):\n        return self.samples_per_class * self.n_classes\n\n    def __getitem__(self, item):\n        class_id = item % self.n_classes\n        return np.array([item] + [class_id]*self.n_features, dtype=np.float), float(class_id)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7506274a567bfeec82da4d918403c52094577381","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"39145c87df4f505c996775d65516d161effae857","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"876c51645b75cb9d1794e35e53f4cc7a5aeed4a1","trusted":false},"cell_type":"code","source":"#app_utils_clf\n\nfrom torch.optim import Adam\nfrom torch.utils.data import DataLoader\nfrom torchvision import models\nfrom torch import nn\n\n#from few_shot.models import get_few_shot_encoder, Flatten\n#from few_shot.core import NShotTaskSampler, EvaluateFewShot, prepare_nshot_task\n#from few_shot.proto import proto_net_episode\n#from few_shot.train import fit\n#from few_shot.callbacks import *\n\n#from dlcliche.utils import *\n\nassert torch.cuda.is_available()\ndevice = torch.device('cuda')\n\n\ndef show_normalized_image(img, ax=None, mono=False):\n    if mono:\n        img.numpy()[..., np.newaxis]\n    np_img = img.numpy().transpose(1, 2, 0)\n    lifted = np_img - np.min(np_img)\n    ranged = lifted / np.max(lifted)\n    show_np_image(ranged, ax=ax)\n\n\nclass MonoTo3ChLayer(nn.Module):\n    def __init__(self):\n        super(MonoTo3ChLayer, self).__init__()\n    def forward(self, x):\n        x.unsqueeze_(1)\n        return x.repeat(1, 3, 1, 1)\n\n\ndef _get_model(weight_file, device, model_fn, mono):\n    base_model = model_fn(pretrained=True)\n    feature_model = nn.Sequential(*list(base_model.children())[:-1],\n                                  nn.AdaptiveAvgPool2d(1),\n                                  Flatten())\n    # Load initial weights\n    if weight_file is not None:\n        feature_model.load_state_dict(torch.load(weight_file))\n    # Add mono image input layer at the bottom of feature model\n    if mono:\n        feature_model = nn.Sequential(MonoTo3ChLayer(), feature_model)\n    if device is not None:\n        feature_model.to(device)\n\n    feature_model.eval()\n    return feature_model\n\n\ndef get_resnet101(weight_file=None, device=None, mono=False):\n    return _get_model(weight_file, device, models.resnet101, mono=mono)\n\n\ndef get_resnet50(weight_file=None, device=None, mono=False):\n    return _get_model(weight_file, device, models.resnet50, mono=mono)\n\n\ndef get_resnet34(weight_file=None, device=None, mono=False):\n    return _get_model(weight_file, device, models.resnet34, mono=mono)\n\n\ndef get_resnet18(weight_file=None, device=None, mono=False):\n    return _get_model(weight_file, device, models.resnet18, mono=mono)\n\n\ndef get_densenet121(weight_file=None, device=None, mono=False):\n    return _get_model(weight_file, device, models.densenet121, mono=mono)\n\n\ndef train_proto_net(args, model, device, n_epochs,\n                    background_taskloader,\n                    evaluation_taskloader,\n                    path='.',\n                    lr=3e-3,\n                    drop_lr_every=100,\n                    evaluation_episodes=100,\n                    episodes_per_epoch=100,\n                   ):\n    # Prepare model\n    model.to(device, dtype=torch.float)\n    model.train(True)\n\n    # Prepare training etc.\n    optimizer = Adam(model.parameters(), lr=lr)\n    loss_fn = torch.nn.NLLLoss().cuda()\n    #ensure_folder(path)\n    #ensure_folder(path)\n\n    def lr_schedule(epoch, lr):\n        if epoch % drop_lr_every == 0:\n            return lr / 2\n        else:\n            return lr\n\n    callbacks = [\n        EvaluateFewShot(\n            eval_fn=proto_net_episode,\n            num_tasks=evaluation_episodes,\n            n_shot=args.n_test,\n            k_way=args.k_test,\n            q_queries=args.q_test,\n            taskloader=evaluation_taskloader,\n            prepare_batch=prepare_nshot_task(args.n_test, args.k_test, args.q_test),\n            distance=args.distance\n        ),\n        ModelCheckpoint(\n            filepath=path + '/models/'+args.param_str+'_e{epoch:02d}.pth',\n            monitor=args.checkpoint_monitor or f'val_{args.n_test}-shot_{args.k_test}-way_acc',\n            period=args.checkpoint_period or 100,\n        ),\n        LearningRateScheduler(schedule=lr_schedule),\n        CSVLogger(path +'train.csv'),\n    ]\n\n    fit(\n        model,\n        optimizer,\n        loss_fn,\n        epochs=n_epochs,\n        dataloader=background_taskloader,\n        prepare_batch=prepare_nshot_task(args.n_train, args.k_train, args.q_train),\n        callbacks=callbacks,\n        metrics=['categorical_accuracy'],\n        epoch_metrics=[f'val_{args.n_test}-shot_{args.k_test}-way_acc'],\n        fit_function=proto_net_episode,\n        fit_function_kwargs={'n_shot': args.n_train, 'k_way': args.k_train, 'q_queries': args.q_train, 'train': True,\n                             'distance': args.distance},\n)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ed1d361c5252fc09f4c19ecef2ebbe5b2f0d4ee9","trusted":false},"cell_type":"code","source":"#whale.whale_utils\n\n#from dlcliche.image import *\n#from dlcliche.math import *\nfrom sklearn.decomposition import PCA\nfrom mpl_toolkits.mplot3d import axes3d\nfrom IPython.display import display\nimport glob\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import transforms\nimport albumentations as A\n\n#sys.path.append('..') # app\n#sys.path.append('../..') # root\n#from few_shot.extmodel_proto_net_clf import ExtModelProtoNetClf\n#from app_utils_clf import *\n\n\ndef _get_test_images(data_test):\n    return [str(f).replace(data_test+'/', '') for f in glob.glob(data_test+'/*.jpg')]\n\n\ndef get_aug(re_size=224, to_size=224, train=True):\n    augs = [A.Resize(height=re_size, width=re_size)]\n    if train:\n        augs.extend([\n            A.RandomCrop(height=to_size, width=to_size),\n            A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.3, rotate_limit=30, p=0.75),\n            A.RandomBrightnessContrast(brightness_limit=0.5, contrast_limit=0.5, p=0.75),\n            A.Blur(p=0.5),\n            A.Cutout(max_h_size=to_size//12, max_w_size=to_size//12, p=0.5),\n        ])\n    else:\n        augs.extend([A.CenterCrop(height=to_size, width=to_size)])\n    return A.Compose(augs + [A.Normalize()])\n\n\ndef get_img_loader(folder, to_gray=False):\n    def _loader(filename):\n        img = cv2.imread(folder + '/' + str(filename))\n        if to_gray:\n            img = np.mean(img, axis=-1).astype(np.uint8)\n            img = np.stack((img,)*3, axis=-1)\n        return img\n    return _loader\n\n\nclass WhaleImages(Dataset):\n    def __init__(self, path, images, labels, re_size=256, to_size=224, train=True):\n        self.datasetid_to_filepath = images\n        self.datasetid_to_class_id = labels\n        self.classes = sorted(list(set(labels)))\n        \n        self.df = pd.DataFrame({'class_id':labels, 'id':list(range(len(images)))})\n\n        self.loader = get_img_loader(path, to_gray=True)\n        self.transform = get_aug(re_size=re_size, to_size=to_size, train=train)\n        self.to_tensor = transforms.ToTensor()\n\n    def __getitem__(self, item):\n        instance = self.loader(self.datasetid_to_filepath[item])\n        instance = self.transform(image=instance)['image']\n        instance = self.to_tensor(instance)\n        label = self.datasetid_to_class_id[item]\n        return instance, label\n\n    def __len__(self):\n        return len(self.df)\n\n    def num_classes(self):\n        return len(self.cls2imgs)\n\n\ndef plot_prototype_2d_space_distribution(prototypes):\n    X = prototypes\n    pca = PCA(n_components=2)\n    X_pca = pca.fit(X).transform(X)\n    print('PCA: Explained variance ratio: %s'\n          % str(pca.explained_variance_ratio_))\n    plt.figure()\n    plt.scatter(X_pca[:, 0], X_pca[:, 1], alpha=.6)\n    plt.title('Prototype Distribution PCA')\n    plt.xlim((-4, 4))\n    plt.ylim((-3, 3))\n    plt.show()\n    return X_pca\n\n\ndef plot_prototype_3d_space_distribution(prototypes):\n    X = prototypes\n    pca = PCA(n_components=3)\n    X_pca = pca.fit(X).transform(X)\n    print('PCA: Explained variance ratio: %s'\n          % str(pca.explained_variance_ratio_))\n    fig = plt.figure(figsize=(10,10))\n    ax = fig.add_subplot(111, projection='3d')\n    ax.scatter3D(X_pca[:, 0],X_pca[:, 1],X_pca[:, 2])\n    ax.set_title('Prototype Distribution PCA')\n    ax.set_xlim((-4, 4))\n    ax.set_ylim((-3, 3))\n    ax.set_zlim((-3, 3))\n    plt.show()\n    return X_pca\n\n\ndef get_classes(data='data', except_new_whale=True, append_new_whale_last=True):\n    df = pd.read_csv(data+'/train.csv')\n    if except_new_whale:\n        df = df[df.Id != 'new_whale']\n    classes = sorted(list(set(df.Id.values)))\n    if append_new_whale_last:\n        classes.append('new_whale')\n    return classes\n\n\ndef calculate_results(weight, SZ, get_model_fn, device, train_csv='../input/train.csv',\n                      data_train='../input/train', data_test='../input/test'):\n    # Training samples\n    df = pd.read_csv(train_csv)\n    df = df[df.Id != 'new_whale']\n    images = df.Image.values\n    labels = df.Id.values\n\n    # Test samples\n    test_images = _get_test_images(data_test)\n    dummy_test_gts = list(range(len(test_images)))\n\n    print(f'Training samples: {len(images)}, # of labels: {len(list(set(labels)))}.')\n    print(f'Test samples: {len(test_images)}.')\n    print(f'Work in progress for {weight}...')\n\n    def get_dl(images, labels, folder, SZ=SZ, batch_size=64):\n        ds = WhaleImages(folder, images, labels, re_size=SZ, to_size=SZ, train=False)\n        dl = DataLoader(ds, batch_size=batch_size)\n        return dl\n\n    # Make prototypes\n    trn_dl = get_dl(images, labels, data_train)\n    model = get_model_fn(device=device, weight_file=weight+'.pth')\n    proto_net = ExtModelProtoNetClf(model, trn_dl.dataset.classes, device)\n\n    proto_net.make_prototypes(trn_dl)\n\n    # Calculate distances\n    test_dl = get_dl(test_images, dummy_test_gts, data_test)\n    test_embs, gts = proto_net.get_embeddings(test_dl)\n    test_dists = proto_net.predict_embeddings(test_embs, softmax=False)\n\n    np.save(f'test_dists_{weight}.npy', test_dists)\n    np.save(f'prototypes_{weight}.npy', np.array([x.mean() for x in proto_net.prototypes]))\n\n\n# Thanks to https://github.com/radekosmulski/whale/blob/master/utils.py\ndef top_5_preds(preds): return np.argsort(preds.numpy())[:, ::-1][:, :5]\n\ndef top_5_pred_labels(preds, classes):\n    top_5 = top_5_preds(preds)\n    labels = []\n    for i in range(top_5.shape[0]):\n        labels.append(' '.join([classes[idx] for idx in top_5[i]]))\n    return labels\n\n\ndef prepare_submission(submission_filename, test_dists, new_whale_thresh, data_test, classes):\n    def _create_proto_submission(preds, name, classes):\n        sub = pd.DataFrame({'Image': _get_test_images(data_test)})\n        sub['Id'] = [classes[i] if not isinstance(i, str) else i for i in \n                     top_5_pred_labels(torch.tensor(preds), classes)]\n        ensure_folder('subs')\n        sub.to_csv(f'subs/{name}.csv.gz', index=False, compression='gzip')\n\n    dist_new_whale = np.ones_like(test_dists[:, :1])\n    dist_new_whale[:] = new_whale_thresh\n    final_answer = np.c_[test_dists, dist_new_whale]\n\n    _create_proto_submission(final_answer, submission_filename, classes)\n    print(submission_filename,pd.read_csv(f'subs/{submission_filename}.csv.gz').Id.str.split().apply(lambda x: x[0] == 'new_whale').mean(),len(set(pd.read_csv(f'subs/{submission_filename}.csv.gz').Id.str.split().apply(lambda x: x[0]).values)))\n    display(pd.read_csv(f'subs/{submission_filename}.csv.gz').head())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"64a300ad7de1386bb4bd49f333b35732d961bc44","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"44a9146c3844dc111585982f4addec7623d4b167","scrolled":true,"trusted":false},"cell_type":"code","source":"#whale.train\n\n#from dlcliche.image import *\n#sys.path.append('..') # app\n#sys.path.append('../..') # root\nfrom easydict import EasyDict\n#from app_utils_clf import *\n#from whale_utils import *\n#from config import DATA_PATH\n\n# Basic training parameters\nargs = EasyDict()\nargs.distance = 'l2'\nargs.n_train = 1\nargs.n_test = 1\nargs.q_train = 1\nargs.q_test = 1\n\nargs.k_train = 50\nargs.k_test = 10\nSZ = 224\nRE_SZ = 256\n\nargs.n_epochs = 1\nargs.drop_lr_every = 50\nargs.lr = 3e-3\nargs.init_weight = None\n\ndata_train = DATA_PATH+'/train'\ndata_test  = DATA_PATH+'/test'\n\nargs.param_str = f'app_whale_n{args.n_train}_k{args.k_train}_q{args.q_train}'\nargs.checkpoint_monitor = 'categorical_accuracy'\nargs.checkpoint_period = 50\n\nprint(f'Training {args.param_str}.')\n\n# Data\ndf = pd.read_csv(DATA_PATH+'train.csv')\ndf = df[df.Id != 'new_whale']\nids = df.Id.values\nclasses = sorted(list(set(ids)))\nimages = df.Image.values\nall_cls2imgs = {cls:images[ids == cls] for cls in classes}\n\ntrn_images = [image for image, _id in zip(images, ids) if len(all_cls2imgs[_id]) >= 2]\ntrn_labels = [_id   for image, _id in zip(images, ids) if len(all_cls2imgs[_id]) >= 2]\nval_images = [image for image, _id in zip(images, ids) if len(all_cls2imgs[_id]) == 2]\nval_labels = [_id   for image, _id in zip(images, ids) if len(all_cls2imgs[_id]) == 2]\n\nargs.episodes_per_epoch = len(trn_images) // args.k_train + 1\nargs.evaluation_episodes = 100 # setting small value, anyway validation set is almost useless here\n\nprint(f'Samples = {len(trn_images)}, {len(val_images)}')\n\n# Model\nfeature_model = get_resnet18(device=device, weight_file=args.init_weight)\n\n# Dataloader\nbackground = WhaleImages(data_train, trn_images, trn_labels, re_size=RE_SZ, to_size=SZ)\nbackground_taskloader = DataLoader(\n    background,\n    batch_sampler=NShotTaskSampler(background, args.episodes_per_epoch, args.n_train, args.k_train, args.q_train)\n)\nevaluation = WhaleImages(data_train, val_images, val_labels, re_size=RE_SZ, to_size=SZ, train=False)\nevaluation_taskloader = DataLoader(\n    evaluation,\n    batch_sampler=NShotTaskSampler(evaluation, args.episodes_per_epoch, args.n_test, args.k_test, args.q_test) \n)\n\n# Train\ntrain_proto_net(args,\n                model=feature_model,\n                device=device,\n                path='.',\n                n_epochs=args.n_epochs,\n                background_taskloader=background_taskloader,\n                evaluation_taskloader=evaluation_taskloader,\n                drop_lr_every=args.drop_lr_every,\n                evaluation_episodes=args.evaluation_episodes,\n                episodes_per_epoch=args.episodes_per_epoch,\n                lr=args.lr,\n               )\ntorch.save(feature_model.state_dict(), f'{args.param_str}_epoch{args.n_epochs}.pth')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ed67b4465350b4aa506c7cc6936530280ce6d62b","trusted":false},"cell_type":"code","source":"os.listdir('../input/')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"72852c7228d313ea365f91e4dba3e52117a324b7","scrolled":true,"trusted":false},"cell_type":"code","source":"from easydict import EasyDict\n#from app_utils_clf import *\n#from whale_utils import *\n#from config import DATA_PATH\n\nweight = f'{args.param_str}_epoch{args.n_epochs}'\ncalculate_results(weight=weight, SZ=224, get_model_fn=get_resnet18, device=device,train_csv='../input/humpback-whale-identification/train.csv', data_train='../input/humpback-whale-identification/train', data_test='../input/humpback-whale-identification/test')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e9d1484cf81031ac00a281be4eacd9c91f759556","trusted":false},"cell_type":"code","source":"test_dists = np.load(f'test_dists_{weight}.npy')\nnp_describe(test_dists)\nprepare_submission(weight, test_dists, data_test='../input/test',classes=get_classes(data=DATA_PATH), new_whale_thresh=-1.85)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}