{"cells":[{"metadata":{"_uuid":"8d398d09416e96406b77bdf060defa711448b2c8","trusted":true},"cell_type":"code","source":"%matplotlib inline\n\nimport os\nimport zipfile\nimport logging\nimport torch\nimport kaggle\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\n\nfrom torchvision import transforms, models\nfrom sklearn.model_selection import train_test_split\nfrom skimage import io, transform, img_as_float\nfrom torch.utils.data import Dataset, DataLoader\n\nos.chdir('..') # change working directory to 1 level up\n\nLOCAL=False # SET THIS\nGPU=True # SET THIS\nsns.set(style=\"white\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"077bbb9bd50929f732d7d3b459e92fa9e7a8b9a4"},"cell_type":"code","source":"train_csv_path = \"data/train.csv\" if LOCAL else \"input/human-protein-atlas-image-classification/train.csv\"\ntrain_images_path = \"data/train_images\" if LOCAL else \"input/human-protein-atlas-image-classification/train\"\ntest_images_path = \"data/test_images\" if LOCAL else \"input/human-protein-atlas-image-classification/test\"\nweights_path = \"work/vgg16/hpa/vgg16-3ch-cyclic_lr-25epoch/25.pth\" if LOCAL else \"input/vgg16pretrainedweights/weights/25.pth\"\nsample_submission_path = \"sample_submission.csv\" if LOCAL else \"input/human-protein-atlas-image-classification/sample_submission.csv\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5c83cadb8305a19ef99b471ea92adc8199a5d88e"},"cell_type":"markdown","source":"## Understanding the dataset"},{"metadata":{"_uuid":"5098289d6332ffa267cf875cb29a59ad3a540d7b","trusted":true},"cell_type":"code","source":"train_labels = pd.read_csv(train_csv_path)\ntrain_labels.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"088bb1b59c32acb66ae304f6407528da9b28ccab","trusted":true},"cell_type":"code","source":"train_labels.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c8445800627ba2295d9a236412369f224d403693"},"cell_type":"markdown","source":"We have 31k training images to work with. This will later be split into a training and test set. Anticipating that this is a multi-classification/multi-label problem, let's split the Target variable, which are strings, into an array of one-hot-encoded variables."},{"metadata":{"_uuid":"9f85257051361bcb15569c93879c88e7304c5f62","trusted":true},"cell_type":"code","source":"one_hot = train_labels.Target.str.get_dummies(sep=' ')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3a15d234a833a6cf545d247630b2b9681a9f922e","trusted":true},"cell_type":"code","source":"one_hot.columns = map(int, one_hot.columns); one_hot.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"67b2bd4107ad708ed1f4a2385a36317c99fa42e5"},"cell_type":"markdown","source":"## Label names"},{"metadata":{"_uuid":"19e288ce354df8fb7b65415367d397a1415370ec","trusted":true},"cell_type":"code","source":"label_names = {\n    0:  \"Nucleoplasm\",  \n    1:  \"Nuclear membrane\",   \n    2:  \"Nucleoli\",   \n    3:  \"Nucleoli fibrillar center\",   \n    4:  \"Nuclear speckles\",\n    5:  \"Nuclear bodies\",   \n    6:  \"Endoplasmic reticulum\",   \n    7:  \"Golgi apparatus\",   \n    8:  \"Peroxisomes\",   \n    9:  \"Endosomes\",   \n    10:  \"Lysosomes\",   \n    11:  \"Intermediate filaments\",   \n    12:  \"Actin filaments\",   \n    13:  \"Focal adhesion sites\",   \n    14:  \"Microtubules\",   \n    15:  \"Microtubule ends\",   \n    16:  \"Cytokinetic bridge\",   \n    17:  \"Mitotic spindle\",   \n    18:  \"Microtubule organizing center\",   \n    19:  \"Centrosome\",   \n    20:  \"Lipid droplets\",   \n    21:  \"Plasma membrane\",   \n    22:  \"Cell junctions\",   \n    23:  \"Mitochondria\",   \n    24:  \"Aggresome\",   \n    25:  \"Cytosol\",   \n    26:  \"Cytoplasmic bodies\",   \n    27:  \"Rods & rings\"\n}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b8b268f596487e5f6c2bc432c0bbeda1ae993272"},"cell_type":"markdown","source":"## Class Counts"},{"metadata":{"_uuid":"ac1de64b56d5474294657bf89200b9404eb849c6","trusted":true},"cell_type":"code","source":"counts = one_hot.agg('sum')[:].rename(lambda x: label_names[x]).sort_values(ascending=False)\nplt.figure(figsize=(12,10))\ncounts.plot('bar')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5fee5d6a7e638ac5b2930b1084d767e95913e070","trusted":true},"cell_type":"code","source":"counts.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e3d224313fb180ebe787c1b12704b7494c2797c4","trusted":true},"cell_type":"code","source":"counts.tail()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"83565c2c5c5aa56bcabcd937aef8af685b5ac59c"},"cell_type":"markdown","source":"Append one_hot labels to our train labels dataframe:"},{"metadata":{"_uuid":"4fd49ef2c8ba2d73f8bd421f3d04210046f31ebf","trusted":true},"cell_type":"code","source":"train_labels = train_labels.join(one_hot.sort_index(axis=1))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ced282c720692c74f881bb9f285a31667d36ac70"},"cell_type":"markdown","source":"## Co-occurences"},{"metadata":{"_uuid":"43edec00e263bfb0d2bba3e1bb72c98809df183d","trusted":true},"cell_type":"code","source":"tmp1 = train_labels.iloc[:,2:]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"79d6ea10e3dfa229f2a6378c44779f0996ab56d6","trusted":true},"cell_type":"code","source":"co_occur = tmp1.T.dot(tmp1); co_occur.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6f097705cd31d632c7c107ca52be3facd393fcc9"},"cell_type":"markdown","source":"Now that we have a co-occurence matrix, let's plot it with a heatmap."},{"metadata":{"_uuid":"ae70dfb50bebe0696da16bcaaf0cc6e9e78012d8","trusted":true},"cell_type":"code","source":"mask = np.zeros_like(co_occur, dtype=np.bool)\nmask[np.triu_indices_from(mask)] = True\ncmap = sns.diverging_palette(220, 10, as_cmap=True)\nplt.figure(figsize=(12,10))\nsns.heatmap(co_occur, mask=mask, cmap=cmap, vmax=10000, center=0,\n            square=True, linewidths=.5, cbar_kws={\"shrink\": .5})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"47d9855d100d53c5c0b2f485f6655c1da0adcb95"},"cell_type":"markdown","source":"The above plot doesn't tell me much because most of the variables have so few occurences that few, can be seen above the 1k number of co-occurences. Let's try plotting it on a log base-10 scale."},{"metadata":{"_uuid":"cdbcb00351a8e592e92a1704f73d8512ac29abae","trusted":true},"cell_type":"code","source":"co_occur1 = co_occur.apply(np.log, args=10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ad1d6a5d91d7dd20fc59cf6f5f8212a1e3aa8796","trusted":true},"cell_type":"code","source":"mask = np.zeros_like(co_occur1, dtype=np.bool)\nmask[np.triu_indices_from(mask)] = True\nplt.figure(figsize=(12,10))\nsns.heatmap(co_occur1, mask=mask, cmap=cmap, vmax=10, vmin=0, center=5,\n            square=True, linewidths=.5, cbar_kws={\"shrink\": .5}, annot=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e315eccf777a69270dc609abf58d6e53ccef20a6"},"cell_type":"markdown","source":"This plot is helpful because, while still showing the high frequency of co-occurence protein 0  and 25 have with many other columns, It also shows the lack of co-occurence that protein 8,9 and 10 have with any other proteins. We may be able to use this later."},{"metadata":{"_uuid":"0d80e55c8a55cc1c7479c81d09a3921701be9c4a"},"cell_type":"markdown","source":"## Image reconnaissance\n\nLet's take a look at some of the images and try to imagine what we want our net to find here"},{"metadata":{"_uuid":"a1b3fe1860b16d57e548f65d068c85e42456356c","trusted":true},"cell_type":"code","source":"print(len([name for name in os.listdir(train_images_path) if os.path.isfile(os.path.join(train_images_path, name))]))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"29e2b79bc918c13fc65bf335277ad73476a096c2","trusted":true},"cell_type":"code","source":"len(train_labels) * 4","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3bec785b72b447b63da7bf086089c60b6cc74f7e"},"cell_type":"markdown","source":"Good! The number of files in the train_images directory matches 4x the number of rows in our train_labels dataframe. There is one stained image - red, blue, green, yellow for each image Id. The below code chooses a uniform random sample of 4 images and displays each color."},{"metadata":{"_uuid":"fa39b8340bee902cc3433883b7ca96b26727d8fb","trusted":true},"cell_type":"code","source":"id_list = train_labels.sample(4).Id.tolist(); id_list","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"11ee5b400769ad644d7ab19d36f04e9fe93a157a","trusted":true},"cell_type":"code","source":"def plot_images_row(img_id, ax_row):\n    filters = ['red', 'green', 'blue', 'yellow']\n    colormaps = ['Reds', 'Greens', 'Blues', 'Oranges']\n    \n    for c, ax, cmap in zip(filters, ax_row, colormaps):\n        filename = img_id + '_' + c + '.png'\n        img=mpimg.imread(os.path.join(train_images_path, filename))\n        imgplot = ax.imshow(img, cmap=cmap)\n\nfig = plt.figure(figsize=(12,10))\naxes = fig.subplots(nrows=4, ncols=4)\n\nfor img_id, ax_row in zip(id_list, axes):\n    plot_images_row(img_id, ax_row)\n\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"16aff54cf85a3bee13964e0a9300e5ceb7c3cdd1"},"cell_type":"markdown","source":"We can see that each color is vaguely reminiscent of the others for the same image. Below is a larger version of one of the images shown in blue."},{"metadata":{"_uuid":"3f3ea13b3b6e61237d6de5c3ebbbf0d879db223f","trusted":true},"cell_type":"code","source":"color = 'blue'\nfilename = id_list[0] + '_' + color + '.png'\nimg=mpimg.imread(os.path.join(train_images_path, filename))\nplt.figure(figsize=(12,10))\nplt.imshow(img, cmap=\"Blues\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bd5ac0b84297264e0905f251996ca396e7103508"},"cell_type":"markdown","source":"# Baseline modeling\n## Convolutional Neural Network"},{"metadata":{"_uuid":"e4a0e6ddf26802078907db839e370ee1688576a2"},"cell_type":"markdown","source":"Conv nets take inputs as two-dimensional images and predict labels based on convolutions applied to outputs from each subsequent layer. Mathematically it is to process cross-correlations as opposed to convolutions although they are related.\n\nFor single-label classification we ask a simple question:\n\n![car](img/streetview.jpg)\n\ni.e. Is this a picture of a car? ∈ {yes, no}"},{"metadata":{"_uuid":"9f109469efc91d329cdde62c7b176d96592d0fc7"},"cell_type":"markdown","source":"For multi-class (per-label) classification we are interested in asking the question, \"which labels are relevant to the picture?\" ⊆ {car, streetlight, pedestrian, cyclist, signpost, etc..}\ni.e., each instance can have multiple labels instead of a single one!\n\nCross-Entropy loss function is commonly used for learning a multi-class classification CNN model where overall loss on a mini-batch of n images is taken as the average additive sum of attribute-level loss with equal weight applied over all labels.\n\n\\begin{equation*}\n\\mathcal{L}_{ce} = -\\frac{1}{n_{bs}} \\sum^{n_{bs}}_{i=1}\\sum^{n_{attr}}_{j=1}log\\left(p(y_{i,j} = a_{i,j} | \\textbf{x}_{i,j})\\right)\n\\end{equation*}"},{"metadata":{"_uuid":"ec0507db281709f9b08dda20e6ad85ea836519d0"},"cell_type":"markdown","source":"$\\textbf{x}_{i,j}$ denotes the feature vector of $\\textbf{I}_{i}$ for the jth attribute label and $p(y_{i,j} = a_{i,j}|\\textbf{x}_{i,j}$) is the corresponding posterior probability of $\\textbf{I}_{i}$ over the ground truth $a_{i, j}$"},{"metadata":{"_uuid":"f3f1fb167e6b8bb9965c9909e14aecadcbaf456d"},"cell_type":"markdown","source":"The cross-entropy loss function is conditioning model learning to minimize training error by assuming that individual samples and classes are of equal importance. In order to achieve good performance and generalization, networks trained with CE need to have large training sets with sufficiently balanced class distributions."},{"metadata":{"_uuid":"3f77fe213302066cf98a0aa80f202a3355951c3e"},"cell_type":"markdown","source":"## Minority Class Hard Sample Mining\n\nWe explore method of mining hard examples to supplement the baseline CE loss by selectively \"borrowing majority class samples from the marginal border regions of the geometric neighborhood structure.\n\nFor hard sample mining, we first profile the minority and majority classes per label in each training mini-batch with n training samples. We profile the class distribution hj"},{"metadata":{"_uuid":"d5e96d5b13bd5c92e8fd6f6d5e9e30e2294cc0f6"},"cell_type":"markdown","source":"First some supporting functions:"},{"metadata":{"_uuid":"b267e45f08e5a1287b016ef2ad29b47117a25045","trusted":true},"cell_type":"code","source":"def to_one_hot(df):\n    tmp = df.Target.str.get_dummies(sep=' ')\n    tmp.columns = map(int, tmp.columns)\n    return df.join(tmp.sort_index(axis=1))\n\ndef get_image_ids_from_dir_contents(image_dir):\n    all_images = [name for name in os.listdir(image_dir) \\\n                  if os.path.isfile(os.path.join(image_dir, name))]\n    return list(set([name.split('_')[0] for name in all_images]))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"9e3081a09f05e7c0afb0d35e299b04cc7df75689","trusted":true},"cell_type":"code","source":"class TrainImageDataset(Dataset):\n    \"\"\"Fluorescence microscopy images of protein structures training dataset\"\"\"\n\n    def __init__(self,\n        image_dir,\n        label_file,\n        transform=None,\n        idxs=None,\n        using_pil=False\n    ):\n        \"\"\"\n        Args:\n            label_file (string): Path to the csv file with annotations.\n            image_dir (string): Directory with all the images.\n            transform (callable, optional): Optional transform to be applied\n                on a sample.\n        \"\"\"\n        self.image_dir = image_dir\n        self.transform = transform\n        self.idxs = idxs\n        self.labels = to_one_hot(pd.read_csv(label_file))\n        self.using_pil = using_pil\n        if self.idxs is not None:\n            self.labels = self.labels.iloc[self.idxs, :].\\\n                                                reset_index(drop=True)\n\n    def __len__(self):\n        return len(self.labels)\n\n    def __getitem__(self, idx):\n        img_name = self.labels.iloc[idx, 0]\n        img_red = img_name + '_red.png'\n        img_blue = img_name + '_blue.png'\n        img_green = img_name + '_green.png'\n        img_yellow = img_name + '_yellow.png'\n\n        if self.using_pil:\n            pth2img = lambda x: io.imread(x)\n        else:\n            pth2img = lambda x: img_as_float(io.imread(x))\n\n        img_red = pth2img(os.path.join(self.image_dir, img_red))\n        img_blue = pth2img(os.path.join(self.image_dir, img_blue))\n        img_green = pth2img(os.path.join(self.image_dir, img_green))\n        img_yellow = pth2img(os.path.join(self.image_dir, img_yellow))\n        labels = self.labels.iloc[idx, 2:].values\n        labels = labels.astype('int')\n        sample = {'image_id': img_name,\n                  'image_red': img_red,\n                  'image_blue': img_blue,\n                  'image_green': img_green,\n                  'image_yellow': img_yellow,\n                  'labels': labels}\n\n        if self.transform:\n            sample = self.transform(sample)\n\n        return sample\n    \n\nclass TestImageDataset(Dataset):\n    \"\"\"Fluorescence microscopy images of protein structures test dataset\"\"\"\n\n    def __init__(self,\n        image_dir,\n        transform=None,\n        using_pil=False\n    ):\n        \"\"\"\n        Args:\n            image_dir (string): Directory with all the images.\n            transform (callable, optional): Optional transform to be applied\n                on a sample.\n        \"\"\"\n        self.image_ids = get_image_ids_from_dir_contents(image_dir)\n        self.image_dir = image_dir\n        self.transform = transform\n        self.using_pil = using_pil\n\n    def __len__(self):\n        return len(self.image_ids)\n\n    def __getitem__(self, idx):\n        img_name = self.image_ids[idx]\n        img_red = img_name + '_red.png'\n        img_blue = img_name + '_blue.png'\n        img_green = img_name + '_green.png'\n        img_yellow = img_name + '_yellow.png'\n\n        if self.using_pil:\n            pth2img = lambda x: io.imread(x)\n        else:\n            pth2img = lambda x: img_as_float(io.imread(x))\n\n        img_red = pth2img(os.path.join(self.image_dir, img_red))\n        img_blue = pth2img(os.path.join(self.image_dir, img_blue))\n        img_green = pth2img(os.path.join(self.image_dir, img_green))\n        img_yellow = pth2img(os.path.join(self.image_dir, img_yellow))\n        sample = {'image_id': img_name,\n                  'image_red': img_red,\n                  'image_blue': img_blue,\n                  'image_green': img_green,\n                  'image_yellow': img_yellow,\n                  'labels' : np.zeros(28)}\n\n        if self.transform:\n            sample = self.transform(sample)\n\n        return sample\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"16a74c5b49612ea61aad6726b44af4e01afe0cc4","trusted":true},"cell_type":"code","source":"class CombineColors(object):\n    \"\"\"Combines the the image in a sample to a given size.\"\"\"\n\n    def __call__(self, sample):\n        img_name = sample['image_id']\n        img_red = sample['image_red']\n        img_blue = sample['image_blue']\n        img_green = sample['image_green']\n        img_yellow = sample['image_yellow']\n        labels = sample['labels']\n        image = np.dstack((img_red, img_green, img_blue, img_yellow))\n\n        return {'image': image, 'labels': labels, 'image_id': img_name}\n\n\nclass ToPILImage(object):\n    \"\"\"Convert ndarrays in sample to Tensors.\"\"\"\n    def __init__(self, mode=None):\n        self.mode = mode\n\n    def __call__(self, sample):\n        img_name = sample['image_id']\n        image = sample['image']\n        labels = sample['labels']\n        image = transforms.ToPILImage(self.mode)(image)\n\n        return {'image': image,\n                'labels': labels,\n                'image_id': img_name}\n\n\nclass RandomResizedCrop(object):\n    \"\"\"Convert ndarrays in sample to Tensors.\"\"\"\n    def __init__(self, size=224):\n        self.size = size\n\n    def __call__(self, sample):\n        img_name = sample['image_id']\n        image = sample['image']\n        labels = sample['labels']\n        image = transforms.RandomResizedCrop(self.size)(image)\n\n        return {'image': image,\n                'labels': labels,\n                'image_id': img_name}\n\n\nclass RandomHorizontalFlip(object):\n    \"\"\"Convert ndarrays in sample to Tensors.\"\"\"\n\n    def __call__(self, sample):\n        img_name = sample['image_id']\n        image = sample['image']\n        labels = sample['labels']\n        image = transforms.RandomHorizontalFlip()(image)\n\n        return {'image': image,\n                'labels': labels,\n                'image_id': img_name}\n\n\nclass Resize(object):\n    \"\"\"Convert ndarrays in sample to Tensors.\"\"\"\n    def __init__(self, size=256):\n        self.size = size\n\n    def __call__(self, sample):\n        img_name = sample['image_id']\n        image = sample['image']\n        labels = sample['labels']\n        image = transforms.Resize(self.size)(image)\n\n        return {'image': image,\n                'labels': labels,\n                'image_id': img_name}\n\n\nclass CenterCrop(object):\n    \"\"\"Convert ndarrays in sample to Tensors.\"\"\"\n    def __init__(self, size=224):\n        self.size = size\n\n    def __call__(self, sample):\n        img_name = sample['image_id']\n        image = sample['image']\n        labels = sample['labels']\n        image = transforms.CenterCrop(self.size)(image)\n\n        return {'image': image,\n                'labels': labels,\n                'image_id': img_name}\n\n\nclass ToTensor(object):\n    \"\"\"Convert ndarrays in sample to Tensors.\"\"\"\n\n    def __call__(self, sample):\n        img_name = sample['image_id']\n        image = sample['image']\n        labels = sample['labels']\n        image = transforms.ToTensor()(image)\n\n        return {'image': image.type(torch.FloatTensor),\n                'labels': torch. \\\n                    from_numpy(labels).type(torch.FloatTensor),\n                'image_id': img_name}\n\n\nclass NumpyToTensor(object):\n    \"\"\"Convert ndarrays in sample to Tensors.\"\"\"\n\n    def __call__(self, sample):\n        img_name = sample['image_id']\n        image = sample['image']\n        labels = sample['labels']\n        # swap color axis because\n        # numpy image: H x W x C\n        # torch image: C X H X W\n        image = image.transpose((2, 0, 1))\n\n        return {'image': torch. \\\n                    from_numpy(image).type(torch.FloatTensor),\n                'labels': torch. \\\n                    from_numpy(labels).type(torch.FloatTensor),\n                'image_id': img_name}\n\n\nclass Normalize(object):\n    \"\"\"Normalize a tensor image with mean and standard deviation.\n    Given mean: ``(M1,...,Mn)`` and std: ``(S1,..,Sn)`` for ``n`` channels,\n    this transform will normalize each channel of the input ``torch.*Tensor``\n    i.e.\n    ``input[channel] = (input[channel] - mean[channel]) / std[channel]``\n    .. note::\n        This transform acts in-place, i.e., it mutates the input tensor.\n    Args:\n        mean (sequence): Sequence of means for each channel.\n        std (sequence): Sequence of standard deviations for each channel.\n    \"\"\"\n\n    def __init__(self, mean, std):\n        self.mean = mean\n        self.std = std\n\n    def __call__(self, sample):\n        \"\"\"\n        Args:\n            tensor (Tensor): Tensor image of size (C, H, W) to be normalized.\n        Returns:\n            Tensor: Normalized Tensor image.\n        \"\"\"\n        img_name = sample['image_id']\n        image = sample['image']\n        labels = sample['labels']\n        image = transforms.Normalize(self.mean, self.std)(image)\n\n        return {'image': image,\n                'labels': labels,\n                'image_id': img_name}\n\n    def __repr__(self):\n        return self.__class__.__name__ + '(mean={0}, std={1})'.\\\n                                            format(self.mean, self.std)\n\n\ndef get_transforms(pretrained=False):\n    if pretrained:\n        transform = {\n            'TRAIN': transforms.Compose(\n                            [CombineColors(),\n                             ToPILImage(),\n                             RandomResizedCrop(224),\n                             RandomHorizontalFlip(),\n                             ToTensor(),\n                             Normalize(mean=[0.485, 0.456, 0.406],\n                                        std=[0.229, 0.224, 0.225])\n                            ]\n            ),\n            'DEV': transforms.Compose(\n                            [CombineColors(),\n                             ToPILImage(),\n                             Resize(256),\n                             CenterCrop(224),\n                             ToTensor(),\n                             Normalize(mean=[0.485, 0.456, 0.406],\n                                        std=[0.229, 0.224, 0.225])\n                            ]\n            )\n        }\n    else:\n        transform = {\n            'TRAIN': transforms.Compose(\n                            [CombineColors(),\n                             NumpyToTensor()\n                             ]\n            ),\n            'DEV': transforms.Compose(\n                            [CombineColors(),\n                             NumpyToTensor()\n                             ]\n            )\n        }\n\n    return transform","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c675a818ff8c3772a435ba3fa7afd6477b23014c","trusted":true},"cell_type":"code","source":"train_dataset = TrainImageDataset(image_dir=train_images_path,\n                                     label_file=train_csv_path)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"adcd3cea183c2abfbae199acc98c9e3175546ccc","trusted":true},"cell_type":"code","source":"sample = train_dataset[120]","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"_uuid":"fc55ec214a4b91cbdaa7c36136865ba94f68c574","trusted":true},"cell_type":"code","source":"sample","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4fca7533da65d1795973aac29b13372334975b1e","trusted":true},"cell_type":"code","source":"sample['image_red'].shape == sample['image_blue'].shape == \\\nsample['image_green'].shape ==  sample['image_yellow'].shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f3b7b1d68f3f7c8a3586a6ff3440a385cdccdd44","trusted":true},"cell_type":"code","source":"len(sample['labels']) == 28","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"db9693be6161f9882f86e2fd8320d7fa5adde8a7","trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12,10))\nplt.imshow(sample['image_yellow'], cmap=\"Oranges\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cf99974292275fc7f593aa3a6e5c03bb36b7bc85","trusted":true},"cell_type":"code","source":"transform = get_transforms(pretrained=False)\ntrain_dataset = TrainImageDataset(image_dir=train_images_path,\n                                     label_file=train_csv_path,\n                                     transform=transform['TRAIN'])\nkwargs = {'batch_size': 32}\ntrainLoader = DataLoader(train_dataset, shuffle=True, **kwargs)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"34bb412fbc616e908f54a9246030b6328bf9d2ce","trusted":true},"cell_type":"code","source":"data = next(iter(trainLoader))\ninputs, labels = data['image'], data['labels']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a0c5cd1d7efdbefa2e3e8906a7b9b367749950f6","trusted":true},"cell_type":"code","source":"nbs = kwargs['batch_size'] # num examples in batch\nncl = 28 # num classes","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"910ba1d50b100d251537b7fb2b7f0121315736d3"},"cell_type":"markdown","source":"#### Class Distribution profile\n\n$h_k^j$ denotes the number of training samples with j-th attribute value assigned to class k"},{"metadata":{"_uuid":"49773ec213d08a0bcfbdb27d59cb74a93a9aac47","trusted":true},"cell_type":"code","source":"hjk = labels.sum(0); hjk","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d7416247fdd663bea84448b77a4fe154f1d85008","trusted":true},"cell_type":"code","source":"cls_labels = torch.arange(ncl); cls_labels # not required ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3a2275a58d9f8a54711f4747b5abf212647e9858","trusted":true},"cell_type":"code","source":"sorted_cls_labels = np.argsort(hjk); sorted_cls_labels","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9cb752534f4916de21e556e68dc2fb2931db06ff","trusted":true},"cell_type":"code","source":"sorted_hjk = hjk[sorted_cls_labels]; sorted_hjk","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c5d6cc8f702d783b48cf10a38d89636a3eb63e37"},"cell_type":"markdown","source":"To determine minority class indexes, for the multiclass setting, we set ρ=50%, meaning that all minority classes collectively account for at most half or less samples per batch. The function below is later vectorized in pytorch"},{"metadata":{"_uuid":"6f6e7aeeb4090576e945cb391b54002994aced18","trusted":true},"cell_type":"code","source":"th = .5 * nbs\n\ndef get_min_class_boundary(arr):\n    for idx in torch.arange(len(arr)):\n        if arr[:idx].sum() > th:\n            return idx - 1\n    return arr.size","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"57c1f5762d11981d6c5c357cbf4b1c411bc09b27","trusted":true},"cell_type":"code","source":"bound = get_min_class_boundary(sorted_hjk); bound","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b2741cf5af9dbef86cde0e410f922f31c46fc213","trusted":true},"cell_type":"code","source":"sorted_hjk[:bound].sum() # should be less than or equal  16","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a59a855b691f61557e7e435ac2b05627ec4bb63a","trusted":true},"cell_type":"code","source":"sorted_hjk = sorted_hjk[:bound]\nmin_cls_labels = sorted_cls_labels[:bound]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d518a0152647dad836e4bfe49b0dc97307ac3f54","trusted":true},"cell_type":"code","source":"idxs = np.argsort(min_cls_labels) # unsort\nmin_cls_labels = min_cls_labels[idxs] \nhjk = sorted_hjk[idxs]\nprint(min_cls_labels)\nprint(hjk)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7410f8f9a5dbc2cefadb511f2fdfc6225bb7c9e0"},"cell_type":"markdown","source":"Unfortunately the minority classes which have less than two samples must be ignored so we must also filter on hjk > 1. This enables us to use a more flexible loss function, e.g. triplet loss which requires at least two matched samples as it is impossible to construct a triple where only one or no sample images exist for that class in the batch."},{"metadata":{"_uuid":"480a951fd1bd1adec752cb5406fa3205941e1be9","trusted":true},"cell_type":"code","source":"msk = hjk > 1\nhjk = hjk[msk]\nmin_cls_labels = min_cls_labels[msk]\nprint(min_cls_labels)\nprint(hjk)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9b62072d513ec1106dc639e0407d0e8b6aafeca2"},"cell_type":"markdown","source":"Below I use a function from source that does the above in pytorch."},{"metadata":{"_uuid":"5d2dd7c6e63e6f53ca6accbf7c35b72dc5f0d4b3","trusted":true},"cell_type":"code","source":"def get_minority_classes(y, batchSz):\n    sorted_hjk, ix = y.sum(0).sort()\n    mask = torch.cumsum(sorted_hjk, 0) <= .5 * batchSz\n    sorted_hjk = sorted_hjk[mask]\n    sorted_, sorted_ix = ix = ix[mask].sort()\n    \n    return sorted_[sorted_hjk[sorted_ix] > 1]\n\nmin_cls_labels = get_minority_classes(labels, nbs); min_cls_labels","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4f4b5f5a27b003fbbc1426bce2d4d18130bd3f7e"},"cell_type":"markdown","source":"Note: possible difference in class indices. Since sort is somewhat random in its ordering of integer values that are the same, class choice may be somewhat random at margin of minority classes and non-minority classes."},{"metadata":{"_uuid":"2d2586d5cb224e959c1700655c81523aac02b613"},"cell_type":"markdown","source":"Given the minortiy classes, let us find the hardness metric for sampling instances which encourage model learning to concentrate on weak recognitions or obvious mistakes. Explicitly, at the class level, we quantify sample hardness regarding a given class per label by saying for each minority class c of the attribute label j, we refer to \"hard-positives\" as follows: \n\n\\begin{align}\nP_{c,j}^{cls} = \\{x_{i,j} | a_{i,j} = c\\text{, low } p(y_{i,j} = c | x_{i,j})\\}\n\\end{align}\n\n\\begin{align}\nN_{c,j}^{cls} = \\{x_{i,j} | a_{i,j} \\neq c\\text{, low } p(y_{i,j} = c | x_{i,j})\\}\n\\end{align}"},{"metadata":{"_uuid":"14cbf2037c7c64decdfd8b1c820de1cc4c87b7d6"},"cell_type":"markdown","source":"Create some random predictions in the range [0-1]:"},{"metadata":{"_uuid":"7b7458f3c6e6c9031e9ae83bbd7234624fa456a4","trusted":true},"cell_type":"code","source":"bs = (nbs, ncl) # batch array size\npreds = np.random.rand(*bs) # random predictions for batch \npreds = torch.Tensor(preds)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f3b85ea90505d351fa7e40404f8b85fed1a4bf04"},"cell_type":"markdown","source":"Identify positive examples which are associated with a minority class.\n\n$x_{i,j} | a_{i,j} = c$"},{"metadata":{"_uuid":"41cae9ab0c48ba0bdefc04ad665a9cc0380dfcb4","trusted":true},"cell_type":"code","source":"y_min = labels[:, min_cls_labels]\n# y_min = labels.numpy()[:, min_cls_labels.numpy()]\nmsk = y_min == 1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e728c0b545605a5760f137f96a8eddf7bf804c19","trusted":true},"cell_type":"code","source":"P = torch.nonzero(msk); P # anchor instances\n# P = np.argwhere(msk)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e6f3654c8413629ab209aa9ba90b6e04af410b91"},"cell_type":"markdown","source":"Identify negative examples which are associated with a minority class.\n\n$x_{i,j} | a_{i,j} \\neq c$"},{"metadata":{"_uuid":"9674cb2bbec9684ff1119ffd28218132ea209dd6","scrolled":true,"trusted":true},"cell_type":"code","source":"N = torch.nonzero(~msk)\n# N = np.argwhere(~msk)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2484c0043667f8c292aa2927e41b65822d039bfa"},"cell_type":"markdown","source":"get probabilities for positive examples associated with a minority class"},{"metadata":{"_uuid":"eb346ea0fe4588f2e0fbcb7d69bb2496bee7f739","trusted":true},"cell_type":"code","source":"preds_min = preds[:, min_cls_labels]\npreds_P = preds_min[msk]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a886dd1001a782691be1ef4f062cd1a910ae5242"},"cell_type":"markdown","source":"get probabilities for negtive examples associated with a minority class"},{"metadata":{"_uuid":"c1a5011d098c019cda0d7fd530bff33833b960d4","trusted":true},"cell_type":"code","source":"preds_N = preds_min[~msk]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c57411a8d7e126059308740ad53ef61285708006"},"cell_type":"markdown","source":"select top 3 (if needed)"},{"metadata":{"_uuid":"448a410e99f8b1b0276cac68d9c8cd3bbad62ca2","trusted":true},"cell_type":"code","source":"k = 3\npreds_P[np.argsort(preds_P)][:k]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d62a54803cbb3c7cc37df76370c3338a4eb58b25","trusted":true},"cell_type":"code","source":"preds_N[np.argsort(preds_N)][-k:]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6e29d4e8ae3dcc4472f62db05d0781535a4dca28"},"cell_type":"markdown","source":"### Incremental Batch-Wise Minority Class Example Mining\n\nNow that we know how to break down the batch into minority classes sorted on predicted probabilities, we can think about how to select hard examples. Specifically, at training time, for a minority class c of attribute label j (or a minority class instance $x_{i,j}$ ) in each training batch data, we select κ hard-positives as the bottom-κ scored on c (or bottom-κ (largest) distances to $x_{i,j}$ ), and κ hard-negatives as the top-κ scored on c (or top-κ (smallest) distance to $x_{i,j}$ ), given the current model (or feature space)"},{"metadata":{"_uuid":"989bcf939f22041ff0f9298a41277b8933352a79"},"cell_type":"markdown","source":"OK, lets put it all together. Given a tensor of y_predictions for minority labels..."},{"metadata":{"_uuid":"e4538f444d1699be8d1da3b999fe71f49347aef4","trusted":true},"cell_type":"code","source":"preds_min[:5] # head","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7362b64327c9c1c59603df8faf42f8d87c0e86bd"},"cell_type":"markdown","source":"We form at most $κ^2$ triplets $T = \\{(x_{a,j}, x_{+,j}, x_{-,j})_s\\}_{s=1}^{\\kappa^2}$ with respect to $x_{a, j}$, and a total of at most $\\left|X_{min}\\right| × κ^2$ triplets $T$ for all anchors $X_{min}$ across all the minority classes of every attribute label. The meshgrid function is used to create row-wise combinations of indexes/probabilities."},{"metadata":{"_uuid":"36ed2b1e0fd0e7f69cabcdee2f735b760ad3423c","scrolled":true,"trusted":true},"cell_type":"code","source":"k = 3\nfor idx, row in enumerate(P):\n    anchor_idx, anchor_class = row\n    mask = (P[:, 1] == anchor_class)\n    mask[idx] = 0\n    pos_idxs = P[mask]\n    pos_preds, sorted_= preds_min[pos_idxs[:, 0], pos_idxs[:, 1]].sort()\n    pos_idxs = pos_idxs[sorted_][:k]\n    pos_preds = pos_preds[:k]\n\n    mask = (N[:, 1] == anchor_class)\n    neg_idxs = N[mask]\n    neg_preds, sorted_= preds_min[neg_idxs[:, 0], neg_idxs[:, 1]].sort()\n    neg_idxs = neg_idxs[sorted_][-k:]\n    neg_preds = neg_preds[:k]\n    \n    a = [idx]\n    n_p = pos_idxs.shape[0]\n    n_n = neg_idxs.shape[0]\n    grid = torch.stack(torch.meshgrid([torch.Tensor(a).long(), torch.arange(n_p), torch.arange(n_n)])).reshape(3, -1).t()\n    print(torch.cat([P[grid[:, 0]], pos_idxs[grid[:, 1]], neg_idxs[grid[:, 2]]], 1))\n    print(\"\")\n    print(torch.stack([preds_P[grid[:, 0]], pos_preds[grid[:, 1]], neg_preds[grid[:, 2]]], 1))\n    print(\"\")\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"38ca90a0ba9fc4982419d0f255510b9b0c3c609d"},"cell_type":"markdown","source":"I had first implemented this in numpy as shown in the commentedc code below for those interested."},{"metadata":{"_kg_hide-input":true,"_uuid":"bd8f38ea6cf175f6551849fd43980b72a92c7326","trusted":true},"cell_type":"code","source":"# def mine_positives(anchor, labels, predictions):\n#     cls = np.argwhere(labels[anchor] == 1)\n#     P = np.argwhere(labels == 1)\n#     preds_P = predictions[labels == 1]\n#     out = P[np.isin(P[:, 1], cls)]\n#     out_preds = preds_P[np.isin(P[:, 1], cls)]\n#     input_mask = out[:, 0] != anchor\n#     out = out[input_mask]\n#     out_preds = out_preds[input_mask]\n#     sorted_ = out_preds.argsort()\n#     return out[sorted_], out_preds[sorted_]\n\n# def mine_negatives(anchor, labels, predictions):\n#     cls = np.argwhere(labels[anchor] == 0)\n#     N = np.argwhere(labels == 0)\n#     preds_N = predictions[labels == 0]\n#     out = N[np.isin(N[:, 1], cls)]\n#     out_preds = preds_N[np.isin(N[:, 1], cls)]\n#     sorted_ = out_preds.argsort()\n#     return out[sorted_], out_preds[sorted_]\n\n\n# anchor_idxs = P[:, 0]\n# k = 3\n# for idx in anchor_idxs:\n#     anc_examples, anc_preds = P[P[:, 0] == idx], preds_P[P[:, 0] == idx]\n#     pos_examples, pos_preds = mine_positives(idx, y_min, preds_min)\n#     neg_examples, neg_preds = mine_negatives(idx, y_min, preds_min)\n#     pos_examples = pos_examples[:k]\n#     neg_examples = neg_examples[-k:]\n#     n_a = anc_examples.shape[0]\n#     n_p = pos_examples.shape[0]\n#     n_n = neg_examples.shape[0]\n#     grid = np.array(np.meshgrid(np.arange(n_a), np.arange(n_p), np.arange(n_n))).T.reshape(-1,3)\n#     print(anc_examples[grid[:, 0]], pos_examples[grid[:, 1]], neg_examples[grid[:, 2]])\n#     print(\"\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"64cb1318783de8819d8138e02c976ea7ea7b890a"},"cell_type":"markdown","source":"We define $\\Omega$ as the minimum percentage count of data samples required over all classes in order to form an overall uniform (i.e. balanced) class distribution in the training data . eta is a hyperparameter to be tuned by cross validation."},{"metadata":{"_kg_hide-input":true,"_uuid":"a95c59824193fe7a2cab8c15f2f0c7c31ed4e26f","trusted":true},"cell_type":"code","source":"class TripletLoss(nn.Module):\n    \"\"\"\n    Triplet loss\n    Takes embeddings of an anchor sample, a positive sample and a negative sample\n\n    source: https://github.com/adambielski/siamese-triplet/blob/master/losses.py\n    \"\"\"\n\n    def __init__(self, margin):\n        super(TripletLoss, self).__init__()\n        self.margin = margin\n\n    def forward(self, anchor, positive, negative, size_average=True):\n        distance_positive = F.l1_loss(anchor, positive, reduction='sum')\n        distance_negative = F.l1_loss(anchor, negative, reduction='sum')\n        losses = F.relu(distance_positive - distance_negative + self.margin)\n        return losses.mean() if size_average else losses.sum()\n\n\nclass IncrementalClassRectificationLoss(nn.Module):\n\n    def __init__(self,\n        margin,\n        alpha,\n        batchSz,\n        k,\n        class_level_hard_mining=True,\n        sigmoid=True\n    ):\n        super(IncrementalClassRectificationLoss, self).__init__()\n\n        self.margin = margin\n        self.alpha = alpha\n        self.batchSz = batchSz\n        self.k = k\n        self.class_level_hard_mining = class_level_hard_mining\n        self.sigmoid = sigmoid\n        self.trip_loss = TripletLoss(margin)\n        self.bce = nn.BCEWithLogitsLoss()\n\n    def forward(self, input, target, X):\n        bce = self.bce(input, target)\n        idxs = get_minority_classes(target, batchSz=self.batchSz)\n        if self.sigmoid:\n            input = torch.sigmoid(input)\n            y_min = target[:, idxs]\n            preds_min = input[:, idxs]\n        else:\n            y_min = target[:, idxs]\n            preds_min = input[:, idxs]\n\n        y_mask = y_min == 1\n        P = torch.nonzero(y_mask)\n        N = torch.nonzero(~y_mask)\n        preds_P = preds_min[y_mask]\n\n        k = self.k\n        idx_tensors = []\n        pred_tensors = []\n        # would like to vectorize this\n        for idx, row in enumerate(P):\n            anchor_idx, anchor_class = row\n            mask = (P[:, 1] == anchor_class)\n            mask[idx] = 0\n            pos_idxs = P[mask]\n            pos_preds, sorted_= preds_min[pos_idxs[:, 0], pos_idxs[:, 1]].sort()\n            pos_idxs = pos_idxs[sorted_][:k]\n            pos_preds = pos_preds[:k]\n\n            mask = (N[:, 1] == anchor_class)\n            neg_idxs = N[mask]\n            neg_preds, sorted_= preds_min[neg_idxs[:, 0], neg_idxs[:, 1]].sort()\n            neg_idxs = neg_idxs[sorted_][-k:]\n            neg_preds = neg_preds[:k]\n\n            a = [idx] # anchor index in P\n            n_p = pos_idxs.shape[0]\n            n_n = neg_idxs.shape[0]\n            # create 2d array with indices for anchor, pos and neg examples\n            grid = torch.stack(torch.meshgrid([torch.Tensor(a).long(), torch.arange(n_p), torch.arange(n_n)])).reshape(3, -1).t()\n            idx_tensors.append(torch.cat([P[grid[:, 0]], pos_idxs[grid[:, 1]], neg_idxs[grid[:, 2]]], 1))\n            pred_tensors.append(torch.stack([preds_P[grid[:, 0]], pos_preds[grid[:, 1]], neg_preds[grid[:, 2]]], 1))\n\n        try:\n            if self.class_level_hard_mining:\n                idx_tensors = torch.cat(idx_tensors, 0)\n                pred_tensors = torch.cat(pred_tensors, 0)\n            else:\n                # TODO: implement instance level hard mining\n                pass\n            crl = self.trip_loss(pred_tensors[:, 0], pred_tensors[:, 1], pred_tensors[:, 2])\n            loss = self.alpha * crl + (1 - self.alpha) * bce\n\n            return loss\n\n        except RuntimeError:\n            # TODO: figure out why we are sometimes getting RuntimeError in test\n            logging.warning('RuntimeError in loss statement')\n\n            return bce","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e949d4f141a99b165e627280f1ef5e6712d7093b","trusted":true},"cell_type":"code","source":"omega = counts.min() / counts.max() # class imbalance measure as \neta = 0.01\nalpha = omega * eta\nprint(alpha)\n# don't use sigmoid layer since preds are already in 0-1 range\ncriterion = IncrementalClassRectificationLoss(0.5, alpha, 28, 3, sigmoid=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"08577a0caa61f030e6b2b77514aacf82f87f4300"},"cell_type":"markdown","source":"And finally, the loss calculated on the mini-batch used as an example is..."},{"metadata":{"_uuid":"cd3b71c246f1f061a21105c6ec8b49b3ef188935","trusted":true},"cell_type":"code","source":"criterion(preds, labels, inputs)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7b5fdd493fdd0c5fa987b4b54e3921c473834844"},"cell_type":"markdown","source":"### Build Datasets/Loaders"},{"metadata":{"_uuid":"f1dc9ddddad95207740012015cb885fd4bd13f85","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class ArgContainer():\n    def __init__(self, \n                 network_name,\n                 crit,\n                 batchSz, \n                 train_images_path,\n                 test_images_path,\n                 train_csv_path, \n                 nSubsample, \n                 pretrained,\n                 cuda,\n                 sigmoid,\n                 thresholds,\n    ):\n        self.network_name = network_name\n        self.crit = crit\n        self.batchSz = batchSz\n        self.train_images_path = train_images_path\n        self.test_images_path = test_images_path\n        self.train_csv_path = train_csv_path\n        self.nSubsample = nSubsample\n        self.pretrained = pretrained\n        self.cuda = cuda\n        self.sigmoid = sigmoid\n        self.thresholds = thresholds","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"8430eb3da2cb0d325d54ae04cac005dea78d03e2","trusted":true},"cell_type":"code","source":"def get_dataset(args, idxs=None, train=True):\n    if args.pretrained:\n        using_pil = True\n    else:\n        using_pil = False\n\n    transform = get_transforms(args.pretrained)\n    if train:\n        image_dir = args.train_images_path\n        label_file = args.train_csv_path\n        if label_file is None:\n            raise ValueError('no label_file provided for training')\n        if idxs is None:\n            raise ValueError('must specify idxs for training')\n        dataset = TrainImageDataset(\n                         image_dir=image_dir,\n                         label_file=label_file,\n                         transform=transform['TRAIN'],\n                         idxs=idxs,\n                         using_pil=using_pil)\n    else:\n        image_dir = args.test_images_path\n        dataset = TestImageDataset(\n                         image_dir=image_dir,\n                         transform=transform['DEV'],\n                         using_pil=using_pil)\n\n    return dataset\n\ndef get_train_test_split(args, val_split=0.10, distributed=False, **kwargs):\n    n_subsample = args.nSubsample\n\n    with open(args.train_csv_path, 'r') as f:\n        n_images = sum(1 for row in f.readlines()) - 1 # -1 for header row\n    if n_subsample != 0:\n        arr = np.random.choice(n_images, n_subsample, replace=False)\n        train_idxs = arr[:int(n_subsample * (1 - val_split))]\n        dev_idxs = arr[int(n_subsample * (1 - val_split)):]\n    else:\n        arr = np.random.choice(n_images, n_images, replace=False)\n        train_idxs = arr[:int(n_images * (1 - val_split))]\n        dev_idxs = arr[int(n_images * (1 - val_split)):]\n\n    trainset = get_dataset(args, idxs=train_idxs)\n    devset = get_dataset(args, idxs=dev_idxs)\n\n    if distributed:\n        trainLoader, devLoader, args.batchSz = partition_dataset(trainset, devset, args.batchSz)\n    else:\n        trainLoader = DataLoader(trainset, shuffle=True, **kwargs)\n        devLoader = DataLoader(devset, shuffle=False, **kwargs)\n\n    return trainLoader, devLoader\n\ndef get_loss_function(lf='bce', args=None):\n    if lf == 'bce':\n        return BCEWithLogitsLoss()\n\n    elif lf == 'f1':\n        return f1_loss\n\n    elif lf == 'crl':\n        if args:\n            return IncrementalClassRectificationLoss(*args)\n        raise ValueError('args for CRL not found')\n    else:\n        raise ModuleNotFoundError('loss function not found')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"efd31cb8574238182721bdeb2f65d5839b36faa3","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import torchvision.models as models\nimport torch.nn as nn\n\nfrom torch import cat\n\n\nRESNET_ENCODERS = {\n    34: models.resnet34,\n    50: models.resnet50,\n    101: models.resnet101,\n    152: models.resnet152,\n}\n\nVGG_CLASSIFIERS = {\n    11: models.vgg11,\n    13: models.vgg13,\n    16: models.vgg16,\n    19: models.vgg19,\n}\n\nVGG_BN_CLASSIFIERS = {\n    11: models.vgg11_bn,\n    13: models.vgg13_bn,\n    16: models.vgg16_bn,\n    19: models.vgg19_bn,\n}\n\n\nclass Net(nn.Module):\n    def __init__(self):\n        super(Net, self).__init__()\n        self.conv1 = nn.Conv2d(4, 6, 5) # 4 channel in, 6 channels out, filter size 5\n        self.pool = nn.MaxPool2d(2, 2) # 6 channel in, 6 channels out, filter size 2, stride 2\n        self.conv2 = nn.Conv2d(6, 16, 5) # 6 channel in, 16 channels out, filter size 5\n        self.fc1 = nn.Linear(16 * 125 * 125, 120)\n        self.fc2 = nn.Linear(120, 84)\n        self.fc3 = nn.Linear(84, 28)\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.pool(torch.relu(x))\n        x = self.conv2(x)\n        x = self.pool(torch.relu(x))\n        x = x.view(-1, 16 * 125 * 125)\n        x = torch.relu(self.fc1(x))\n        x = torch.relu(self.fc2(x))\n\n        return x\n\n\nclass Resnet4Channel(nn.Module):\n    def __init__(self, encoder_depth=34, pretrained=True, num_classes=28):\n        super().__init__()\n\n        encoder = RESNET_ENCODERS[encoder_depth](pretrained=pretrained)\n\n        if pretrained:\n            for param in encoder.parameters():\n                param.requires_grad=False\n\n        # we initialize this conv to take in 4 channels instead of 3\n        # we keeping corresponding weights and initializing new weights with zeros\n        # this trick taken from https://www.kaggle.com/iafoss/pretrained-resnet34-with-rgby-0-460-public-lb\n        w = encoder.conv1.weight\n        self.conv1 = nn.Conv2d(4, 64, kernel_size=7, stride=2, padding=3,\n                               bias=False)\n        self.conv1.weight = nn.Parameter(cat((w,w[:,:1,:,:]),dim=1))\n\n        self.bn1 = encoder.bn1\n        self.relu = nn.ReLU(inplace=True)\n        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n\n        self.layer1 = encoder.layer1\n        self.layer2 = encoder.layer2\n        self.layer3 = encoder.layer3\n        self.layer4 = encoder.layer4\n\n        self.avgpool = encoder.avgpool\n        num_features = encoder.fc.in_features\n        self.fc = nn.Linear(num_features, 28)\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.maxpool(x)\n\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n\n        x = self.avgpool(x)\n        x = x.view(x.size(0), -1)\n        x = self.fc(x)\n\n        return x\n\n\nclass VGG4Channel(nn.Module):\n    def __init__(self, n_layers=11, batch_norm=False, pretrained=True, num_classes=28):\n        super().__init__()\n\n        if batch_norm:\n            vgg_net = VGG_BN_CLASSIFIERS[n_layers](pretrained=pretrained)\n        else:\n            vgg_net = VGG_CLASSIFIERS[n_layers](pretrained=pretrained)\n\n        if pretrained:\n            for param in vgg_net.features.parameters():\n                param.requires_grad=False\n            for param in vgg_net.classifier.parameters():\n                param.requires_grad=False\n\n        # initialize conv2d to take in 4 channels instead of 3\n        feature_layers = []\n        w = vgg_net.features[0].weight\n        conv2d = nn.Conv2d(4, 64, kernel_size=3, padding=1) # Create 2d conv layer\n        conv2d.weight = nn.Parameter(cat((w,w[:,:1,:,:]),dim=1))\n        feature_layers.append(conv2d)\n\n        remaining_features = list(vgg_net.features.children())[1:] # Remove first layer\n        feature_layers.extend(remaining_features)\n\n        # swap last layer for fc layer with 28 outputs\n        num_features = vgg_net.classifier[-1].in_features\n        classifier_layers = list(vgg_net.classifier.children())[:-1] # Remove last layer\n        classifier_layers.extend([nn.Linear(num_features, 28)]) # Add layer with 28 outputs. activation in loss function\n\n        self.features = nn.Sequential(*feature_layers)\n        self.classifier = nn.Sequential(*classifier_layers)\n\n    def forward(self, x):\n        x = self.features(x)\n        x = x.view(x.size(0), -1)\n        x = self.classifier(x)\n        return x\n\n\ndef resnet34(pretrained):\n    net = Resnet4Channel(encoder_depth=34, pretrained=pretrained)\n    return net\n\ndef resnet50(pretrained):\n    net = Resnet4Channel(encoder_depth=50, pretrained=pretrained)\n    return net\n\ndef resnet101(pretrained):\n    net =  Resnet4Channel(encoder_depth=101, pretrained=pretrained)\n    return net\n\ndef resnet152(pretrained):\n    net = Resnet4Channel(encoder_depth=152, pretrained=pretrained)\n    return net\n\ndef vgg11(pretrained):\n    net = VGG4Channel(n_layers=11, batch_norm=False, pretrained=pretrained)\n    return net\n\ndef vgg13(pretrained):\n    net = VGG4Channel(n_layers=13, batch_norm=False, pretrained=pretrained)\n    return net\n\ndef vgg16(pretrained):\n    net = VGG4Channel(n_layers=16, batch_norm=False, pretrained=pretrained)\n    return net\n\ndef vgg19(pretrained):\n    net = VGG4Channel(n_layers=19, batch_norm=False, pretrained=pretrained)\n    return net\n\ndef vgg11_bn(pretrained):\n    net = VGG4Channel(n_layers=11, batch_norm=True, pretrained=pretrained)\n    return net\n\ndef vgg13_bn(pretrained):\n    net = VGG4Channel(n_layers=13, batch_norm=True, pretrained=pretrained)\n    return net\n\ndef vgg16_bn(pretrained):\n    net = VGG4Channel(n_layers=16, batch_norm=True, pretrained=pretrained)\n    return net\n\ndef vgg19_bn(pretrained):\n    net = VGG4Channel(n_layers=19, batch_norm=True, pretrained=pretrained)\n    return net\n\ndef baseline(pretrained):\n    if pretrained:\n        print('Baseline net not pretrained. Training from scratch')\n    return Net()\n\nNETWORKS_DICT = {\n    'resnet34' : resnet34,\n    'resnet50' : resnet50,\n    'resnet101' : resnet101,\n    'resnet152' : resnet152,\n    'vgg11' : vgg11,\n    'vgg13' : vgg13,\n    'vgg16' : vgg16,\n    'vgg19' : vgg19,\n    'vgg11_bn' : vgg11_bn,\n    'vgg13_bn' : vgg13_bn,\n    'vgg16_bn' : vgg16_bn,\n    'vgg19_bn' : vgg19_bn,\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"beb013b15327964eaf41f9cac43a060d259e577c"},"cell_type":"code","source":"args = ArgContainer(\"resnet152\", \"crl\", 64, train_images_path, test_images_path, train_csv_path, 0, True, True if GPU else False, True, None)\n\nkwargs = {'batch_size': args.batchSz}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5eb1a85010cc44eda042b61b088c55537bd1b44c"},"cell_type":"code","source":"net = NETWORKS_DICT[args.network_name](args.pretrained)\nnet.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c9b6362ae52e2ec8bd3647033291affbf414611c"},"cell_type":"code","source":"trainLoader, devLoader = get_train_test_split(args, **kwargs)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1919e965e02e9327a8c2055a712e14142e7a29d2","trusted":true},"cell_type":"code","source":"net = torch.nn.DataParallel(net) if LOCAL else torch.nn.DataParallel(net).cuda()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"025db5d82b0a7121fc38ebaadc71feddde3cc00b","trusted":true},"cell_type":"code","source":"optimizer = optim.SGD(net.parameters(), lr=0.01, momentum=0.9, weight_decay=1e-4)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7bd8db7b9ea77270eeb5b47ef8d9667f7bf70827","trusted":true},"cell_type":"code","source":"lf_args = [0.5, 8.537058595265812e-06, args.batchSz, 5, True, True]\ncriterion = get_loss_function('crl', lf_args)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"544643fb7d650aaa8a7c3e829776fb5c31e55d35","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def plot_lr(optimizer, net, trainLoader, criterion, start_lr=-7, end_lr=-1, num_iter=100):\n    xs = np.logspace(start_lr, end_lr, num_iter)\n    ys = []\n    for i, data in enumerate(trainLoader, 0):\n        if i == 100:\n            break\n        for param_group in optimizer.param_groups:\n            param_group['lr'] = xs[i]\n        net.train()\n        # get the inputs\n        inputs, labels = data['image'].cuda(), data['labels'].cuda()\n\n        # zero the parameter gradients\n        optimizer.zero_grad()\n\n        # forward + backward + optimize\n        outputs = net(inputs)\n        loss_inputs = (outputs, labels, inputs)\n        loss = criterion(*loss_inputs)\n        loss.backward()\n        optimizer.step()\n        ys.append(loss.item())\n    plt.figure(figsize=(10, 4), dpi=100).set_facecolor('white')\n    plt.plot(xs, ys, 'b-')\n    plt.xscale('log')\n    plt.title('Learning rate curve')\n    plt.ylabel('loss')\n    plt.xlabel('learning rate')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f974ffbf6ff845c7bbccebdaee0543c6d60acc99"},"cell_type":"code","source":"# if not LOCAL and GPU:\n#     plot_lr(optimizer, net, trainLoader, criterion)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"9839e34dbd54f59f9c97144d7f6efb9d331467aa"},"cell_type":"code","source":"def positive_predictions(predictions):\n    positives = []\n\n    for prediction in predictions:\n        output = []\n        i = 0\n        for label in prediction:\n            if(label == 1):\n                output.append(str(i))\n            i += 1\n        positives.append(' '.join(output))\n\n    return positives\n\ndef predict(args, net, dataLoader, predF):\n    net.eval()\n\n    with torch.no_grad():\n        predF.write('Id,Predicted\\n')\n        print('writing predictions...')\n        for batch_idx, data in enumerate(dataLoader):\n            inputs, image_ids = data['image'], data['image_id']\n            if args.cuda:\n                inputs = inputs.cuda()\n\n            outputs = net(inputs)\n            if args.sigmoid:\n                outputs = torch.sigmoid(outputs)\n            if args.thresholds is not None:\n                thresholds = [float(val) for val in\n                                            args.thresholds.split(\",\")]\n\n                thresholds = torch.tensor(thresholds)\n                if args.cuda:\n                    thresholds = thresholds.cuda()\n                pred = outputs.data.gt(thresholds)\n            else:\n                pred = outputs.data.gt(0.5)\n            preds = positive_predictions(pred)\n            for _ in zip(image_ids, preds):\n                predF.write(\",\".join(_) + '\\n')\n                predF.flush()\n\ndef get_testloader(args, **kwargs):\n    testset = get_dataset(args, train=False)\n    testloader = DataLoader(testset, shuffle=False, **kwargs)\n\n    return testloader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"661fcd8609a792e329e2159c073c1b2db8d79d8f"},"cell_type":"code","source":"testLoader = get_testloader(args, **kwargs)\nif GPU:\n    net.module.load_state_dict(torch.load(weights_path))\nelse:\n    net.module.load_state_dict(torch.load(weights_path), map_location='cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7413d46d51c409947a9d7b85e5a49758cc7ebdc4"},"cell_type":"markdown","source":"## Inference"},{"metadata":{"trusted":true,"_uuid":"868c66ada5a74b5965a4729ec33d42108f914c7f"},"cell_type":"code","source":"args.thresholds = \".3\"\nsubmission_path = 'working/submission.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13de80b1a58639aaa131fc33fb4ddedbbd810d78"},"cell_type":"code","source":"predF = open(submission_path, 'a')\n\npredict(args, net, testLoader, predF)\n\npredF.close","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"680a6e015c9c2a02e076037c41a21eea2c7c3c4d","trusted":true},"cell_type":"code","source":"sample_df = pd.read_csv(sample_submission_path, encoding='utf-8')\noutput_df = pd.read_csv(submission_path, encoding='utf-8')\noutput_df = output_df.replace(np.nan, '', regex=True)\nnew_df = sample_df.merge(output_df, left_on='Id', right_on='Id', how='outer')\nnew_df = new_df.loc[:, ['Id', 'Predicted_y']]\nnew_df.columns = ['Id','Predicted']\nnew_df.to_csv(submission_path, index=False, encoding='utf-8')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a31e8162a296d0569118b0a6b99275ad784b4dc5"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}