{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.nn as nn\nfrom torch.optim import Adam\nimport random\nfrom pathlib import Path\nfrom torchvision.transforms import ToTensor\nfrom tqdm import tqdm_notebook as tqdm\nimport matplotlib.pyplot as plt\nfrom collections import OrderedDict\nimport torchvision.datasets as datasets\nfrom torchvision import models\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"random.seed(42)\ntorch.manual_seed(42)\nnp.random.seed(42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pixel_stats = pd.read_csv('../input/recursion-cellular-image-classification/pixel_stats.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pixel_stats.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_img_path(suffix, experiment, plate, well, site=1):\n    return suffix + experiment.lower() + '/' + 'plate' + str(plate) + '/' + well + '_s{}.npy'.format(site)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# tr_df = pd.read_csv('../input/recursion-cellular-image-classification/train_controls.csv')\ntr_df = pd.read_csv('../input/recursion-cellular-image-classification/train.csv')\ntr_df = tr_df.append(pd.read_csv('../input/recursion-cellular-image-classification/train_controls.csv'), ignore_index=True, sort=False)\ntr_df['site'] = 1\ntr_df_copy = tr_df.copy()\ntr_df_copy['site'] = 2\ntr_df = tr_df.append(tr_df_copy, ignore_index=True)\ntr_df['cellline'] = tr_df.apply(lambda row : row.experiment[:-3], axis=1)\ntr_df['exp_plate'] = tr_df.apply(lambda row: row.experiment + '_' + str(row.plate), axis=1)\ntr_df['path'] = tr_df.apply(lambda row : get_img_path('recursion-npy-train-', row.experiment, row.plate, row.well, row.site), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"te_controls = pd.read_csv('../input/recursion-cellular-image-classification/test_controls.csv')\nte_controls['site'] = 1\nte_controls_copy = te_controls.copy()\nte_controls_copy['site'] = 2\nte_controls = te_controls.append(te_controls_copy, ignore_index=True)\nte_controls['cellline'] = te_controls.apply(lambda row : row.experiment[:-3], axis=1)\nte_controls['exp_plate'] = te_controls.apply(lambda row: row.experiment + '_' + str(row.plate), axis=1)\nte_controls['path'] = te_controls.apply(lambda row : get_img_path('recursion-npy-test-', row.experiment, row.plate, row.well, row.site), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_df = tr_df.append(te_controls, ignore_index=True, sort=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"experiments = list(tr_df['experiment'].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(experiments)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# va_exps = ['HUVEC-01', 'HEPG2-01', 'RPE-01', 'U2OS-01']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# tr_df['is_valid'] = tr_df.experiment.str.contains('|'.join(va_exps))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# va_df = tr_df[tr_df['is_valid']]\n# tr_df = tr_df[~tr_df['is_valid']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_df, va_df = train_test_split(tr_df, random_state=42, stratify=tr_df.sirna)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(va_df), len(tr_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# te_df = pd.read_csv('../input/recursion-cellular-image-classification/test_controls.csv')\n# # te_df = pd.read_csv('../input/recursion-cellular-image-classification/test.csv')\n# # te_df = te_df.append(pd.read_csv('../input/recursion-cellular-image-classification/test_controls.csv'), ignore_index=True, sort=False)\n# te_df['site'] = 1\n# te_df['cellline'] = te_df.apply(lambda row : row.experiment[:-3], axis=1)\n# te_df = te_df[te_df['cellline'] == 'HUVEC']\n# te_df['path'] = te_df.apply(lambda row : get_img_path('recursion-npy-test-', row.experiment, row.plate, row.well, row.site), axis=1)\n# te_df1 = te_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# te_df = pd.read_csv('../input/recursion-cellular-image-classification/test_controls.csv')\n# # te_df = pd.read_csv('../input/recursion-cellular-image-classification/test.csv')\n# # te_df = te_df.append(pd.read_csv('../input/recursion-cellular-image-classification/test_controls.csv'), ignore_index=True, sort=False)\n# te_df['site'] = 2\n# te_df['cellline'] = te_df.apply(lambda row : row.experiment[:-3], axis=1)\n# te_df = te_df[te_df['cellline'] == 'HUVEC']\n# te_df['path'] = te_df.apply(lambda row : get_img_path('recursion-npy-test-', row.experiment, row.plate, row.well, row.site), axis=1)\n# te_df2 = te_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"exp_plate_to_index = dict((ep,i) for i,ep in enumerate(tr_df['exp_plate'].unique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"celllines_to_index = {'HUVEC': 0, 'HEPG2': 1, 'RPE': 2, 'U2OS': 3}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"well_to_index = dict((ep,i) for i,ep in enumerate(tr_df['well'].unique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RecursionDataset(Dataset):\n    def __init__(self, path, df, augment=False, img_size=512):\n        self.df = df\n        self.len = len(df)\n        self.path = path\n        if 'sirna' in df.columns:\n            self.labels = list(df['sirna'])\n        else:\n            self.labels = list(df['id_code'])\n        self.plates = list(map(lambda ep: exp_plate_to_index[ep], list(df['exp_plate'])))\n        self.celllines = list(map(lambda c: celllines_to_index[c], list(df['cellline'])))\n        self.wells = list(map(lambda c: well_to_index[c], list(df['well'])))\n        self.img_paths = list(df['path'])\n        self.totensor = ToTensor()\n        self.augment = augment\n        self.img_size = img_size\n        \n    def __getitem__(self, index):\n        label = self.labels[index]\n        path = self.path/self.img_paths[index]\n        img = np.load(path)\n#         img = img / 255.0\n        img = img.astype(np.float32)\n        if self.augment:\n            # random crop a img_size by img_size patch\n            row_start = random.randint(0, img.shape[0] - self.img_size)\n            col_start = random.randint(0, img.shape[1] - self.img_size)\n        else:\n            row_start = int(0.5 * (img.shape[0] - self.img_size))\n            col_start = int(0.5 * (img.shape[1] - self.img_size))\n        img = img[row_start:row_start+self.img_size,col_start:col_start+self.img_size,:]\n        if self.augment:\n            # flip vert with p=0.5\n            if random.random() < 0.5:\n                img = img[:,::-1,:]\n            # flip hori with p=0.5\n            if random.random() < 0.5:\n                img = img[::-1,:,:]\n            img = img.copy()\n        img = self.totensor(img)\n        return img, label, self.plates[index], self.celllines[index], self.wells[index]\n    \n    def __len__(self):\n        return self.len","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bs = 512\nnw = 4\nimg_size = 128","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_ds = RecursionDataset(Path('../input'), tr_df, augment=True, img_size=img_size)\nva_ds = RecursionDataset(Path('../input'), va_df, img_size=img_size)\n# te_ds1 = RecursionDataset(Path('../input'), te_df1)\n# te_ds2 = RecursionDataset(Path('../input'), te_df2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_dl = DataLoader(tr_ds, batch_size=bs, num_workers=nw, pin_memory=True, shuffle=True, drop_last=False)\nva_dl = DataLoader(va_ds, batch_size=bs, num_workers=2, pin_memory=True, shuffle=False)\n# te_dl1 = DataLoader(te_ds1, batch_size=bs, num_workers=nw, pin_memory=True, shuffle=False)\n# te_dl2 = DataLoader(te_ds2, batch_size=bs, num_workers=nw, pin_memory=True, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(tr_ds), len(va_ds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(tr_df.sirna.unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(tr_df.exp_plate.unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_sirnas = len(tr_df.sirna.unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_plates = len(tr_df.exp_plate.unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_celllines = len(tr_df.cellline.unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_wells = len(tr_df.well.unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Flatten(nn.Module):\n    def forward(self, x):\n        return x.view(x.size()[0], -1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Linear classifier\n# model = nn.Sequential(\n#     nn.AvgPool2d(kernel_size=4),\n#     Flatten(),\n#     nn.Linear(in_features=128 * 128 * 6,\n#              out_features=num_classes)\n# ).cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Smaller version of Lenet\n# model = nn.Sequential(\n#     nn.AvgPool2d(kernel_size=16),\n#     nn.Conv2d(in_channels=6,\n#              out_channels=12,\n#              kernel_size=5),\n#     nn.ReLU(),\n#     nn.MaxPool2d(kernel_size=2),\n#     nn.Conv2d(in_channels=12,\n#              out_channels=24,\n#              kernel_size=5),\n#     nn.ReLU(),\n#     nn.MaxPool2d(kernel_size=2),\n#     Flatten(),\n#     nn.Linear(in_features=24 * 5 * 5,\n#              out_features=num_classes)\n# ).cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Lenet-5\n# # From https://engmrk.com/lenet-5-a-classic-cnn-architecture/\n# model = nn.Sequential(\n# #     nn.AvgPool2d(kernel_size=16),\n#     nn.Conv2d(in_channels=6,\n#              out_channels=6,\n#              kernel_size=5),\n#     nn.ReLU(),\n#     nn.AvgPool2d(kernel_size=2),\n#     nn.Conv2d(in_channels=6,\n#              out_channels=16,\n#              kernel_size=5),\n#     nn.ReLU(),\n# #     nn.AvgPool2d(kernel_size=2),\n#     Flatten(),\n#     nn.Linear(in_features=1600,\n#              out_features=120),\n#     nn.ReLU(),\n#     nn.Linear(in_features=120,\n#              out_features=84),\n#     nn.ReLU(),\n#     nn.Linear(in_features=84,\n#              out_features=num_classes)\n# ).cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # vgg like for 512x512\n# # From https://neurohive.io/en/popular-networks/vgg16/\n# model = nn.Sequential(\n# #     nn.AvgPool2d(kernel_size=2),\n#     nn.Sequential(\n#         nn.Conv2d(in_channels=6,\n#                  out_channels=16,\n#                  kernel_size=3,\n#                  padding=1),\n#         nn.ReLU(),\n#         nn.Conv2d(in_channels=16,\n#                  out_channels=16,\n#                  kernel_size=3,\n#                  padding=1),\n#         nn.ReLU(),\n#     ),\n#     nn.MaxPool2d(kernel_size=2),\n#     nn.Sequential(\n#         nn.Conv2d(in_channels=16,\n#                  out_channels=32,\n#                  kernel_size=3,\n#                  padding=1),\n#         nn.ReLU(),\n#         nn.Conv2d(in_channels=32,\n#                  out_channels=32,\n#                  kernel_size=3,\n#                  padding=1),\n#         nn.ReLU(),\n#     ),\n#     nn.MaxPool2d(kernel_size=2),\n#     nn.Sequential(\n#         nn.Conv2d(in_channels=32,\n#                  out_channels=64,\n#                  kernel_size=3,\n#                  padding=1),\n#         nn.ReLU(),\n#         nn.Conv2d(in_channels=64,\n#                  out_channels=64,\n#                  kernel_size=3,\n#                  padding=1),\n#         nn.ReLU(),\n#     ),\n#     nn.MaxPool2d(kernel_size=2),\n#     nn.Sequential(\n#         nn.Conv2d(in_channels=64,\n#                  out_channels=128,\n#                  kernel_size=3,\n#                  padding=1),\n#         nn.ReLU(),\n#         nn.Conv2d(in_channels=128,\n#                  out_channels=128,\n#                  kernel_size=3,\n#                  padding=1),\n#         nn.ReLU(),\n#     ),\n#     nn.MaxPool2d(kernel_size=2),\n#     nn.Sequential(\n#         nn.Conv2d(in_channels=128,\n#                  out_channels=256,\n#                  kernel_size=3,\n#                  padding=1),\n#         nn.ReLU(),\n#         nn.Conv2d(in_channels=256,\n#                  out_channels=256,\n#                  kernel_size=3,\n#                  padding=1),\n#         nn.ReLU(),\n#     ),\n#     nn.MaxPool2d(kernel_size=2),\n#     nn.Sequential(\n#         nn.Conv2d(in_channels=256,\n#                  out_channels=512,\n#                  kernel_size=3,\n#                  padding=1),\n#         nn.ReLU(),\n#         nn.Conv2d(in_channels=512,\n#                  out_channels=512,\n#                  kernel_size=3,\n#                  padding=1),\n#         nn.ReLU(),\n#     ),\n#     nn.MaxPool2d(kernel_size=2),\n#     Flatten(),\n#     nn.Sequential(\n#         nn.Linear(in_features=512 * 8 * 8,\n#                  out_features=num_classes),\n#     )\n# ).cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = models.vgg16_bn(num_classes=num_classes)\n# model.features[0] = nn.Conv2d(6, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n# model = model.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = models.vgg16_bn(pretrained=True)\n# w = model.features[0].weight.mean(dim=1, keepdim=True)\n# b = model.features[0].bias\n# model.features[0] = nn.Conv2d(6, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n# with torch.no_grad():\n#     model.features[0].weight *= 0.0\n#     model.features[0].weight += w\n#     model.features[0].bias = b\n# for param in model.features:\n#     param.requires_grad = False\n# model.classifier[6] = nn.Linear(in_features=4096, out_features=num_classes)\n# model.features[0].requires_grad = True\n# model.classifier.requires_grad = True\n# model = model.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# features = models.densenet201(pretrained=True).features\n# w = features.conv0.weight.mean(dim=1, keepdim=True)\n# features.conv0 = nn.Conv2d(6, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n# with torch.no_grad():\n#     features.conv0.weight *= 0.0\n#     features.conv0.weight += w\n# features = features.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# classifier = nn.Sequential(\n#     nn.AdaptiveAvgPool2d(output_size=1),\n#     nn.Flatten(),\n#     nn.BatchNorm1d(1920, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n#     nn.Dropout(p=0.5, inplace=False),\n#     nn.Linear(in_features=1920, out_features=512),\n#     nn.ReLU(inplace=True),\n#     nn.BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n#     nn.Dropout(p=0.5, inplace=False),\n#     nn.Linear(in_features=512, out_features=num_classes),\n# )\n# classifier = classifier.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# domain_classifier = nn.Sequential(\n#     nn.AdaptiveAvgPool2d(output_size=1),\n#     nn.Flatten(),\n#     nn.BatchNorm1d(1920, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n#     nn.Dropout(p=0.5, inplace=False),\n#     nn.Linear(in_features=1920, out_features=512),\n#     nn.ReLU(inplace=True),\n#     nn.BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n#     nn.Dropout(p=0.5, inplace=False),\n#     nn.Linear(in_features=512, out_features=num_domains),\n# )\n# domain_classifier = domain_classifier.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = models.resnet18(pretrained=True)\nw = features.conv1.weight.mean(dim=1, keepdim=True)\nfeatures.conv1 = nn.Conv2d(6, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\nwith torch.no_grad():\n    features.conv1.weight *= 0.0\n    features.conv1.weight += w\nfeatures.fc = nn.Identity()\nfeatures = features.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_classifier(num_outputs, num_inputs):\n    classifier = nn.Sequential(\n        nn.Flatten(),\n        nn.BatchNorm1d(num_inputs, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n        nn.Dropout(p=0.5, inplace=False),\n        nn.Linear(in_features=num_inputs, out_features=128),\n        nn.ReLU(inplace=True),\n        nn.BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n        nn.Dropout(p=0.5, inplace=False),\n        nn.Linear(in_features=128, out_features=num_outputs),\n    )\n    classifier = classifier.cuda()\n    return classifier","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_dim = 512","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sirna_classifier = make_classifier(num_sirnas, feature_dim)\nplate_classifier = make_classifier(num_plates, feature_dim)\ncellline_classifier = make_classifier(num_celllines, feature_dim)\nwell_classifier = make_classifier(num_wells, feature_dim)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = nn.ModuleDict({'features': features, 'sirna': sirna_classifier, 'plate': plate_classifier, 'cellline': cellline_classifier, 'well': well_classifier})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = models.densenet201(pretrained=True)\n# w = model.features.conv0.weight.mean(dim=1, keepdim=True)\n# model.features.conv0 = nn.Conv2d(6, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n# with torch.no_grad():\n#     model.features.conv0.weight *= 0.0\n#     model.features.conv0.weight += w\n# model.classifier = nn.Sequential(\n#     nn.BatchNorm1d(1920, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n#     nn.Dropout(p=0.5, inplace=False),\n#     nn.Linear(in_features=1920, out_features=512),\n#     nn.ReLU(inplace=True),\n#     nn.BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n#     nn.Dropout(p=0.5, inplace=False),\n#     nn.Linear(in_features=512, out_features=num_classes),\n# )\n# model = model.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# optimizer = Adam(list(filter(lambda p: p.requires_grad, model.parameters())))\noptimizer = Adam(model.parameters(), lr=3e-4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for name, param in model.named_parameters():\n# # for layer in model.features:\n#     if 'norm' in name:\n# #     if isinstance(layer, nn.BatchNorm2d):\n# #         for param in layer.parameters():\n#         param.requires_grad = True\n#     else:\n# #         for param in layer.parameters():\n#         param.requires_grad = False\n\n# for param in model.parameters():\n#     param.requires_grad = True\n    \n# # for layer in model.modules():\n# #     if isinstance(layer, nn.BatchNorm1d):\n# #         for param in layer.parameters():\n# #             param.requires_grad = False\n    \n# # for param in model.features.transition1.parameters():\n# #     param.requires_grad = True\n    \n# # for param in model.features.transition2.parameters():\n# #     param.requires_grad = True\n    \n# # for param in model.features.transition3.parameters():\n# #     param.requires_grad = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for p in model.layer1.parameters():\n#     p.requires_grad = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for name, param in model.named_parameters():\n#     print('{:<48}\\t{}\\t{}'.format(name, param.requires_grad, torch.numel(param)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"l_c = 10.0\nl_p = -1.0\nl_w = -1.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_one_epoch(model, optimizer, criterion, dl):\n    metrics = []\n    model.train()\n#     for inputs, sirnas, plates, celllines, wells in tqdm(dl):\n    for inputs, sirnas, plates, celllines, wells in dl:\n        inputs, sirnas, plates, celllines, wells = inputs.cuda(), sirnas.cuda(), plates.cuda(), celllines.cuda(), wells.cuda()\n        optimizer.zero_grad()\n        features = model['features'](inputs)\n#         print(features.shape)\n#         print(nn.Flatten()(features).shape)\n        sirna_pred = model['sirna'](features)\n        plate_pred = model['plate'](features)\n        cellline_pred = model['cellline'](features)\n        well_pred = model['well'](features)\n        sirna_loss = criterion(sirna_pred, sirnas)\n        plate_loss = criterion(plate_pred, plates)\n        cellline_loss = criterion(cellline_pred, celllines)\n        well_loss = criterion(well_pred, wells)\n        loss = sirna_loss + l_p * plate_loss + l_c * cellline_loss + l_w * well_loss\n        loss.backward()\n        for p in model['plate'].parameters():\n            p.grad.data.mul_(1.0 / l_p)\n        for p in model['well'].parameters():\n            p.grad.data.mul_(1.0 / l_w)\n        optimizer.step()\n        \n        with torch.no_grad():\n            model.eval()\n            sirna_class_preds = torch.argmax(sirna_pred, dim=1)\n            plate_class_preds = torch.argmax(plate_pred, dim=1)\n            cellline_class_preds = torch.argmax(cellline_pred, dim=1)\n            well_class_preds = torch.argmax(well_pred, dim=1)\n            sirna_accu = (sirna_class_preds == sirnas).sum().item() / float(inputs.shape[0])\n            plate_accu = (plate_class_preds == plates).sum().item() / float(inputs.shape[0])\n            cellline_accu = (cellline_class_preds == celllines).sum().item() / float(inputs.shape[0])\n            well_accu = (well_class_preds == wells).sum().item() / float(inputs.shape[0])\n            metrics.append((loss.item(), sirna_accu, plate_accu, cellline_accu, well_accu, sirna_loss.item(), plate_loss.item(), cellline_loss.item(), well_loss.item()))\n            model.train()\n    \n    return metrics","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def validate_model_on(model, dl):\n    with torch.no_grad():\n        model.eval()\n        sirna_loss = 0.0\n        plate_loss = 0.0\n        cellline_loss = 0.0\n        well_loss = 0.0\n        sirna_accu = 0.0\n        plate_accu = 0.0\n        cellline_accu = 0.0\n        well_accu = 0.0\n#         for inputs, sirnas, plates, celllines, wells in tqdm(dl):\n        for inputs, sirnas, plates, celllines, wells in dl:\n            inputs, sirnas, plates, celllines, wells = inputs.cuda(), sirnas.cuda(), plates.cuda(), celllines.cuda(), wells.cuda()\n            features = model['features'](inputs)\n        #         print(features.shape)\n        #         print(nn.Flatten()(features).shape)\n            sirna_pred = model['sirna'](features)\n            plate_pred = model['plate'](features)\n            cellline_pred = model['cellline'](features)\n            well_pred = model['well'](features)\n            sirna_loss += criterion(sirna_pred, sirnas).item()\n            plate_loss += criterion(plate_pred, plates).item()\n            cellline_loss += criterion(cellline_pred, celllines).item()\n            well_loss += criterion(well_pred, wells).item()\n            sirna_class_preds = torch.argmax(sirna_pred, dim=1)\n            plate_class_preds = torch.argmax(plate_pred, dim=1)\n            cellline_class_preds = torch.argmax(cellline_pred, dim=1)\n            well_class_preds = torch.argmax(well_pred, dim=1)\n            sirna_accu += (sirna_class_preds == sirnas).sum().item() / float(inputs.shape[0])\n            plate_accu += (plate_class_preds == plates).sum().item() / float(inputs.shape[0])\n            cellline_accu += (cellline_class_preds == celllines).sum().item() / float(inputs.shape[0])\n            well_accu += (well_class_preds == wells).sum().item() / float(inputs.shape[0])\n        sirna_loss /= len(dl)\n        plate_loss /= len(dl)\n        cellline_loss /= len(dl)\n        well_loss /= len(dl)\n        loss = sirna_loss + l_p * plate_loss + l_c * cellline_loss + l_w * well_loss\n        sirna_accu /= len(dl)\n        plate_accu /= len(dl)\n        cellline_accu /= len(dl)\n        well_accu /= len(dl)\n    return loss, sirna_accu, plate_accu, cellline_accu, well_accu, sirna_loss, plate_loss, cellline_loss, well_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_metrics = []\nva_metrics = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint = torch.load('../input/domain-adaptation/checkpoint.tar')\nmodel.load_state_dict(checkpoint['model'])\noptimizer.load_state_dict(checkpoint['optimizer'])\ntr_metrics, va_metrics = checkpoint['metric']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer.param_groups[0]['lr'] = 3e-4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_epochs = 50","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for epoch in tqdm(range(num_epochs)):\nfor epoch in range(num_epochs):\n    print('Epoch: {}'.format(epoch))\n    metrics = train_one_epoch(model, optimizer, criterion, tr_dl)\n    mean = lambda l: sum(l) / len(l)\n    mean_metric = tuple(map(lambda i: mean(list(map(lambda m:m[i], metrics))), range(len(metrics[0]))))\n    tr_metrics.extend(metrics)\n    print('Train Loss: {:.4f} Sirna Acc: {:.4f} Plate Acc: {:.4f} Cellline Acc: {:.4f} Well Acc: {:.4f} Sirna Loss: {:.4f} Plate Loss: {:.4f} Cellline Loss: {:.4f} Well Loss: {:.4f}'.format(*mean_metric))\n    va_metric = validate_model_on(model, va_dl)\n    print('Valid Loss: {:.4f} Sirna Acc: {:.4f} Plate Acc: {:.4f} Cellline Acc: {:.4f} Well Acc: {:.4f} Sirna Loss: {:.4f} Plate Loss: {:.4f} Cellline Loss: {:.4f} Well Loss: {:.4f}'.format(*va_metric))\n    va_metrics.append(va_metric)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"max([metric[1] for metric in va_metrics])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"min([metric[0] for metric in va_metrics])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!nvidia-smi","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot(i):\n    fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(16, 8))\n    axes[0].plot([metric[i] for metric in tr_metrics])\n    start, end = axes[0].get_ylim()\n    axes[0].yaxis.set_ticks(np.arange(start, end, (end-start)/30))\n    axes[1].plot([metric[i] for metric in va_metrics])\n    start, end = axes[1].get_ylim()\n    axes[1].yaxis.set_ticks(np.arange(start, end, (end-start)/30))\n    axes[1].xaxis.set_ticks(np.arange(0, len(va_metrics), 5))\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot(4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot(6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.save({'model': model.state_dict(),\n           'optimizer': optimizer.state_dict(),\n           'metric': (tr_metrics, va_metrics)}, '/kaggle/working/checkpoint.tar')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint = torch.load('/kaggle/working/checkpoint.tar')\nmodel.load_state_dict(checkpoint['model'])\noptimizer.load_state_dict(checkpoint['optimizer'])\ntr_metrics, va_metrics = checkpoint['metric']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"# submission = pd.read_csv('../input/best-190819/best_190819.csv')\n# id_code = list(submission['id_code'])\n# sirna = list(submission['sirna'])\n# submission_dict = dict(zip(id_code, sirna))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# with torch.no_grad():\n#     model.eval()\n#     for (img1, id_code1), (img2, id_code2) in zip(tqdm(te_ds1), te_ds2):\n#         assert(id_code1 == id_code2)\n#         id_code = id_code1\n#         imgs = torch.stack([img1, img2]).cuda()\n#         out = model.forward(imgs)\n#         pred = torch.argmax(torch.mean(out, dim=0))\n#         submission_dict[id_code] = int(pred)\n#     model.train()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# submission = pd.DataFrame.from_dict({'id_code' : list(submission_dict.keys()), 'sirna' : list(submission_dict.values())})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# submission.to_csv('submission.csv', index=False, columns=['id_code','sirna'])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}