{"cells":[{"metadata":{},"cell_type":"markdown","source":"**Simple example of transfer learning from pretrained model using PyTorch.**\n* Metrics: f1_score"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import os\n\nimport numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\n\nimport cv2\nimport torch\nfrom tqdm import tqdm_notebook\nfrom PIL import Image\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models\nfrom sklearn.model_selection import train_test_split\nfrom torchvision import transforms\n\n%matplotlib inline","execution_count":1,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def kaggle_commit_logger(str_to_log, need_print = True):\n if need_print:\n print(str_to_log)\n os.system('echo ' + str_to_log)","execution_count":24,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df_all = pd.read_csv('../input/train.csv')\ntrain_df_all.head()","execution_count":2,"outputs":[{"output_type":"execute_result","execution_count":2,"data":{"text/plain":" category_id date_captured ... width height\n0 19 2011-05-13 23:43:18 ... 1024 747\n1 19 2012-03-17 03:48:44 ... 1024 747\n2 0 2014-05-11 11:56:46 ... 1024 747\n3 0 2013-10-06 02:00:00 ... 1024 747\n4 0 2011-07-12 13:11:16 ... 1024 747\n\n[5 rows x 11 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
category_iddate_capturedfile_nameframe_numidlocationrights_holderseq_idseq_num_frameswidthheight
0192011-05-13 23:43:185998cfa4-23d2-11e8-a6a3-ec086b02610b.jpg15998cfa4-23d2-11e8-a6a3-ec086b02610b33Justin Brown6f084ccc-5567-11e8-bc84-dca9047ef27731024747
1192012-03-17 03:48:44588a679f-23d2-11e8-a6a3-ec086b02610b.jpg2588a679f-23d2-11e8-a6a3-ec086b02610b115Justin Brown6f12067d-5567-11e8-b3c0-dca9047ef27731024747
202014-05-11 11:56:4659279ce3-23d2-11e8-a6a3-ec086b02610b.jpg159279ce3-23d2-11e8-a6a3-ec086b02610b96Erin Boydston6faa92d1-5567-11e8-b1ae-dca9047ef27711024747
302013-10-06 02:00:005a2af4ab-23d2-11e8-a6a3-ec086b02610b.jpg15a2af4ab-23d2-11e8-a6a3-ec086b02610b57Erin Boydston6f7d4702-5567-11e8-9e03-dca9047ef27711024747
402011-07-12 13:11:16599fbd89-23d2-11e8-a6a3-ec086b02610b.jpg3599fbd89-23d2-11e8-a6a3-ec086b02610b46Justin Brown6f1728a1-5567-11e8-9be7-dca9047ef27731024747
\n
"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 64\nIMG_SIZE = 64\nN_EPOCHS = 10\nID_COLNAME = 'file_name'\nANSWER_COLNAME = 'category_id'\nTRAIN_IMGS_DIR = '../input/train_images/'\nTEST_IMGS_DIR = '../input/test_images/'","execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df, test_df = train_test_split(train_df_all[[ID_COLNAME, ANSWER_COLNAME]],\n test_size = 0.15, \n shuffle = True\n )","execution_count":15,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CLASSES_TO_USE = train_df_all['category_id'].unique()","execution_count":4,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CLASSES_TO_USE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_CLASSES = len(CLASSES_TO_USE)\nNUM_CLASSES","execution_count":5,"outputs":[{"output_type":"execute_result","execution_count":5,"data":{"text/plain":"14"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"CLASSMAP = dict(\n [(i, j) for i, j\n in zip(CLASSES_TO_USE, range(NUM_CLASSES))\n ]\n)\nCLASSMAP","execution_count":12,"outputs":[{"output_type":"execute_result","execution_count":12,"data":{"text/plain":"{19: 0,\n 0: 1,\n 3: 2,\n 8: 3,\n 4: 4,\n 13: 5,\n 1: 6,\n 11: 7,\n 16: 8,\n 17: 9,\n 14: 10,\n 18: 11,\n 10: 12,\n 22: 13}"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"REVERSE_CLASSMAP = dict([(v, k) for k, v in CLASSMAP.items()])\nREVERSE_CLASSMAP","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"model = models.densenet121(pretrained='imagenet')","execution_count":26,"outputs":[{"output_type":"stream","text":"Downloading: \"https://download.pytorch.org/models/densenet121-a639ec97.pth\" to /tmp/.torch/models/densenet121-a639ec97.pth\n\n0it [00:00, ?it/s]\u001b[A\n57344it [00:00, 462863.39it/s]\u001b[A\n303104it [00:00, 603474.72it/s]\u001b[A\n892928it [00:00, 817339.86it/s]\u001b[A\n1417216it [00:00, 1075345.38it/s]\u001b[A\n1794048it [00:00, 1356334.99it/s]\u001b[A\n2179072it [00:00, 1586170.85it/s]\u001b[A\n2457600it [00:01, 1327172.67it/s]\u001b[A\n3170304it [00:01, 1711555.47it/s]\u001b[A\n3571712it [00:01, 1980921.83it/s]\u001b[A\n3973120it [00:01, 2306239.27it/s]\u001b[A\n4390912it [00:01, 2487943.64it/s]\u001b[A\n4792320it [00:01, 2726875.70it/s]\u001b[A\n5226496it [00:01, 2865217.04it/s]\u001b[A\n5644288it [00:01, 3077139.04it/s]\u001b[A\n6062080it [00:02, 3168781.07it/s]\u001b[A\n6496256it [00:02, 3182526.69it/s]\u001b[A\n6930432it [00:02, 3325648.53it/s]\u001b[A\n7282688it [00:02, 3379772.19it/s]\u001b[A\n7634944it [00:02, 2981445.07it/s]\u001b[A\n8044544it [00:02, 3074573.27it/s]\u001b[A\n8495104it [00:02, 3197464.46it/s]\u001b[A\n8937472it [00:02, 3256469.05it/s]\u001b[A\n9388032it [00:03, 3361639.61it/s]\u001b[A\n9838592it [00:03, 3438303.70it/s]\u001b[A\n10272768it [00:03, 3446496.68it/s]\u001b[A\n10723328it [00:03, 3504671.01it/s]\u001b[A\n11182080it [00:03, 3547073.43it/s]\u001b[A\n11632640it [00:03, 3492330.33it/s]\u001b[A\n12083200it [00:03, 3621398.87it/s]\u001b[A\n12533760it [00:03, 3620465.52it/s]\u001b[A\n12992512it [00:04, 3648830.56it/s]\u001b[A\n13443072it [00:04, 3643234.86it/s]\u001b[A\n13893632it [00:04, 3621941.25it/s]\u001b[A\n14344192it [00:04, 3651372.28it/s]\u001b[A\n14802944it [00:04, 3675814.21it/s]\u001b[A\n15253504it [00:04, 3865540.60it/s]\u001b[A\n15704064it [00:04, 3931350.26it/s]\u001b[A\n16105472it [00:04, 3803825.45it/s]\u001b[A\n16490496it [00:04, 3602498.77it/s]\u001b[A\n16908288it [00:05, 3616350.06it/s]\u001b[A\n17375232it [00:05, 3645263.19it/s]\u001b[A\n17850368it [00:05, 3851183.23it/s]\u001b[A\n18243584it [00:05, 3652093.49it/s]\u001b[A\n18644992it [00:05, 3601267.49it/s]\u001b[A\n19079168it [00:05, 3624119.91it/s]\u001b[A\n19570688it [00:05, 3697205.35it/s]\u001b[A\n20054016it [00:05, 3827483.39it/s]\u001b[A\n20545536it [00:06, 3961805.77it/s]\u001b[A\n21028864it [00:06, 4157930.51it/s]\u001b[A\n21454848it [00:06, 3997457.94it/s]\u001b[A\n21864448it [00:06, 3803257.46it/s]\u001b[A\n22323200it [00:06, 3930274.29it/s]\u001b[A\n22806528it [00:06, 4073033.93it/s]\u001b[A\n23224320it [00:06, 4097802.67it/s]\u001b[A\n23642112it [00:06, 3916156.31it/s]\u001b[A\n24092672it [00:06, 4074242.14it/s]\u001b[A\n24616960it [00:07, 4152212.36it/s]\u001b[A\n25075712it [00:07, 4273538.46it/s]\u001b[A\n25509888it [00:07, 4017078.80it/s]\u001b[A\n25944064it [00:07, 4085692.51it/s]\u001b[A\n26443776it [00:07, 4263559.18it/s]\u001b[A\n26877952it [00:07, 4090319.91it/s]\u001b[A\n27295744it [00:07, 4100436.04it/s]\u001b[A\n27787264it [00:07, 4158997.72it/s]\u001b[A\n28213248it [00:07, 4132457.76it/s]\u001b[A\n28688384it [00:08, 3894596.02it/s]\u001b[A\n29229056it [00:08, 4173069.21it/s]\u001b[A\n29720576it [00:08, 4155528.71it/s]\u001b[A\n30277632it [00:08, 4226435.12it/s]\u001b[A\n30834688it [00:08, 4319369.99it/s]\u001b[A\n31391744it [00:08, 4357814.07it/s]\u001b[A\n31834112it [00:08, 4061365.46it/s]\u001b[A\n32333824it [00:08, 4084088.03it/s]\u001b[A\n32342954it [00:08, 3640070.88it/s]\u001b[A","name":"stderr"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_head = torch.nn.Linear(model.classifier.in_features, NUM_CLASSES)\nmodel.classifier = new_head","execution_count":28,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.cuda();","execution_count":29,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"normalizer = transforms.Normalize(mean=[0.485, 0.456, 0.406],\n std=[0.229, 0.224, 0.225])\n\ntrain_augmentation = transforms.Compose([\n transforms.Resize((IMG_SIZE,IMG_SIZE)),\n transforms.ToTensor(),\n normalizer,\n])\n\nval_augmentation = transforms.Compose([\n transforms.Resize((IMG_SIZE,IMG_SIZE)),\n transforms.ToTensor(),\n normalizer,\n])","execution_count":30,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class IMetDataset(Dataset):\n \n def __init__(self,\n df,\n images_dir,\n n_classes = NUM_CLASSES,\n id_colname = ID_COLNAME,\n answer_colname = ANSWER_COLNAME,\n label_dict = CLASSMAP,\n transforms = None\n ):\n self.df = df\n self.images_dir = images_dir\n self.n_classes = n_classes\n self.id_colname = id_colname\n self.answer_colname = answer_colname\n self.label_dict = label_dict\n self.transforms = transforms\n \n def __len__(self):\n return self.df.shape[0]\n \n def __getitem__(self, idx):\n cur_idx_row = self.df.iloc[idx]\n img_id = cur_idx_row[self.id_colname]\n img_name = img_id # + self.img_ext\n img_path = os.path.join(self.images_dir, img_name)\n \n img = cv2.imread(img_path)\n img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n img = Image.fromarray(img)\n \n if self.transforms is not None:\n img = self.transforms(img)\n \n if self.answer_colname is not None: \n label = torch.zeros((self.n_classes,), dtype=torch.float32)\n label[self.label_dict[cur_idx_row[self.answer_colname]]] = 1.0\n\n return img, label\n \n else:\n return img, img_id","execution_count":31,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = IMetDataset(train_df, TRAIN_IMGS_DIR, transforms = train_augmentation)\ntest_dataset = IMetDataset(test_df, TRAIN_IMGS_DIR, transforms = val_augmentation)","execution_count":32,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BS = 24\n\ntrain_loader = DataLoader(train_dataset, batch_size=BS, shuffle=True, num_workers=2, pin_memory=True)\ntest_loader = DataLoader(test_dataset, batch_size=BS, shuffle=False, num_workers=2, pin_memory=True)","execution_count":33,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def cuda(x):\n return x.cuda(non_blocking=True)","execution_count":34,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def f1_score(y_true, y_pred, threshold=0.5):\n return fbeta_score(y_true, y_pred, 1, threshold)\n\n\ndef fbeta_score(y_true, y_pred, beta, threshold, eps=1e-9):\n beta2 = beta**2\n\n y_pred = torch.ge(y_pred.float(), threshold).float()\n y_true = y_true.float()\n\n true_positive = (y_pred * y_true).sum(dim=1)\n precision = true_positive.div(y_pred.sum(dim=1).add(eps))\n recall = true_positive.div(y_true.sum(dim=1).add(eps))\n\n return torch.mean(\n (precision*recall).\n div(precision.mul(beta2) + recall + eps).\n mul(1 + beta2))","execution_count":35,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_one_epoch(model, train_loader, criterion, optimizer, steps_upd_logging = 250):\n model.train();\n \n total_loss = 0.0\n \n train_tqdm = tqdm_notebook(train_loader)\n \n for step, (features, targets) in enumerate(train_tqdm):\n features, targets = cuda(features), cuda(targets)\n \n optimizer.zero_grad()\n \n logits = model(features)\n \n loss = criterion(logits, targets)\n loss.backward()\n optimizer.step()\n \n total_loss += loss.item()\n \n if (step + 1) % steps_upd_logging == 0:\n logstr = f'Train loss on step {step + 1} was {round(total_loss / (step + 1), 5)}'\n train_tqdm.set_description(logstr)\n kaggle_commit_logger(logstr, need_print=False)\n \n return total_loss / (step + 1)","execution_count":36,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def validate(model, valid_loader, criterion, need_tqdm = False):\n model.eval();\n \n test_loss = 0.0\n TH_TO_ACC = 0.5\n \n true_ans_list = []\n preds_cat = []\n \n with torch.no_grad():\n \n if need_tqdm:\n valid_iterator = tqdm_notebook(valid_loader)\n else:\n valid_iterator = valid_loader\n \n for step, (features, targets) in enumerate(valid_iterator):\n features, targets = cuda(features), cuda(targets)\n\n logits = model(features)\n loss = criterion(logits, targets)\n\n test_loss += loss.item()\n true_ans_list.append(targets)\n preds_cat.append(torch.sigmoid(logits))\n\n all_true_ans = torch.cat(true_ans_list)\n all_preds = torch.cat(preds_cat)\n \n f1_eval = f1_score(all_true_ans, all_preds).item()\n\n logstr = f'Mean val f1: {round(f1_eval, 5)}'\n kaggle_commit_logger(logstr)\n return test_loss / (step + 1), f1_eval","execution_count":37,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = torch.nn.BCEWithLogitsLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=0.0005)\nsheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.5, patience=3)","execution_count":38,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nTRAIN_LOGGING_EACH = 500\n\ntrain_losses = []\nvalid_losses = []\nvalid_f1s = []\nbest_model_f1 = 0.0\nbest_model = None\nbest_model_ep = 0\n\nfor epoch in range(1, N_EPOCHS + 1):\n ep_logstr = f\"Starting {epoch} epoch...\"\n kaggle_commit_logger(ep_logstr)\n tr_loss = train_one_epoch(model, train_loader, criterion, optimizer, TRAIN_LOGGING_EACH)\n train_losses.append(tr_loss)\n tr_loss_logstr = f'Mean train loss: {round(tr_loss,5)}'\n kaggle_commit_logger(tr_loss_logstr)\n \n valid_loss, valid_f1 = validate(model, test_loader, criterion) \n valid_losses.append(valid_loss) \n valid_f1s.append(valid_f1) \n val_loss_logstr = f'Mean valid loss: {round(valid_loss,5)}'\n kaggle_commit_logger(val_loss_logstr)\n sheduler.step(valid_loss)\n \n if valid_f1 >= best_model_f1: \n best_model = model \n best_model_f1 = valid_f1 \n best_model_ep = epoch","execution_count":39,"outputs":[{"output_type":"stream","text":"Starting 1 epoch...\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"HBox(children=(IntProgress(value=0, max=6953), HTML(value='')))","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"afc7369f2fa84164a48a59dd28971b18"}},"metadata":{}},{"output_type":"error","ename":"KeyboardInterrupt","evalue":"","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)","\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n","\u001b[0;32m\u001b[0m in \u001b[0;36mtrain_one_epoch\u001b[0;34m(model, train_loader, criterion, optimizer, steps_upd_logging)\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0mtrain_tqdm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtqdm_notebook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargets\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_tqdm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 9\u001b[0m \u001b[0mfeatures\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtargets\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tqdm/_tqdm_notebook.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 219\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__iter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 220\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 221\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtqdm_notebook\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__iter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 222\u001b[0m \u001b[0;31m# return super(tqdm...) will not catch exception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 223\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tqdm/_tqdm.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1020\u001b[0m \"\"\"), fp_write=getattr(self.fp, 'write', sys.stderr.write))\n\u001b[1;32m 1021\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1022\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1023\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1024\u001b[0m \u001b[0;31m# Update and possibly print the progressbar.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 629\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 630\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshutdown\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatches_outstanding\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 631\u001b[0;31m \u001b[0midx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 632\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatches_outstanding\u001b[0m \u001b[0;34m-=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 633\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0midx\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrcvd_idx\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_get_batch\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 599\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory_thread\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_alive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 600\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 601\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata_queue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mMP_STATUS_CHECK_INTERVAL\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 602\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mqueue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEmpty\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 603\u001b[0m \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/queue.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, block, timeout)\u001b[0m\n\u001b[1;32m 171\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mremaining\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0;36m0.0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 172\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mEmpty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 173\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnot_empty\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mremaining\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 174\u001b[0m \u001b[0mitem\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 175\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnot_full\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnotify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 297\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 298\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 299\u001b[0;31m \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 300\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 301\u001b[0m \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mKeyboardInterrupt\u001b[0m: "]}]},{"metadata":{"trusted":true},"cell_type":"code","source":"bestmodel_logstr = f'Best f1 is {round(best_model_f1, 5)} on epoch {best_model_ep}'\nkaggle_commit_logger(bestmodel_logstr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xs = list(range(1, len(train_losses) + 1))\n\nplt.plot(xs, train_losses, label = 'Train loss');\n# plt.plot(xs, valid_losses, label = 'Val loss');\nplt.plot(xs, valid_f1s, label = 'Val f1');\nplt.legend();\nplt.xticks(xs);\nplt.xlabel('Epochs');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SAMPLE_SUBMISSION_DF = pd.read_csv('../input/sample_submission.csv')\nSAMPLE_SUBMISSION_DF.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SAMPLE_SUBMISSION_DF.rename(columns={'Id':'file_name','Predicted':'category_id'}, inplace=True)\nSAMPLE_SUBMISSION_DF['file_name'] = SAMPLE_SUBMISSION_DF['file_name'] + '.jpg'\nSAMPLE_SUBMISSION_DF.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm_dataset = IMetDataset(SAMPLE_SUBMISSION_DF,\n TEST_IMGS_DIR,\n transforms = val_augmentation,\n answer_colname=None\n )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SUMB_BS = 48\n\nsubm_dataloader = DataLoader(subm_dataset,\n batch_size=SUMB_BS,\n shuffle=False,\n pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_subm_answers(model, subm_dataloader, need_tqdm = False):\n model.eval();\n preds_cat = []\n ids = []\n \n with torch.no_grad():\n \n if need_tqdm:\n subm_iterator = tqdm_notebook(subm_dataloader)\n else:\n subm_iterator = subm_dataloader\n \n for step, (features, subm_ids) in enumerate(subm_iterator):\n features = cuda(features)\n\n logits = model(features)\n preds_cat.append(torch.sigmoid(logits))\n ids += subm_ids\n\n all_preds = torch.cat(preds_cat)\n all_preds = torch.argmax(all_preds, dim=1).int().cpu().numpy()\n return all_preds, ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nbest_model.cuda();\n\nsubm_preds, submids = get_subm_answers(best_model, subm_dataloader, True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(subm_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ans_dict = dict(zip(submids, subm_preds.astype(str)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_to_process = (\n pd.DataFrame\n .from_dict(ans_dict, orient='index', columns=['Predicted'])\n .reset_index()\n .rename({'index':'Id'}, axis=1) \n)\ndf_to_process['Id'] = df_to_process['Id'].map(lambda x: str(x)[:-4])\ndf_to_process.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def process_one_id(id_classes_str):\n if id_classes_str:\n return REVERSE_CLASSMAP[int(id_classes_str)]\n else:\n return id_classes_str","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_to_process['Predicted'] = df_to_process['Predicted'].apply(process_one_id)\ndf_to_process.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_to_process.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}