{"cells":[{"metadata":{},"cell_type":"markdown","source":"**Simple example of transfer learning from pretrained model using PyTorch.**\n* Metrics: f1_score"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import os\n\nimport numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\nimport glob\n\nimport cv2\nimport torch\nfrom tqdm import tqdm_notebook\nfrom PIL import Image, ImageFile\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models\nfrom sklearn.model_selection import train_test_split\nfrom torchvision import transforms\nfrom skimage import io\nImageFile.LOAD_TRUNCATED_IMAGES = True\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def kaggle_commit_logger(str_to_log, need_print = True):\n    if need_print:\n        print(str_to_log)\n    os.system('echo ' + str_to_log)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import json\nwith open(r'/kaggle/input/iwildcam-2020-fgvc7/iwildcam2020_train_annotations.json') as json_file:\n    train_data = json.load(json_file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.DataFrame({'id': [item['id'] for item in train_data['annotations']],\n                                'category_id': [item['category_id'] for item in train_data['annotations']],\n                                'image_id': [item['image_id'] for item in train_data['annotations']],\n                                'file_name': [item['file_name'] for item in train_data['images']]})\n\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_image = pd.DataFrame.from_records(train_data['images'])\n\nindices = []\nfor _id in df_image[df_image['location'] == 537]['id'].values:\n    indices.append( df_train[ df_train['image_id'] == _id ].index )\n\nfor the_index in indices:\n    df_train = df_train.drop(df_train.index[the_index])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Image.open('../input/iwildcam-2020-fgvc7/train/8792549a-21bc-11ea-a13a-137349068a90.jpg')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nindices = []\nfor i in df_train['file_name']:\n    try:\n        Image.open('/kaggle/input/iwildcam-2020-fgvc7/train/' + i)\n    except:        \n        print(i)\n        df_train.drop(df_train.loc[df_train['file_name']==i].index, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(r'/kaggle/input/iwildcam-2020-fgvc7/iwildcam2020_test_information.json') as f:\n    test_data = json.load(f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = pd.DataFrame.from_records(test_data['images'])\ndf_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 64\nIMG_SIZE = 64\n\nN_EPOCHS = 1\n\nID_COLNAME = 'file_name'\nANSWER_COLNAME = 'category_id'\nTRAIN_IMGS_DIR = r'../input/iwildcam-2020-fgvc7/train/'\nTEST_IMGS_DIR = r'../input/iwildcam-2020-fgvc7/test/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df, test_df = train_test_split(df_train[[ID_COLNAME, ANSWER_COLNAME]],\n                                     test_size = 0.15,                                     \n                                     shuffle = True\n                                    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"CLASSES_TO_USE = df_train['category_id'].unique()\nCLASSES_TO_USE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_CLASSES = len(CLASSES_TO_USE)\nNUM_CLASSES","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CLASSMAP = dict(\n    [(i, j) for i, j\n     in zip(CLASSES_TO_USE, range(NUM_CLASSES))\n    ]\n)\nCLASSMAP","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"REVERSE_CLASSMAP = dict([(v, k) for k, v in CLASSMAP.items()])\nREVERSE_CLASSMAP","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"model = models.densenet121(pretrained='imagenet')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_head = torch.nn.Linear(model.classifier.in_features, NUM_CLASSES)\nmodel.classifier = new_head","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_state_dict(torch.load('../input/iwild2020-torch/model'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.cuda();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"normalizer = transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                 std=[0.229, 0.224, 0.225])\n\ntrain_augmentation = transforms.Compose([\n    transforms.Resize((IMG_SIZE,IMG_SIZE)),\n    transforms.ToTensor(),\n    normalizer,\n])\n\nval_augmentation = transforms.Compose([\n    transforms.Resize((IMG_SIZE,IMG_SIZE)),\n    transforms.ToTensor(),\n    normalizer,\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class IMetDataset(Dataset):\n    \n    def __init__(self,\n                 df,\n                 images_dir,\n                 n_classes = NUM_CLASSES,\n                 id_colname = ID_COLNAME,\n                 answer_colname = ANSWER_COLNAME,\n                 label_dict = CLASSMAP,\n                 transforms = None\n                ):\n        self.df = df\n        self.images_dir = images_dir\n        self.n_classes = n_classes\n        self.id_colname = id_colname\n        self.answer_colname = answer_colname\n        self.label_dict = label_dict\n        self.transforms = transforms\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, idx):        \n        cur_idx_row = self.df.iloc[idx]\n        img_id = cur_idx_row[self.id_colname]\n        img_name = img_id # + self.img_ext\n        img_path = os.path.join(self.images_dir, img_name)\n          \n        img = Image.open(img_path)\n\n        if self.transforms is not None:\n            img = self.transforms(img)\n\n        if self.answer_colname is not None:              \n            label = torch.zeros((self.n_classes,), dtype=torch.float32)\n            label[self.label_dict[cur_idx_row[self.answer_colname]]] = 1.0\n\n            return img, label\n\n        else:\n            return img, img_id  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = IMetDataset(train_df, TRAIN_IMGS_DIR, transforms = train_augmentation)\ntest_dataset = IMetDataset(test_df, TRAIN_IMGS_DIR, transforms = val_augmentation)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BS = 24\n\ntrain_loader = DataLoader(train_dataset, batch_size=BS, shuffle=True, num_workers=2, pin_memory=True)\ntest_loader = DataLoader(test_dataset, batch_size=BS, shuffle=False, num_workers=2, pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def cuda(x):\n    return x.cuda(non_blocking=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def f1_score(y_true, y_pred, threshold=0.5):\n    return fbeta_score(y_true, y_pred, 1, threshold)\n\n\ndef fbeta_score(y_true, y_pred, beta, threshold, eps=1e-9):\n    beta2 = beta**2\n\n    y_pred = torch.ge(y_pred.float(), threshold).float()\n    y_true = y_true.float()\n\n    true_positive = (y_pred * y_true).sum(dim=1)\n    precision = true_positive.div(y_pred.sum(dim=1).add(eps))\n    recall = true_positive.div(y_true.sum(dim=1).add(eps))\n\n    return torch.mean(\n        (precision*recall).\n        div(precision.mul(beta2) + recall + eps).\n        mul(1 + beta2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_one_epoch(model, train_loader, criterion, optimizer, steps_upd_logging = 250):\n    model.train();\n    \n    total_loss = 0.0\n    \n    train_tqdm = tqdm_notebook(train_loader)\n    \n    \n    for step, (features, targets) in enumerate(train_tqdm):\n        try:        \n            features, targets = cuda(features), cuda(targets)\n\n            optimizer.zero_grad()\n\n            logits = model(features)\n\n            loss = criterion(logits, targets)\n            loss.backward()\n            optimizer.step()\n\n            total_loss += loss.item()\n\n            if (step + 1) % steps_upd_logging == 0:\n                logstr = f'Train loss on step {step + 1} was {round(total_loss / (step + 1), 5)}'\n                train_tqdm.set_description(logstr)\n                kaggle_commit_logger(logstr, need_print=False)\n                \n        except:\n            pass\n        \n    return total_loss / (step + 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def validate(model, valid_loader, criterion, need_tqdm = False):\n    model.eval();\n    \n    test_loss = 0.0\n    TH_TO_ACC = 0.5\n    \n    true_ans_list = []\n    preds_cat = []\n    \n    with torch.no_grad():\n        \n        if need_tqdm:\n            valid_iterator = tqdm_notebook(valid_loader)\n        else:\n            valid_iterator = valid_loader\n        \n        for step, (features, targets) in enumerate(valid_iterator):\n            features, targets = cuda(features), cuda(targets)\n\n            logits = model(features)\n            loss = criterion(logits, targets)\n\n            test_loss += loss.item()\n            true_ans_list.append(targets)\n            preds_cat.append(torch.sigmoid(logits))\n\n        all_true_ans = torch.cat(true_ans_list)\n        all_preds = torch.cat(preds_cat)\n                \n        f1_eval = f1_score(all_true_ans, all_preds).item()\n\n    logstr = f'Mean val f1: {round(f1_eval, 5)}'\n    kaggle_commit_logger(logstr)\n    return test_loss / (step + 1), f1_eval","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = torch.nn.BCEWithLogitsLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=0.0005)\nsheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.5, patience=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nTRAIN_LOGGING_EACH = 500\n\ntrain_losses = []\nvalid_losses = []\nvalid_f1s = []\nbest_model_f1 = 0.0\nbest_model = None\nbest_model_ep = 0\n\nfor epoch in range(1, N_EPOCHS + 1):\n    ep_logstr = f\"Starting {epoch} epoch...\"\n    kaggle_commit_logger(ep_logstr)\n    tr_loss = train_one_epoch(model, train_loader, criterion, optimizer, TRAIN_LOGGING_EACH)\n    train_losses.append(tr_loss)\n    tr_loss_logstr = f'Mean train loss: {round(tr_loss,5)}'\n    kaggle_commit_logger(tr_loss_logstr)\n\n    valid_loss, valid_f1 = validate(model, test_loader, criterion)  \n    valid_losses.append(valid_loss)    \n    valid_f1s.append(valid_f1)       \n    val_loss_logstr = f'Mean valid loss: {round(valid_loss,5)}'\n    kaggle_commit_logger(val_loss_logstr)\n    sheduler.step(valid_loss)\n\n    if valid_f1 >= best_model_f1:    \n        best_model = model        \n        best_model_f1 = valid_f1        \n        best_model_ep = epoch        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.save(best_model.state_dict(), 'model')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bestmodel_logstr = f'Best f1 is {round(best_model_f1, 5)} on epoch {best_model_ep}'\nkaggle_commit_logger(bestmodel_logstr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xs = list(range(1, len(train_losses) + 1))\n\nplt.plot(xs, train_losses, label = 'Train loss');\n# plt.plot(xs, valid_losses, label = 'Val loss');\nplt.plot(xs, valid_f1s, label = 'Val f1');\nplt.legend();\nplt.xticks(xs);\nplt.xlabel('Epochs');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SAMPLE_SUBMISSION_DF = pd.read_csv(r'../input/iwildcam-2020-fgvc7/sample_submission.csv')\nSAMPLE_SUBMISSION_DF.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SAMPLE_SUBMISSION_DF.rename(columns={'Id':'file_name','Category':'category_id'}, inplace=True)\nSAMPLE_SUBMISSION_DF['file_name'] = SAMPLE_SUBMISSION_DF['file_name'] + '.jpg'\nSAMPLE_SUBMISSION_DF.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm_dataset = IMetDataset(SAMPLE_SUBMISSION_DF,\n                           TEST_IMGS_DIR,\n                           transforms = val_augmentation,\n                           answer_colname=None\n                          )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SUMB_BS = 48\n\nsubm_dataloader = DataLoader(subm_dataset,\n                             batch_size=SUMB_BS,\n                             shuffle=False,\n                             pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_subm_answers(model, subm_dataloader, need_tqdm = False):\n    model.eval();\n    preds_cat = []\n    ids = []\n    \n    with torch.no_grad():\n        \n        if need_tqdm:\n            subm_iterator = tqdm_notebook(subm_dataloader)\n        else:\n            subm_iterator = subm_dataloader\n        \n        for step, (features, subm_ids) in enumerate(subm_iterator):\n            features = cuda(features)\n\n            logits = model(features)\n            preds_cat.append(torch.sigmoid(logits))\n            ids += subm_ids\n\n        all_preds = torch.cat(preds_cat)\n        all_preds = torch.argmax(all_preds, dim=1).int().cpu().numpy()\n    return all_preds, ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nbest_model.cuda();\n\nsubm_preds, submids = get_subm_answers(best_model, subm_dataloader, True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(subm_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ans_dict = dict(zip(submids, subm_preds.astype(str)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_to_process = (\n    pd.DataFrame\n    .from_dict(ans_dict, orient='index', columns=['Category'])\n    .reset_index()\n    .rename({'index':'Id'}, axis=1)    \n)\ndf_to_process['Id'] = df_to_process['Id'].map(lambda x: str(x)[:-4])\ndf_to_process.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def process_one_id(id_classes_str):\n    if id_classes_str:\n        return REVERSE_CLASSMAP[int(id_classes_str)]\n    else:\n        return id_classes_str","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_to_process['Category'] = df_to_process['Category'].apply(process_one_id)\ndf_to_process.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_to_process.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.4"}},"nbformat":4,"nbformat_minor":4}