{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# fastai training with the data-block API\nfastai is a great tool to create a strong baseline quickly. I use pretty much out of the box approach for multilabel classification, with resnet50 backbone, one cycle training, lr finder etc. The data block API is a great way to prepare the data, and comes with a default set of augmentations that I use as well.\n\nSolution overview: https://www.kaggle.com/c/hpa-single-cell-image-classification/discussion/221550\n\n### I will smile for every upvote :) ","metadata":{}},{"cell_type":"code","source":"!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom fastai.vision.all import *\nimport fastai.vision.all\nimport pickle\nimport os\n\n# dir(fastai.vision.all)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Making pretrained weights work without needing to find the default filename\nif not os.path.exists('/root/.cache/torch/hub/checkpoints/'):\n        os.makedirs('/root/.cache/torch/hub/checkpoints/')\n!cp '../input/resnet50/resnet50.pth' '/root/.cache/torch/hub/checkpoints/resnet50-19c8e357.pth'\n# !cp '../input/resnet34/resnet34.pth' '/root/.cache/torch/hub/checkpoints/resnet34-333f7ec4.pth'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = Path('../input/hpa-cell-tiles-sample-balanced-dataset')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(path/'cell_df.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = [str(i) for i in range(19)]\nfor x in labels: df[x] = df['image_labels'].apply(lambda r: int(x in r.split('|')))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Change below to `frac=1` to run on the whole training sample","metadata":{}},{"cell_type":"code","source":"dfs = df.sample(frac=0.1, random_state=42)\ndfs = dfs.reset_index(drop=True)\nlen(dfs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_counts = {}\nfor lbl in labels:\n    unique_counts[lbl] = len(dfs[dfs.image_labels == lbl])\n\nfull_counts = {}\nfor lbl in labels:\n    count = 0\n    for row_label in dfs['image_labels']:\n        if lbl in row_label.split('|'): count += 1\n    full_counts[lbl] = count\n    \ncounts = list(zip(full_counts.keys(), full_counts.values(), unique_counts.values()))\ncounts = np.array(sorted(counts, key=lambda x:-x[1]))\ncounts = pd.DataFrame(counts, columns=['label', 'full_count', 'unique_count'])\ncounts.set_index('label').T\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(dfs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Using multilabel stratification for the train-validation split.\n\nThere is some leakage in the code below (cells belonging to the same image should be in the same split). However, when I fixed that, I got a lower score... coincidence? ","metadata":{}},{"cell_type":"code","source":"nfold = 5\nseed = 42\n\ny = dfs[labels].values\nX = dfs[['image_id', 'cell_id']].values\n\ndfs['fold'] = np.nan\n\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\nmskf = MultilabelStratifiedKFold(n_splits=nfold, random_state=seed)\nfor i, (_, test_index) in enumerate(mskf.split(X, y)):\n    dfs.iloc[test_index, -1] = i\n    \ndfs['fold'] = dfs['fold'].astype('int')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs['is_valid'] = False\ndfs['is_valid'][dfs['fold'] == 0] = True","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs.is_valid.value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Using fastai data block API with item and batch transforms\n\nRead more: https://docs.fast.ai/tutorial.datablock.html","metadata":{}},{"cell_type":"code","source":"def get_x(r): return path/'cells'/(r['image_id']+'_'+str(r['cell_id'])+'.jpg')\nimg = get_x(dfs.loc[12])\nimg = PILImage.create(img)\nimg.show();","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_y(r): return r['image_labels'].split('|')\nget_y(dfs.loc[12])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_stats = ([0.07237246, 0.04476176, 0.07661699], [0.17179589, 0.10284516, 0.14199627])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item_tfms = RandomResizedCrop(224, min_scale=0.75, ratio=(1.,1.))\nbatch_tfms = [*aug_transforms(flip_vert=True, size=128, max_warp=0), Normalize.from_stats(*sample_stats)]\nbs=256\nitem_tfms","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dblock = DataBlock(blocks=(ImageBlock, MultiCategoryBlock(vocab=labels)),\n                splitter=ColSplitter(col='is_valid'),\n                get_x=get_x,\n                get_y=get_y,\n                item_tfms=item_tfms,\n                batch_tfms=batch_tfms\n                )\ndls = dblock.dataloaders(dfs, bs=bs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dblock.summary(dfs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.show_batch(nrows=3, ncols=3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Let's train!","metadata":{}},{"cell_type":"code","source":"!pip install efficientnet-pytorch\nfrom efficientnet_pytorch import EfficientNet\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = EfficientNet.from_pretrained('efficientnet-b4', num_classes=dls.c)\nlearn = Learner(dls, model, metrics=[accuracy_multi, PrecisionMulti()]).to_fp16()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# learn = cnn_learner(dls, resnet101, metrics=[accuracy_multi, PrecisionMulti()]).to_fp16()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# learn.lr_find()\n# SuggestedLRs(lr_min=0.03630780577659607, lr_steep=0.02754228748381138)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nlearn.lr_find()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I trained for 10 epochs in the 0.342 leaderboard submission. ","metadata":{}},{"cell_type":"code","source":"lr = 0.0.006918309628963471\nlearn.fine_tune(10, base_lr=lr)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(resnet18)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.recorder.plot_loss()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Where are the mistakes? ","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import multilabel_confusion_matrix as cm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# val_targ = torch.stack([x[1] for x in learn.dls.valid_ds], dim=0).numpy()\n# val_targ.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_targ = dfs[labels][dfs.is_valid == True].values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_targ.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_preds_all = learn.get_preds(dl=learn.dls.valid)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_preds = val_preds_all[0].numpy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_preds = val_preds > 0.5","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_preds = val_preds_all[0].numpy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vis_arr = cm(val_targ, val_preds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# i = 60\n# print(learn.dls.valid.dataset[i][1])\n# print(val_preds[i])\n# print(full_preds[i])\n# learn.dls.valid.dataset[i][0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n\ndef print_confusion_matrix(confusion_matrix, axes, class_label, class_names, fontsize=14):\n\n    df_cm = pd.DataFrame(\n        confusion_matrix, index=class_names, columns=class_names,\n    )\n\n    try:\n        heatmap = sns.heatmap(df_cm, annot=True, fmt=\"d\", cbar=False, ax=axes)\n    except ValueError:\n        raise ValueError(\"Confusion matrix values must be integers.\")\n    heatmap.yaxis.set_ticklabels(heatmap.yaxis.get_ticklabels(), rotation=0, ha='right', fontsize=fontsize)\n    heatmap.xaxis.set_ticklabels(heatmap.xaxis.get_ticklabels(), rotation=45, ha='right', fontsize=fontsize)\n    axes.set_ylabel('True label')\n    axes.set_xlabel('Predicted label')\n    axes.set_title(\"Confusion Matrix for the class - \" + class_label)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(5, 4, figsize=(12, 16))\n    \nfor axes, cfs_matrix, label in zip(ax.flatten(), vis_arr, labels):\n    print_confusion_matrix(cfs_matrix, axes, label, [\"0\", \"1\"])\n\nfig.tight_layout()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val = dfs[dfs.is_valid==True]\nlen(val[val['16'] == 1])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import average_precision_score\naverage_precision = average_precision_score(val_targ, val_preds)\naverage_precision","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import precision_recall_curve\n\nprecision = dict()\nrecall = dict()\naverage_precision = dict()\nfor i in range(19):\n    precision[i], recall[i], _ = precision_recall_curve(val_targ[:, i], val_preds[:, i])\n    average_precision[i] = average_precision_score(val_targ[:, i], val_preds[:, i])\n\n# A \"micro-average\": quantifying score on all classes jointly\nprecision[\"micro\"], recall[\"micro\"], _ = precision_recall_curve(val_targ.ravel(), val_preds.ravel())\naverage_precision[\"micro\"] = average_precision_score(val_targ, val_preds, average=\"micro\")\nprint('Average precision score, micro-averaged over all classes: {0:0.2f}'.format(average_precision[\"micro\"]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"average_precision","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference\nThis is running on the public test data preprocessed in the same way as train. We will save both regular preds and preds with TTA so that we can use them later in a separate submission notebook. ","metadata":{}},{"cell_type":"code","source":"path = Path('../input/hpa-cell-tiles-test-with-enc-dataset')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(path/'cell_df.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv('cell_df.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dl = learn.dls.test_dl(df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dl.show_batch()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds, _ = learn.get_preds(dl=test_dl)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('preds.pickle', 'wb') as handle:\n    pickle.dump(preds, handle)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tta, _ = learn.tta(dl=test_dl)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tta.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('tta.pickle', 'wb') as handle:\n    pickle.dump(tta, handle)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cls_prds = torch.argmax(preds, dim=-1)\nlen(cls_prds), cls_prds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv('../input/hpa-single-cell-image-classification/sample_submission.csv')\nsample_submission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['cls'] = cls_prds\ndf['pred'] = df[['cls', 'enc']].apply(lambda r: str(r[0]) + ' 1 ' + r[1], axis=1)\ndf.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm = df.groupby(['image_id'])['pred'].apply(lambda x: ' '.join(x)).reset_index()\n# subm = subm.loc[3:]\nsubm.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.merge(\n    sample_submission,\n    subm,\n    how=\"left\",\n    left_on='ID',\n    right_on='image_id',\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def isNaN(num):\n    return num != num","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, row in sub.iterrows():\n    if isNaN(row['pred']): continue\n    sub.PredictionString.loc[i] = row['pred']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = sub[sample_submission.columns]\nsub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Thank you for your attention! Looking forward to questions and comments!","metadata":{}},{"cell_type":"code","source":"sub\n\\","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}