{"cells":[{"metadata":{},"cell_type":"markdown","source":"# fastai training loop with the data-block API\nfastai is a great tool to create a strong baseline quickly. I use pretty much out of the box approach for multilabel classification, with resnet50 backbone, one cycle training, lr finder etc. The data block API is a great way to prepare the data, and comes with a default set of augmentations that I use as well.\n\nthis is based on \nSolution overview: https://www.kaggle.com/c/hpa-single-cell-image-classification/discussion/221550\n\nand forked to illustrate an issue with cutmix"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom fastai.vision.all import *\nimport pickle\nimport os","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Making pretrained weights work without needing to find the default filename\nif not os.path.exists('/root/.cache/torch/hub/checkpoints/'):\n        os.makedirs('/root/.cache/torch/hub/checkpoints/')\n!cp '../input/resnet50/resnet50.pth' '/root/.cache/torch/hub/checkpoints/resnet50-19c8e357.pth'\n# !cp '../input/resnet34/resnet34.pth' '/root/.cache/torch/hub/checkpoints/resnet34-333f7ec4.pth'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = Path('../input/hpa-cell-tiles-sample-balanced-dataset')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(path/'cell_df.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = [str(i) for i in range(19)]\nfor x in labels: df[x] = df['image_labels'].apply(lambda r: int(x in r.split('|')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfs = df.sample(frac=1, random_state=42)\ndfs = dfs.reset_index(drop=True)\nlen(dfs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unique_counts = {}\nfor lbl in labels:\n    unique_counts[lbl] = len(dfs[dfs.image_labels == lbl])\n\nfull_counts = {}\nfor lbl in labels:\n    count = 0\n    for row_label in dfs['image_labels']:\n        if lbl in row_label.split('|'): count += 1\n    full_counts[lbl] = count\n    \ncounts = list(zip(full_counts.keys(), full_counts.values(), unique_counts.values()))\ncounts = np.array(sorted(counts, key=lambda x:-x[1]))\ncounts = pd.DataFrame(counts, columns=['label', 'full_count', 'unique_count'])\ncounts.set_index('label').T\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(dfs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nfold = 5\nseed = 42\n\ny = dfs[labels].values\nX = dfs[['image_id', 'cell_id']].values\n\ndfs['fold'] = np.nan\n\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\nmskf = MultilabelStratifiedKFold(n_splits=nfold, random_state=seed)\nfor i, (_, test_index) in enumerate(mskf.split(X, y)):\n    dfs.iloc[test_index, -1] = i\n    \ndfs['fold'] = dfs['fold'].astype('int')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfs['is_valid'] = False\ndfs['is_valid'][dfs['fold'] == 0] = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfs.is_valid.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_x(r): return path/'cells'/(r['image_id']+'_'+str(r['cell_id'])+'.jpg')\nimg = get_x(dfs.loc[12])\nimg = PILImage.create(img)\nimg.show();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_y(r): return r['image_labels'].split('|')\nget_y(dfs.loc[12])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_stats = ([0.07237246, 0.04476176, 0.07661699], [0.17179589, 0.10284516, 0.14199627])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"item_tfms = RandomResizedCrop(224, min_scale=0.75, ratio=(1.,1.))\nbatch_tfms = [*aug_transforms(flip_vert=True, size=128,max_warp=0.2,max_lighting = 0.5,max_rotate =60), Normalize.from_stats(*sample_stats)]\nbs=256","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_y_bce(r): \n    \n    categories = r['image_labels'].split('|')\n    n_categories = len(categories)\n    arr = np.zeros(len(labels))\n    for l in categories:\n        \n        arr[int(l)]= 1 #+ np.log(1/n_categories)/5\n    \n    \n    return arr\n\nget_y_bce(dfs.loc[12])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dblock = DataBlock(blocks=(ImageBlock,RegressionBlock(n_out=len(labels))),\n                splitter=ColSplitter(col='is_valid'),\n                get_x=get_x,\n                get_y=get_y_bce,\n                item_tfms=item_tfms,\n                batch_tfms=batch_tfms\n                )\ndls = dblock.dataloaders(dfs, bs=bs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dblock.summary(dfs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dls.show_batch(nrows=9, ncols=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cutmix = CutMix(0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = cnn_learner(dls, resnet50, metrics=[SpearmanCorrCoef()]).to_fp16()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":" learn.lr_find()\n# SuggestedLRs(lr_min=0.03630780577659607, lr_steep=0.02754228748381138)"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr=3e-2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# calback error"},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fine_tune(1,base_lr=lr,cbs=cutmix)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.recorder.plot_loss()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Workaround"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CutMix(MixHandler):\n    \"Implementation of `https://arxiv.org/abs/1905.04899`\"\n    def __init__(self, alpha=1.): super().__init__(alpha)\n    def before_batch(self):\n        bs, _, H, W = self.x.size()\n        self.lam = self.distrib.sample((1,)).to(self.x.device)\n        shuffle = torch.randperm(bs).to(self.x.device)\n        xb1,self.yb1 = self.x[shuffle], tuple((self.y[shuffle],))\n        x1, y1, x2, y2 = self.rand_bbox(W, H, self.lam)\n        self.learn.xb[0][..., y1:y2, x1:x2] = xb1[..., y1:y2, x1:x2]\n        self.lam = (1 - ((x2-x1)*(y2-y1))/float(W*H))\n        if not self.stack_y:\n            ny_dims = len(self.y.size())\n            self.learn.yb = tuple(L(self.yb1,self.yb).map_zip(torch.lerp,weight=unsqueeze(self.lam, n=ny_dims-1)))\n\n    def rand_bbox(self, W, H, lam):\n        cut_rat = torch.sqrt(1. - lam).to(self.x.device)\n        cut_w = torch.round(W * cut_rat).type(torch.long).to(self.x.device)\n        cut_h = torch.round(H * cut_rat).type(torch.long).to(self.x.device)\n        # uniform\n        cx = torch.randint(0, W, (1,)).to(self.x.device)\n        cy = torch.randint(0, H, (1,)).to(self.x.device)\n        x1 = torch.clamp(cx - cut_w // 2, 0, W)\n        y1 = torch.clamp(cy - cut_h // 2, 0, H)\n        x2 = torch.clamp(cx + cut_w // 2, 0, W)\n        y2 = torch.clamp(cy + cut_h // 2, 0, H)\n        return x1, y1, x2, y2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cutmix = CutMix(0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = cnn_learner(dls, resnet50,loss_func=torch.nn.BCEWithLogitsLoss(), metrics=[SpearmanCorrCoef()]).to_fp16()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fine_tune(1,base_lr=lr,cbs=cutmix)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.recorder.plot_loss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}