{"cells":[{"metadata":{},"cell_type":"markdown","source":"Implementation of https://openreview.net/pdf?id=ZPa2SyGcbwh using Fastai"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from fastai.vision.all import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"set_seed(999, reproducible=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"datapath = Path(\"/kaggle/input/cassava-leaf-disease-classification/\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"files = get_image_files(datapath/'train_images')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(datapath/'train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab2id = {'Cassava Bacterial Blight (CBB)':0,\n            'Cassava Brown Streak Disease (CBSD)':1,\n            'Cassava Green Mottle (CGM)':2,\n            'Cassava Mosaic Disease (CMD)':3,\n            'Healthy':4, \n              }\nid2vocab = {v:k for k,v in vocab2id.items()}\n\nvocab = list(vocab2id.keys()); vocab","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class GetLabel(DisplayedTransform):\n    def __init__(self, fname2labels): store_attr()\n    def encodes(self, o):             return self.fname2labels[str(o)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fnames = str(datapath/'train_images') + \"/\" + train_df['image_id'].values\nlabels = train_df['label'].map(id2vocab)\nfname2labels = dict(zip(fnames, labels))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dls(files, fname2labels, size = (512,512), bs=32):     \n\n    x_tfms = [PILImage.create]\n    y_tfms = [GetLabel(fname2labels), Categorize(vocab)]\n    tfms = [x_tfms, y_tfms]\n\n    dsets = Datasets(files, tfms=tfms, splits=RandomSplitter(0.2)(files))\n\n    batch_tfms = []\n    batch_tfms.append(Dihedral(p=0.5))\n    batch_tfms.append(Rotate(p=0.5, max_deg=45))\n    batch_tfms.append(RandomErasing(p=0.5, sl=0.05, sh=0.05, min_aspect=1., max_count=15))\n    batch_tfms.append(Brightness(p=0.5, max_lighting=0.3, batch=False))\n    batch_tfms.append(Hue(p=0.5, max_hue=0.1, batch=False))\n    batch_tfms.append(Saturation(p=0.5, max_lighting=0.1, batch=False))   \n    batch_tfms.append(RandomResizedCropGPU(size, min_scale=0.4))\n     \n    item_tfms = [ToTensor()]\n\n    batch_tfms = [IntToFloatTensor] + batch_tfms + [Normalize.from_stats(*imagenet_stats)]\n\n    train_dl = TfmdDL(dsets.train, shuffle=True, bs=bs, after_item=item_tfms, after_batch=batch_tfms, drop_last=False)\n    valid_dl = TfmdDL(dsets.valid, shuffle=False, bs=bs*2, after_item=item_tfms, after_batch=batch_tfms)\n    \n    dls = DataLoaders(train_dl, valid_dl, device=default_device())\n    \n    return dls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dls = get_dls(fnames[:32], fname2labels, bs=8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Counter([dls.tfms[1][0].fname2labels[str(o)] for o in dls.items])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dls.show_batch()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Timm Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q timm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Source: https://github.com/walkwithfastai/walkwithfastai.github.io/blob/master/wwf/vision/timm.py#L13\n# Cell\nfrom timm import create_model\nfrom fastai.vision.learner import _update_first_layer\n\n# Cell\ndef create_timm_body(arch:str, pretrained=True, cut=None, n_in=3):\n    \"Creates a body from any model in the `timm` library.\"\n    model = create_model(arch, pretrained=pretrained, num_classes=0, global_pool='')\n    _update_first_layer(model, n_in, pretrained)\n    if cut is None:\n        ll = list(enumerate(model.children()))\n        cut = next(i for i,o in reversed(ll) if has_pool_type(o))\n    if isinstance(cut, int): return nn.Sequential(*list(model.children())[:cut])\n    elif callable(cut): return cut(model)\n    else: raise NamedError(\"cut must be either integer or function\")\n\n# Cell\ndef create_timm_model(arch:str, n_out, cut=None, pretrained=True, n_in=3, init=nn.init.kaiming_normal_, custom_head=None,\n                     concat_pool=True, **kwargs):\n    \"Create custom architecture using `arch`, `n_in` and `n_out` from the `timm` library\"\n    body = create_timm_body(arch, pretrained, None, n_in)\n    if custom_head is None:\n        nf = num_features_model(nn.Sequential(*body.children())) * (2 if concat_pool else 1)\n        head = create_head(nf, n_out, concat_pool=concat_pool, **kwargs)\n    else: head = custom_head\n    model = nn.Sequential(body, head)\n    if init is not None: apply_init(model[1], init)\n    return model\n\n# Cell\nfrom fastai.vision.learner import _add_norm\n\n# Cell\ndef timm_learner(dls, arch:str, loss_func=None, pretrained=True, cut=None, splitter=None,\n                y_range=None, config=None, n_out=None, normalize=True, **kwargs):\n    \"Build a convnet style learner from `dls` and `arch` using the `timm` library\"\n    if config is None: config = {}\n    if n_out is None: n_out = get_c(dls)\n    assert n_out, \"`n_out` is not defined, and could not be inferred from data, set `dls.c` or pass `n_out`\"\n    if y_range is None and 'y_range' in config: y_range = config.pop('y_range')\n    model = create_timm_model(arch, n_out, default_split, pretrained, y_range=y_range, **config)\n    learn = Learner(dls, model, loss_func=loss_func, splitter=default_split, **kwargs)\n    if pretrained: learn.freeze()\n    return learn","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### ProgressiveLabelCorrection Callback"},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai.vision.all import *\n\nclass ProgressiveLabelCorrection(Callback):\n    'https://openreview.net/pdf?id=ZPa2SyGcbwh'\n    run_valid=False\n    def __init__(self, num_classes=5, theta_start=0.3, theta_end=0.05, warm_up=0.2, sched_func=sched_lin):\n        store_attr()\n        self.theta = theta_start\n\n    def before_fit(self):\n        self.eye = torch.eye(self.num_classes).to(self.dls.device)\n    \n    def after_step(self):        \n        if self.pct_train > self.warm_up:\n#             set_trace()\n            # get batch items\n            self.idxs = self.dl._DataLoader__idxs\n            self.b_items = L(list(self.dl.items))[self.idxs[self.iter*self.dl.bs:self.iter*self.dl.bs+self.dl.bs]]\n\n            # get incorrectly classified item boolean mask\n            preds_max = self.pred.argmax(-1)\n            mislabeled_idxs = preds_max != self.y\n            \n            tot_mislabeled = sum(mislabeled_idxs)\n            if tot_mislabeled > 0:\n                # get probas and targs for incorrect batch items\n                mislabeled_probas = self.pred[mislabeled_idxs].softmax(-1)\n                mislabeled_targs = self.y[mislabeled_idxs]\n\n                # get predicted and actual probas and targ\n                predicted_probas = mislabeled_probas.max(-1).values\n                predicted_targs = mislabeled_probas.max(-1).indices\n                actual_probas = mislabeled_probas[self.eye[mislabeled_targs].bool()]\n\n                # update labels or theta \n                msk = torch.abs(predicted_probas - actual_probas) > self.theta\n                self.epoch_tot_updated += sum(msk)\n                \n                new_targs = self.dl.tfms[1][1].vocab[predicted_targs[msk]]\n                items_to_change = self.b_items[mislabeled_idxs][msk]\n\n                update_dict = dict(zip(items_to_change, new_targs))\n                self.dl.tfms[1][0].fname2labels.update(update_dict)\n                print(f\"Total changed items in this epoch: {self.epoch_tot_updated}\")\n                print(\"New Label Distribution\")\n                print(Counter([dls.tfms[1][0].fname2labels[str(o)] for o in dls.items]))\n                \n    def before_epoch(self): \n        self.epoch_tot_updated = 0\n        \n    def after_epoch(self):   \n        if (self.pct_train > self.warm_up) and (self.epoch_tot_updated == 0):\n            self.theta = self.sched_func(self.theta_start, self.theta_end, self.pct_train)\n            print(f\"Reduced theta to: {self.theta}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Learner"},{"metadata":{"trusted":true},"cell_type":"code","source":"arch_name = 'efficientnet_b3a'\nmodel = create_timm_model(arch_name, 5, ps=0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _timm_split(m): return L(m[0], m[1]).map(params)\nloss_func = LabelSmoothingCrossEntropy(0.01)\nmetric = accuracy\ncbs = [SaveModelCallback(metric.__name__, fname=arch_name), \n       TerminateOnNaNCallback()]\n\ndls = get_dls(fnames, fname2labels, bs=32)\nlearner = Learner(dls, model, metrics=metric, loss_func=loss_func, cbs=cbs, splitter=_timm_split)\nlearner.to_native_fp16();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.freeze_to(-1)\nlearner.fit_flat_cos(3, 1e-2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.unfreeze()\nlearner.fit_flat_cos(3, slice(5e-4, 1e-3), pct_start=0.0, wd=1e-2,\n                     cbs=[ProgressiveLabelCorrection(theta_start=0.4, theta_end=0.1, warm_up=0.2)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.export(f\"models/{arch_name}_export.pkl\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Upvote if you find it useful and let me know if you have any feedback!"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}