{"cells":[{"metadata":{"papermill":{"duration":0.019006,"end_time":"2021-01-21T12:45:55.737653","exception":false,"start_time":"2021-01-21T12:45:55.718647","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Imports"},{"metadata":{"trusted":true},"cell_type":"code","source":"#!pip install efficientnet_pytorch\n!pip install pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":7.176828,"end_time":"2021-01-21T12:46:13.123812","exception":false,"start_time":"2021-01-21T12:46:05.946984","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from fastai.vision.all import *\nfrom fastai.vision.core import *\nfrom fastai.callback.fp16 import *\n\nfrom fastai.callback.cutmix import *\nfrom torch.distributions.beta import Beta\n\nfrom fastai.callback.wandb import *\nimport torchvision.models as models\nimport pandas as pd\nimport numpy as np\nimport distillation\n\n#from efficientnet_pytorch import EfficientNet\nimport pretrainedmodels\nimport albumentations\nimport wandb\n#import sys\n#sys.path.append('../input/timm2021/pytorch-image-models-master')\n#import timm","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.008713,"end_time":"2021-01-21T12:46:13.141788","exception":false,"start_time":"2021-01-21T12:46:13.133075","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Setup"},{"metadata":{"papermill":{"duration":0.016584,"end_time":"2021-01-21T12:46:13.217428","exception":false,"start_time":"2021-01-21T12:46:13.200844","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class Config:\n    testing     = False # must be same as create-folds.ipynb\n    image_size  = 512\n    batch_size  = 16\n    epochs      = 10\n    f_epochs    = 1\n    train_folds = ['f1']\n    arch        = 'efficientnet-b4'\n    \ncfg = Config()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_seeds():\n    random.seed(42)\n    np.random.seed(42)\n    torch.manual_seed(42)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    \nset_seeds()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.041026,"end_time":"2021-01-21T12:46:13.191866","exception":false,"start_time":"2021-01-21T12:46:13.15084","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"path_str = '../input/cassava-leaf-disease-merged'\n\nimages_path = Path(path_str + '/train')\ncsv_path = Path(path_str + '/merged.csv')\nfolds_path = Path('../input/fold-indexes/folds-merged.csv')\n\nfull_df = pd.read_csv(csv_path)\nfolds_df = pd.read_csv(folds_path)\n\n# drop rows so we get an even number for our folds and remove duplicates\nfull_df = full_df[~full_df['image_id'].isin(['1562043567.jpg', '3551135685.jpg', '2252529694.jpg', '1000015157.jpg', '1000201771.jpg', '100042118.jpg', '1001723730.jpg'])]","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.008952,"end_time":"2021-01-21T12:46:13.235438","exception":false,"start_time":"2021-01-21T12:46:13.226486","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Create a test dataset"},{"metadata":{"papermill":{"duration":1.957548,"end_time":"2021-01-21T12:46:15.202108","exception":false,"start_time":"2021-01-21T12:46:13.24456","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"if cfg.testing:\n    full_df = full_df.tail(120)#.head(120)\nelse:\n    wandb.login(key=\"11b470b697ff94b3896d2243b147d42177a5cb7a\")\n    wandb.init(project=\"cassava\", entity=\"teo03\")\n\nlen(full_df)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.015792,"end_time":"2021-01-21T12:46:15.229592","exception":false,"start_time":"2021-01-21T12:46:15.2138","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Augmentation and train functions"},{"metadata":{"papermill":{"duration":0.033375,"end_time":"2021-01-21T12:46:15.281098","exception":false,"start_time":"2021-01-21T12:46:15.247723","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class AlbumentationsTransform(RandTransform):\n    split_idx,order = None, 2\n    \n    def __init__(self, train_aug, valid_aug): \n        store_attr()\n    \n    def before_call(self, b, split_idx):\n        self.idx = split_idx\n    \n    def encodes(self, img: PILImage):\n        if self.idx == 0:\n            aug_img = self.train_aug(image=np.array(img))['image']\n        else:\n            aug_img = self.valid_aug(image=np.array(img))['image']\n        return PILImage.create(aug_img)\n\n\ndef get_train_aug(size): \n    return albumentations.Compose([\n            albumentations.RandomResizedCrop(size,size),\n            albumentations.Transpose(p=0.5),\n            albumentations.HorizontalFlip(p=0.5),\n            albumentations.VerticalFlip(p=0.5),\n            albumentations.ShiftScaleRotate(p=0.5),\n            albumentations.HueSaturationValue(\n                hue_shift_limit=0.2, \n                sat_shift_limit=0.2, \n                val_shift_limit=0.2, \n                p=0.5\n            ),\n            albumentations.RandomBrightnessContrast(\n                brightness_limit=(-0.1,0.1), \n                contrast_limit=(-0.1, 0.1), \n                p=0.5\n            ),\n            albumentations.CoarseDropout(p=0.5),\n            albumentations.Cutout(p=0.5)\n])\n\ndef get_valid_aug(size): \n    return albumentations.Compose([\n        albumentations.Resize(size, size),\n        albumentations.CenterCrop(size, size, p=1.),\n], p=1.)\n\ndef get_x(row): return images_path/row['image_id']\ndef get_y(row): return row['label']","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.029305,"end_time":"2021-01-21T12:46:15.325979","exception":false,"start_time":"2021-01-21T12:46:15.296674","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def train(dls, fold):\n    \n    #model = EfficientNet.from_pretrained(cfg.arch, num_classes=5)\n    \"\"\"\n    model = models.resnext50_32x4d(pretrained=True)\n    n_features = model.fc.in_features\n    model.fc = nn.Linear(n_features, 5)\n    \"\"\"\n    model = pretrainedmodels.__dict__['se_resnext50_32x4d'](pretrained='imagenet')\n    model.avg_pool = nn.AdaptiveAvgPool2d((1,1))\n    model.last_linear = nn.Linear(2048,5,bias=True)\n    \n    \n    fold_name = f'model-{fold}'\n\n    # define learners\n    t_learn = Learner(\n        dls=dls,\n        model=model,\n        opt_func=ranger,\n        metrics=accuracy,\n        loss_func=LabelSmoothingCrossEntropy(),\n        cbs=[CutMix()]\n    ).to_fp16()\n    \n    s_learn = Learner(\n        dls=dls,\n        model = model,\n        opt_func=ranger,\n        metrics=accuracy,\n        loss_func=LabelSmoothingCrossEntropy(),\n        cbs=[\n            WandbCallback(log_preds=False, log_model=True, n_preds=2),\n            SaveModelCallback(\n                monitor='accuracy',\n                fname=fold_name,\n                with_opt=True,\n                every_epoch=False\n            ),\n            CutMix(),\n        ]\n    ).to_fp16()\n    \n    lr = 0.001\n    \n    # teacher model training\n    if not cfg.testing:\n        lr_min, lr_steep = t_learn.lr_find(show_plot=False)\n        lr = round(lr_min, 5)\n        print(f'found lr of({lr_min}): {round(lr_min, 5)}')\n    \n\n    t_learn.fine_tune(\n        cfg.epochs,\n        base_lr=lr,\n        freeze_epochs=cfg.f_epochs,\n    )\n    \n    \n    # student model training\n    if not cfg.testing:\n        lr_min, lr_steep = s_learn.lr_find(show_plot=False)\n        lr = round(lr_min, 5)\n        print(f'found lr of({lr_min}): {round(lr_min,5)}')\n        \n    s_learn.fine_tune(\n        cfg.epochs,\n        base_lr=lr,\n        freeze_epochs=cfg.f_epochs,\n        cbs = [distillation.KnowledgeDistillation(s_learn,t_learn)]\n    )\n    \n    s_learn.load(fold_name) # load the best .pth\n    s_learn.export(fold_name + '.pkl') # export as .pkl\n    \n    return s_learn","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.011518,"end_time":"2021-01-21T12:46:15.385793","exception":false,"start_time":"2021-01-21T12:46:15.374275","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Training\ntrain models on different folds of our data"},{"metadata":{"trusted":true},"cell_type":"code","source":"for fold in cfg.train_folds:\n    val_index = folds_df[fold].to_numpy()\n    \n    print(f'started training on {fold}')\n    \n    train_block = DataBlock(\n        blocks=(ImageBlock, CategoryBlock),\n        get_x=get_x,\n        get_y=get_y,\n        splitter=IndexSplitter(val_index),\n        item_tfms= [\n            AlbumentationsTransform(\n                get_train_aug(size=cfg.image_size),\n                get_valid_aug(size=cfg.image_size)\n            )\n        ],\n        batch_tfms=[Normalize.from_stats(*imagenet_stats)]\n    )\n\n    dls = train_block.dataloaders(full_df, bs=cfg.batch_size)\n    learn = train(dls, fold)\n\nprint(f'training on {cfg.train_folds} done')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}