{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport sys\nimport math\nimport time\nimport copy\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom pathlib import Path\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split, StratifiedShuffleSplit\nPath.ls = lambda x: list(x.iterdir())\n\nimport albumentations\nfrom albumentations.pytorch import ToTensor, ToTensorV2\n\nimport torch\nfrom torch import nn, optim\nfrom torchvision import transforms, models\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n#efficientnet from https://github.com/lukemelas/EfficientNet-PyTorch\neffnet_path = '../input/efficientnet/EfficientNet-PyTorch/'\nsys.path.append(effnet_path)\nfrom efficientnet_pytorch import EfficientNet","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"markdown","source":"# Overview\n\n### This notebook highly leverages https://www.kaggle.com/moeinshariatnia/pytorch-better-normalization-onecycle-lr-train\n### The above notebook was used for core training logic. I wanted to explore rapidly iterating on training configs\n\n### Next steps of this exploration would be to bring the optimization method and learning rate scheduler into the experiment"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define Your Experiment\n\n\n# In order to speed up training we'll use a percent of the initial training set\nrapid_train_data_percent = 0.1 #IE: 10%\nbase_configs = {\n    'num_classes': 5,\n    'crop_height': 256,\n    'crop_width': 256,\n    'horiz_flip':0.5,\n    'rotate':0.5,\n    'hue':0.2,\n    'sat':0.2,\n    'val':0.2,\n    'hue_sat_val_p':0.5,\n    'brightness_limit_min':-0.1,\n    'brightness_limit_max':0.1,\n    'contrast_limit_min':-0.1,\n    'contrast_limit_max':0.1,\n    'brightness_contrast_p':0.5,\n    'coarse_dropout':0.5,\n    'cutout':0.5,\n    'batch_size': 32,\n    'dropout': 0.2,\n    'epochs':7,\n    'max_lr':1e-3,\n    'pct_start':0.25\n\n}\n\n# We do a run with base configs, so only supply experiment_registry values which differ from base_configs\nexperiment_registry = {\n    'dropout': [0.3, 0.1],\n    'pct_start': [0.5, 0.1]\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## The following is a larger code block for the core training logic. See linked notebook to learn much more about this logic"},{"metadata":{"trusted":true},"cell_type":"code","source":"path = Path(\"../input/cassava-leaf-disease-classification\")\ndf_path = path/\"train.csv\"\ntrain_path = path/\"train_images\"\ntrain_fnames = train_path.ls()\n\ndf = pd.read_csv(df_path)\nnum_classes = df['label'].nunique() # number of clases (5 in our case)\n\n\ndf.reset_index(inplace=True, drop=True)\nsss = StratifiedShuffleSplit(n_splits=2, test_size=0.2, random_state=42)\nfor train_idx, val_idx in sss.split(X=df, y=df['label']):\n    train_df_pre = df.loc[train_idx]\n    val_df_pre = df.loc[val_idx]\n\ntrain_df = train_df_pre.iloc[::int(rapid_train_data_percent*100), :]\nval_df = val_df_pre.iloc[::int(rapid_train_data_percent*100), :]\n\n\n\nmean = [0.4589, 0.5314, 0.3236]\nstd = [0.2272, 0.2297, 0.2200]\n\n\ndef accuracy(preds, target):\n    preds = preds.argmax(dim=1)\n    return (preds == target).float().mean()\n\ndef one_epoch(model, dl, loss_func, opt=None, lr_schedule=None):\n    running_loss = 0.\n    running_acc = 0\n    lrs = []\n\n\n    for xb, yb in tqdm(dl):\n        xb, yb = xb.to(device), yb.to(device)\n        preds = model(xb)\n        loss = loss_func(preds, yb)\n\n        if opt is not None:\n            opt.zero_grad()\n            loss.backward()\n            opt.step()\n            lrs.append(opt.param_groups[0][\"lr\"])\n            if lr_schedule is not None:\n                lr_schedule.step()\n\n        running_acc += accuracy(preds, yb).item()\n        running_loss += loss.item()\n\n    return running_loss / len(dl), running_acc / len(dl), lrs\n\n\ndef get_lr(opt):\n    for param_group in opt.param_groups:\n        return param_group['lr']\n\ndef train_val(model, params):\n\n    num_epochs = params[\"num_epochs\"]\n    loss_func = params[\"loss_func\"]\n    opt = params[\"optimizer\"]\n    train_dl = params[\"train_dl\"]\n    val_dl = params[\"val_dl\"]\n    lr_scheduler = params[\"lr_scheduler\"]\n    path2weights = params[\"path2weights\"]\n    one_cycle = params[\"one_cycle\"]\n\n    loss_history = {\n        \"train\": [],\n        \"val\": [],\n    }\n\n    metric_history = {\n        \"train\": [],\n        \"val\": [],\n    }\n\n    lrs_by_epoch = []\n\n    best_model_wts = copy.deepcopy(model.state_dict())\n\n    best_loss=float('inf')\n\n    for epoch in range(num_epochs):\n        start = time.time()\n        current_lr = get_lr(opt)\n        print(f'Epoch {epoch + 1}/{num_epochs}, current lr = {current_lr:5f}')\n\n        model.train()\n        train_loss, train_metric, lrs = one_epoch(model, train_dl, loss_func, opt, lr_scheduler if one_cycle else None)\n        lrs_by_epoch.append({epoch: lrs})\n\n        loss_history[\"train\"].append(train_loss)\n        metric_history[\"train\"].append(train_metric)\n\n        model.eval()\n        with torch.no_grad():\n            val_loss, val_metric, lrs2 = one_epoch(model, val_dl, loss_func, opt=None)\n\n\n        if val_loss < best_loss:\n            best_loss = val_loss\n            best_model_wts = copy.deepcopy(model.state_dict())\n            torch.save(model.state_dict(), path2weights)\n            print(\"Copied best model weights!\")\n\n        loss_history[\"val\"].append(val_loss)\n        metric_history[\"val\"].append(val_metric)\n\n        if not one_cycle:\n            lr_scheduler.step(val_loss)\n            if current_lr != get_lr(opt):\n                print(\"Loading best model weights!\")\n                model.load_state_dict(best_model_wts)\n\n        print(f\"Train Loss: {train_loss:.4f}, Val Loss: {val_loss:.4f}\\n\"\n              f\"Train Acc: {train_metric:.4f}, Val Acc: {val_metric:.4f}\\n\"\n              f\"Completed in {time.time() - start:.3f}\")\n\n        print(\"-\"*10)\n\n    model.load_state_dict(best_model_wts)\n\n    return model, loss_history, metric_history, lrs_by_epoch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## We wrap the actual training piece in a function so we can iterate our experiment over it, but the logic remains mostly the same from the source notebook"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_cassava_model(base_configs):\n    train_tfms = albumentations.Compose([\n                albumentations.RandomResizedCrop(base_configs['crop_height'], base_configs['crop_width']),\n                albumentations.HorizontalFlip(p=base_configs['horiz_flip']),\n                albumentations.ShiftScaleRotate(p=base_configs['rotate']),\n                albumentations.HueSaturationValue(\n                    hue_shift_limit=base_configs['hue'],\n                    sat_shift_limit=base_configs['sat'],\n                    val_shift_limit=base_configs['val'],\n                    p=base_configs['hue_sat_val_p']\n                ),\n                albumentations.RandomBrightnessContrast(\n                    brightness_limit=(base_configs['brightness_limit_min'],base_configs['brightness_limit_max']),\n                    contrast_limit=(base_configs['contrast_limit_min'], base_configs['contrast_limit_max']),\n                    p=base_configs['brightness_contrast_p']\n                ),\n                albumentations.Normalize(\n                    mean=mean,\n                    std=std,\n                    max_pixel_value=255.0,\n                    p=1.0\n                ),\n                albumentations.CoarseDropout(p=base_configs['coarse_dropout']),\n                albumentations.Cutout(p=base_configs['cutout']),\n                ToTensorV2()], p=1.)\n\n\n    valid_tfms = albumentations.Compose([\n                albumentations.CenterCrop(256, 256, p=1.),\n                albumentations.Resize(256, 256),\n                albumentations.Normalize(\n                    mean=mean,\n                    std=std,\n                    max_pixel_value=255.0,\n                    p=1.0\n                ),\n                ToTensorV2()], p=1.)\n\n    class LeafData(Dataset):\n        def __init__(self, df, split=\"train\"):\n            if split == \"train\":\n                self.transforms = train_tfms\n            elif split == \"val\":\n                self.transforms = valid_tfms\n\n            self.paths = [train_path/id_ for id_ in df['image_id'].values]\n            self.labels = df['label'].values\n\n        def __getitem__(self, idx):\n            img = cv2.imread(str(self.paths[idx]))[..., ::-1] # ::-1 is here because cv2 loads the images in BGR rather than RGB\n            img = self.transforms(image=img)['image']\n            label = self.labels[idx]\n\n            return img, label\n\n        def __len__(self):\n            return len(self.paths)\n\n    def make_dataloaders(batch_size=base_configs['batch_size'], num_workers=4, pin_memory=True, **kwargs):\n        dataset = LeafData(**kwargs)\n        dataloader = DataLoader(dataset, batch_size=batch_size, num_workers=num_workers,\n                                pin_memory=pin_memory, shuffle=True if kwargs['split'] == \"train\" else False)\n        return dataloader\n\n\n\n    train_dl = make_dataloaders(df=train_df, split=\"train\")\n    val_dl = make_dataloaders(df=val_df, split=\"val\")\n    xb, yb = next(iter(train_dl))\n    xb.shape, yb.shape, xb.mean(dim=(0, 2, 3)), xb.std(dim=(0, 2, 3))\n\n\n\n\n\n    class EfficientNetModel(nn.Module):\n        def __init__(self, arch=\"b4\", dropout=base_configs['dropout'], n_out=5,\n                     pretrained=True, freeze=True):\n            super().__init__()\n            if pretrained:\n                self.model = EfficientNet.from_pretrained(f\"efficientnet-{arch}\")\n                if freeze:\n                    for p in self.model.parameters():\n                        p.requires_grad = False\n            else:\n                self.model = EfficientNet.from_name(f\"efficientnet-{arch}\")\n\n            self.lin1 = nn.Linear(1792 * 2, 512) # 1792 is the final output shape of the efficientnet backbone.\n            self.lin2 = nn.Linear(512, n_out)    # I'm multiplying by two because we are concatenating the avg pool\n            self.bn1 = nn.BatchNorm1d(1792 * 2)  # and max pool layers.\n            self.bn2 = nn.BatchNorm1d(512)\n            self.dropout = dropout\n\n        def forward(self, x):\n            x = self.model.extract_features(x)\n            avg = F.adaptive_avg_pool2d(x, 1)\n            max_ = F.adaptive_max_pool2d(x, 1)\n            cat = torch.cat((avg.squeeze(), max_.squeeze()), dim=1)\n            x = self.bn1(cat)\n            x = F.dropout(x, self.dropout)\n            x = F.relu(self.bn2(self.lin1(x)))\n            x = self.lin2(x)\n            return x\n\n\n    model = EfficientNetModel(pretrained=True, freeze=False,\n                              arch=\"b4\", n_out=num_classes, dropout=base_configs['dropout']).to(device) # I'm using pretrained weights but not freezing the backbone\n\n    criterion = nn.CrossEntropyLoss()\n    opt = optim.Adam(model.parameters())\n    epochs = base_configs['epochs']\n    lr_sch = optim.lr_scheduler.OneCycleLR(opt, max_lr=base_configs['max_lr'], epochs=epochs,\n                                           steps_per_epoch=len(train_dl), pct_start=base_configs['pct_start'],)\n\n    params_train = {\n     \"num_epochs\": epochs,\n     \"optimizer\": opt,\n     \"loss_func\": criterion,\n     \"train_dl\": train_dl,\n     \"val_dl\": val_dl,\n     \"lr_scheduler\": lr_sch,\n     \"path2weights\": \"/kaggle/working/effnet.pt\",\n     \"one_cycle\": True\n    }\n\n    model, loss_hist, metric_hist, lrs_by_epoch = train_val(model, params_train)\n    return model, loss_hist, metric_hist, lrs_by_epoch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Basic functions to plot performance and basic model evaluation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_model_performance(results_dict, experiment_registry):\n    for exp_param in results_dict.keys():\n        registry = experiment_registry.copy()\n        if exp_param == 'base':\n            registry = ['base']\n        else:\n            registry = experiment_registry[exp_param]\n        for exp_setting in registry:\n\n\n            experiment = exp_param + ': ' + str(exp_setting)\n            print('Model Performance With Base Configs and Experimentation: {}'.format(experiment))\n\n            epochs = results_dict[exp_param][exp_setting]['configs']['epochs']\n            loss_hist = results_dict[exp_param][exp_setting]['loss_hist']\n            metric_hist = results_dict[exp_param][exp_setting]['metric_hist']\n            lrs_by_epoch = results_dict[exp_param][exp_setting]['lrs_by_epoch']\n\n            x_axis = list(range(0, epochs))\n            legend_keys = []\n            for i in loss_hist:\n                plt.plot(x_axis, loss_hist[i])\n                legend_keys.append(i + '_loss')\n\n            for i in metric_hist:\n                plt.plot(x_axis, metric_hist[i])\n                legend_keys.append(i + '_accuracy')\n            plt.legend(legend_keys, loc='upper left')\n            plt.show()\n\n            epoch = 0\n            num_steps = 54 # see if this changes\n            x_axis = []\n            rates = []\n            for i in lrs_by_epoch:\n                cur_list = i[epoch]\n                for index in range(0, len(cur_list)):\n                    rates.append(cur_list[index])\n                    x_axis.append(epoch + round(index/num_steps,2))\n                epoch += 1\n            plt.plot(x_axis, rates)\n            plt.legend(['Learing Rate'], loc='upper left')\n            plt.show()\n\n\ndef evaluate_best_model(results_dict, experiment_registry):\n    best_model = ''\n    best_performance = 100\n\n    for exp_param in results_dict.keys():\n        registry = experiment_registry.copy()\n        if exp_param == 'base':\n            registry = ['base']\n        else:\n            registry = experiment_registry[exp_param]\n        for exp_setting in registry:\n\n            experiment = exp_param + ': ' + str(exp_setting)\n\n            val_loss = results_dict[exp_param][exp_setting]['loss_hist']['val'][-1:][0]\n            val_accuracy = results_dict[exp_param][exp_setting]['metric_hist']['val'][-1:][0]\n\n            performance = (val_loss - 0) + (1-val_accuracy)\n\n            if best_performance > performance:\n                best_model = experiment\n    return best_model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Run our experiment"},{"metadata":{"trusted":true},"cell_type":"code","source":"results_dict = {}\n\nmodel, loss_hist, metric_hist, lrs_by_epoch = train_cassava_model(base_configs)\nresults_dict['base'] = {}\nresults_dict['base']['base'] = {}\nresults_dict['base']['base']['loss_hist'] = loss_hist\nresults_dict['base']['base']['metric_hist'] = metric_hist\nresults_dict['base']['base']['lrs_by_epoch'] = lrs_by_epoch\nresults_dict['base']['base']['configs'] = base_configs\n\nfor i in experiment_registry.keys():\n    results_dict[i] = {}\n    for y in experiment_registry[i]:\n        results_dict[i][y] = {}\n\n        adjusted_configs = base_configs.copy()\n        adjusted_configs[i] = y\n        print('Configs:')\n        print(adjusted_configs)\n\n        model, loss_hist, metric_hist, lrs_by_epoch = train_cassava_model(adjusted_configs)\n        results_dict[i][y]['loss_hist'] = loss_hist\n        results_dict[i][y]['metric_hist'] = metric_hist\n        results_dict[i][y]['lrs_by_epoch'] = lrs_by_epoch\n        results_dict[i][y]['configs'] = adjusted_configs\n\n\nplot_model_performance(results_dict, experiment_registry)\nbest_model = evaluate_best_model(results_dict, experiment_registry)\nprint('Basic Model Evaliation Shows Top Performing Experiment Config: {}'.format(best_model))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}