{"cells":[{"metadata":{"executionInfo":{"elapsed":188635,"status":"ok","timestamp":1612803061811,"user":{"displayName":"Tai Le","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWZbmr6Qd_9_pd7I0T1mzatkMurbnfV4t0IuDt=s64","userId":"01228406727075634515"},"user_tz":-420},"id":"e_bxve75I9f0","outputId":"72366c23-0b71-4c09-fe58-d889b540eeb3","trusted":true},"cell_type":"code","source":"# import os\n# from google.colab import drive\n\n# os.mkdir(\"/content/input\")\n# os.chdir(\"/content/input\")\n\n# drive.mount(\"/content/drive\")\n\n# !cp \"/content/drive/MyDrive/Colab Notebooks/datasets/cassava-leaf-disease-classification.zip\" .\n# !unzip cassava-leaf-disease-classification.zip","execution_count":null,"outputs":[]},{"metadata":{"executionInfo":{"elapsed":197719,"status":"ok","timestamp":1612803070903,"user":{"displayName":"Tai Le","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWZbmr6Qd_9_pd7I0T1mzatkMurbnfV4t0IuDt=s64","userId":"01228406727075634515"},"user_tz":-420},"id":"smooth-transition","trusted":true},"cell_type":"code","source":"import os\nimport torch\nimport torch.nn as nn\nimport torchvision\nimport torchvision.transforms as transforms\nimport cv2\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport json\nimport numpy as np\nimport PIL.Image as Image\nimport time\nimport seaborn as sns\n\nfrom sklearn.metrics import confusion_matrix\nfrom torch.utils.data import DataLoader, Dataset\nfrom sklearn.model_selection import train_test_split, StratifiedKFold","execution_count":null,"outputs":[]},{"metadata":{"id":"willing-newark"},"cell_type":"markdown","source":"### Steps:\n- Build dataset loader\n- Visualize some transformed images\n- Define model + choose pretrained backbone\n- Define loss + optimization function\n- Train\n- Test\n\n### Notes:\n- Just need 15 epochs for an experiment\n- In an experiment, we can:\n+ Change initialization methods\n+ Change backbone\n+ Change augmentation\n+ Apply variet methods to prevent overfitting"},{"metadata":{"executionInfo":{"elapsed":197668,"status":"ok","timestamp":1612803070909,"user":{"displayName":"Tai Le","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWZbmr6Qd_9_pd7I0T1mzatkMurbnfV4t0IuDt=s64","userId":"01228406727075634515"},"user_tz":-420},"id":"otherwise-buying","outputId":"a15d68a1-4163-4115-abf5-a65a0e0f7ac8","trusted":true},"cell_type":"code","source":"df = pd.read_csv(\"../input/cassava-leaf-disease-classification/train.csv\")\ndf.head(5)","execution_count":null,"outputs":[]},{"metadata":{"executionInfo":{"elapsed":197667,"status":"ok","timestamp":1612803070910,"user":{"displayName":"Tai Le","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWZbmr6Qd_9_pd7I0T1mzatkMurbnfV4t0IuDt=s64","userId":"01228406727075634515"},"user_tz":-420},"id":"KjquMh2gI5d2","trusted":true},"cell_type":"code","source":"ignored_images = []\n\ndf = df.loc[~df[\"image_id\"].isin(ignored_images)]","execution_count":null,"outputs":[]},{"metadata":{"executionInfo":{"elapsed":197661,"status":"ok","timestamp":1612803070911,"user":{"displayName":"Tai Le","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWZbmr6Qd_9_pd7I0T1mzatkMurbnfV4t0IuDt=s64","userId":"01228406727075634515"},"user_tz":-420},"id":"hungry-harbor","outputId":"22c14923-5ddb-4c37-a6af-2a2aba86fdff","trusted":true},"cell_type":"code","source":"labels = open(\"../input/cassava-leaf-disease-classification/label_num_to_disease_map.json\", \"r\").read()\nlabels = json.loads(labels)\nlabels_name = list(labels.values())\nlabels","execution_count":null,"outputs":[]},{"metadata":{"executionInfo":{"elapsed":197659,"status":"ok","timestamp":1612803070911,"user":{"displayName":"Tai Le","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWZbmr6Qd_9_pd7I0T1mzatkMurbnfV4t0IuDt=s64","userId":"01228406727075634515"},"user_tz":-420},"id":"understood-union","trusted":true},"cell_type":"code","source":"shortcut_labels = [label.split(\" \")[-1] for label in labels.values()]\nlabels_value = df[\"label\"].value_counts().sort_index().values","execution_count":null,"outputs":[]},{"metadata":{"executionInfo":{"elapsed":197653,"status":"ok","timestamp":1612803070912,"user":{"displayName":"Tai Le","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWZbmr6Qd_9_pd7I0T1mzatkMurbnfV4t0IuDt=s64","userId":"01228406727075634515"},"user_tz":-420},"id":"diagnostic-phone","outputId":"f0da42ef-7f80-437d-9af4-7053c0ddc588","trusted":true},"cell_type":"code","source":"indices = np.arange(len(shortcut_labels))\nplt.bar(indices, labels_value)\nplt.xticks(indices, shortcut_labels)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"executionInfo":{"elapsed":197651,"status":"ok","timestamp":1612803070912,"user":{"displayName":"Tai Le","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWZbmr6Qd_9_pd7I0T1mzatkMurbnfV4t0IuDt=s64","userId":"01228406727075634515"},"user_tz":-420},"id":"occasional-business","trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, path, transform=None):\n        self.df = df\n        self.path = path\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.df.index)\n    \n    def __getitem__(self, i):\n        row = self.df.iloc[i]\n        image_id = row[\"image_id\"]\n        label = row[\"label\"] if \"label\" in row else 0\n        \n        img = cv2.imread(f\"{self.path}/{image_id}\")\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        if self.transform:\n            img = self.transform(image=img)[\"image\"]\n            \n        return img, label, image_id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PARAMS = {\n    \"TRAINING_BATCH_SIZE\": 8,\n    \"VAL_BATCH_SIZE\": 2,\n    \"IMG_SIZE\": 512,\n    \"NUM_WORKERS\": 4,\n    \"PIN_MEMORY\": True,\n    \"EPOCHS\": 15,\n    \"T_0\": 10,\n    \"LEARNING_RATE\": 1e-4,\n    \"MIN_LEARNING_RATE\": 1e-6,\n    \"MODEL\": \"tf_efficientnet_b4_ns\",\n    \"WEIGHT_DECAY\": 0.0001,\n    \"LS_EPSILON\": 0.2,\n    \"EFF_SKIPPED_LAYERS\": 0,\n    \"RANDOM_SEED\": 719,\n    \"TOTAL_FOLDS\": 5\n}","execution_count":null,"outputs":[]},{"metadata":{"executionInfo":{"elapsed":206739,"status":"ok","timestamp":1612803080010,"user":{"displayName":"Tai Le","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWZbmr6Qd_9_pd7I0T1mzatkMurbnfV4t0IuDt=s64","userId":"01228406727075634515"},"user_tz":-420},"id":"private-forestry","trusted":true},"cell_type":"code","source":"from albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\nfrom albumentations.pytorch import ToTensorV2\n\n# define config here\nPIN_MEMORY = True\ndataset_path = \"../input/cassava-leaf-disease-classification/train_images\"\n\n# TODO: modify a little more\ntraining_transform = Compose([\n#     Resize(PARAMS[\"IMG_SIZE\"], PARAMS[\"IMG_SIZE\"]),\n    RandomResizedCrop(PARAMS[\"IMG_SIZE\"], PARAMS[\"IMG_SIZE\"]),\n    Transpose(p=0.5),\n    HorizontalFlip(p=0.5),\n    VerticalFlip(p=0.5),\n#     ShiftScaleRotate(p=0.5),\n#     HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n#     RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n    Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n    CoarseDropout(p=0.5),\n#     Cutout(p=0.5),\n    ToTensorV2(p=1.0),\n], p=1.)\n\nval_transform = Compose([\n    CenterCrop(PARAMS[\"IMG_SIZE\"], PARAMS[\"IMG_SIZE\"], p=1.),\n    Resize(PARAMS[\"IMG_SIZE\"], PARAMS[\"IMG_SIZE\"]),\n    Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n    ToTensorV2(p=1.0),\n], p=1.)","execution_count":null,"outputs":[]},{"metadata":{"executionInfo":{"elapsed":206742,"status":"ok","timestamp":1612803080015,"user":{"displayName":"Tai Le","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWZbmr6Qd_9_pd7I0T1mzatkMurbnfV4t0IuDt=s64","userId":"01228406727075634515"},"user_tz":-420},"id":"further-operations","trusted":true},"cell_type":"code","source":"training_set, val_set = train_test_split(df, test_size=0.2, random_state=PARAMS[\"RANDOM_SEED\"])\n\ntraining_set = CassavaDataset(training_set, dataset_path, training_transform)\ntraining_loader = DataLoader(\n    dataset=training_set,\n    batch_size=PARAMS[\"TRAINING_BATCH_SIZE\"],\n    num_workers=PARAMS[\"NUM_WORKERS\"],\n    pin_memory=PARAMS[\"PIN_MEMORY\"],\n    shuffle=True,\n    drop_last=True\n)\n\nval_set = CassavaDataset(val_set, dataset_path, val_transform)\nval_loader = DataLoader(\n    dataset=val_set,\n    batch_size=PARAMS[\"VAL_BATCH_SIZE\"],\n    num_workers=PARAMS[\"NUM_WORKERS\"],\n    pin_memory=PARAMS[\"PIN_MEMORY\"],\n    shuffle=False,\n    drop_last=True\n)","execution_count":null,"outputs":[]},{"metadata":{"executionInfo":{"elapsed":221769,"status":"ok","timestamp":1612803095049,"user":{"displayName":"Tai Le","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWZbmr6Qd_9_pd7I0T1mzatkMurbnfV4t0IuDt=s64","userId":"01228406727075634515"},"user_tz":-420},"id":"standing-difference","outputId":"57a0be23-bcd9-45a8-b1bf-ddc8ff72a42d","trusted":true},"cell_type":"code","source":"nrows = 2\nncols = 5\nfig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(22, 8))\n\nfor X, Y, _ in training_loader:\n    for i, x in enumerate(X[:(nrows * ncols)]):\n        label = labels[str(Y[i].item())]\n\n        row = int(i / ncols)\n        col = i % ncols\n        ax[row][col].imshow(x.permute(1, 2, 0)) # permute and denormalize\n        ax[row][col].set_title(label)\n        \n    break","execution_count":null,"outputs":[]},{"metadata":{"executionInfo":{"elapsed":221763,"status":"ok","timestamp":1612803095050,"user":{"displayName":"Tai Le","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWZbmr6Qd_9_pd7I0T1mzatkMurbnfV4t0IuDt=s64","userId":"01228406727075634515"},"user_tz":-420},"id":"regular-thousand","outputId":"ab2004be-7b3d-4278-a79e-ca1df96e7689","trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.device_count() > 0 else \"cpu\")\ndevice","execution_count":null,"outputs":[]},{"metadata":{"executionInfo":{"elapsed":222914,"status":"ok","timestamp":1612803096210,"user":{"displayName":"Tai Le","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWZbmr6Qd_9_pd7I0T1mzatkMurbnfV4t0IuDt=s64","userId":"01228406727075634515"},"user_tz":-420},"id":"proved-colleague","outputId":"082d2bae-b5f5-40e0-b498-2f3a283ae5bf","trusted":true},"cell_type":"code","source":"# freeze layers\n# def freeze(model):\n#     # To freeze the residual layers\n#     for param in model.parameters():\n#         param.requires_grad = False\n\n#     for param in model.fc.parameters():\n#         param.requires_grad = True\n\n#     return model\n\n# def unfreeze(model):\n#     # Unfreeze all layers\n#     for param in model.parameters():\n#         param.requires_grad = True\n\n#     return model\n\n# resnext50\n# finetuned_net = torchvision.models.resnext50_32x4d(pretrained=True)\n# finetuned_net.fc = nn.Linear(finetuned_net.fc.in_features, len(labels_name))\n\n# for param in finetuned_net.layer1.parameters():\n#     param.requires_grad = False\n# for param in finetuned_net.layer2.parameters():\n#     param.requires_grad = False\n# for param in finetuned_net.layer3.parameters():\n#     param.requires_grad = False\n# for param in finetuned_net.layer4.parameters():\n#     param.requires_grad = True\n\n!pip install timm\n# efficientnet b4\nimport timm\n\ndef load_model():\n    finetuned_net = timm.create_model(PARAMS[\"MODEL\"], pretrained=True)\n    finetuned_net.classifier = nn.Linear(finetuned_net.classifier.in_features, len(labels_name))\n\n    for block in finetuned_net.blocks[:PARAMS[\"EFF_SKIPPED_LAYERS\"]]:\n        for param in block.parameters():\n            param.requires_grad = False\n\n    return finetuned_net\n# TODO: Try different types of initilization","execution_count":null,"outputs":[]},{"metadata":{"executionInfo":{"elapsed":222915,"status":"ok","timestamp":1612803096213,"user":{"displayName":"Tai Le","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWZbmr6Qd_9_pd7I0T1mzatkMurbnfV4t0IuDt=s64","userId":"01228406727075634515"},"user_tz":-420},"id":"UmKq3xYgI5d7","trusted":true},"cell_type":"code","source":"class EarlyStopping:\n    \"\"\"Early stops the training if validation loss doesn't improve after a given patience.\"\"\"\n    def __init__(self, patience=7, verbose=False, delta=0, path='checkpoint.pt', trace_func=print):\n        \"\"\"\n        Args:\n            patience (int): How long to wait after last time validation loss improved.\n                            Default: 7\n            verbose (bool): If True, prints a message for each validation loss improvement. \n                            Default: False\n            delta (float): Minimum change in the monitored quantity to qualify as an improvement.\n                            Default: 0\n            path (str): Path for the checkpoint to be saved to.\n                            Default: 'checkpoint.pt'\n            trace_func (function): trace print function.\n                            Default: print            \n        \"\"\"\n        self.patience = patience\n        self.verbose = verbose\n        self.counter = 0\n        self.best_score = None\n        self.early_stop = False\n        self.val_loss_min = np.Inf\n        self.delta = delta\n        self.path = path\n        self.trace_func = trace_func\n        \n    def __call__(self, val_loss, model):\n        score = -val_loss\n\n        if self.best_score is None:\n            self.best_score = score\n            self.save_checkpoint(val_loss, model)\n        elif score < self.best_score + self.delta:\n            self.counter += 1\n            self.trace_func(f'EarlyStopping counter: {self.counter} out of {self.patience}')\n            if self.counter >= self.patience:\n                self.early_stop = True\n        else:\n            self.best_score = score\n            self.save_checkpoint(val_loss, model)\n            self.counter = 0\n\n    def save_checkpoint(self, val_loss, model):\n        '''Saves model when validation loss decrease.'''\n        if self.verbose:\n            self.trace_func(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).  Saving model ...')\n        torch.save(model.state_dict(), self.path)\n        self.val_loss_min = val_loss","execution_count":null,"outputs":[]},{"metadata":{"executionInfo":{"elapsed":222915,"status":"ok","timestamp":1612803096215,"user":{"displayName":"Tai Le","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWZbmr6Qd_9_pd7I0T1mzatkMurbnfV4t0IuDt=s64","userId":"01228406727075634515"},"user_tz":-420},"id":"8keEryvdNs_u","trusted":true},"cell_type":"code","source":"import torch.nn.functional as F\n\n# idea: https://amaarora.github.io/2020/07/18/label-smoothing.html\n# from fastai\ndef linear_combination(x, y, epsilon):\n    return epsilon * x + (1 - epsilon) * y\n\n\ndef reduce_loss(loss, reduction='mean'):\n    return loss.mean() if reduction == 'mean' else loss.sum() if reduction == 'sum' else loss\n\n\nclass LabelSmoothingCrossEntropy(nn.Module):\n    def __init__(self, epsilon: float = 0.1, reduction='mean'):\n        super().__init__()\n        self.epsilon = epsilon\n        self.reduction = reduction\n\n    def forward(self, preds, target):\n        n = preds.size()[-1]\n        log_preds = F.log_softmax(preds, dim=-1)\n        loss = reduce_loss(-log_preds.sum(dim=-1), self.reduction)\n        nll = F.nll_loss(log_preds, target, reduction=self.reduction)\n        \n        return linear_combination(loss / n, nll, self.epsilon)\n    \n    \ndef split_df_into_folds(df, n_splits=5):\n    splits = []\n    splitter = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=PARAMS[\"RANDOM_SEED\"])\n    \n    for training_index, val_index in splitter.split(df[\"image_id\"], df[\"label\"]):\n        training_set = pd.DataFrame({\"image_id\": df[\"image_id\"][training_index], \"label\": df[\"label\"][training_index]})\n        training_set = CassavaDataset(training_set, dataset_path, training_transform)\n        training_loader = DataLoader(\n            dataset=training_set,\n            batch_size=PARAMS[\"TRAINING_BATCH_SIZE\"],\n            num_workers=PARAMS[\"NUM_WORKERS\"],\n            pin_memory=PARAMS[\"PIN_MEMORY\"],\n            shuffle=True,\n            drop_last=True\n        )\n\n        val_set = pd.DataFrame({\"image_id\": df[\"image_id\"][val_index], \"label\": df[\"label\"][val_index]})\n        val_set = CassavaDataset(val_set, dataset_path, val_transform)\n        val_loader = DataLoader(\n            dataset=val_set,\n            batch_size=PARAMS[\"VAL_BATCH_SIZE\"],\n            num_workers=PARAMS[\"NUM_WORKERS\"],\n            pin_memory=PARAMS[\"PIN_MEMORY\"],\n            shuffle=False,\n            drop_last=True\n        )\n\n        splits.append([training_loader, val_loader])\n\n    return splits","execution_count":null,"outputs":[]},{"metadata":{"id":"exotic-basketball","outputId":"5ee84237-1f78-4bf0-ad87-7dd2778c3057","trusted":true},"cell_type":"code","source":"def calculate_accurate_percent(outputs, labels):\n    return torch.sum(torch.argmax(outputs, dim=1) == labels.data), len(outputs)\n\n\ndef get_learning_rate(optimizer):\n    for param_group in optimizer.param_groups:\n        return param_group['lr']\n    \n    \ndef train_one_epoch(training_loader, net, criterion, optimizer, device):\n    training_phase_loss = []\n    training_phase_correct = torch.tensor(0, device=device, dtype=torch.int)\n    training_phase_trained = torch.tensor(0, device=device, dtype=torch.int)\n\n    start_time = time.time()\n    net.train()\n    for (inputs, labels, _) in training_loader:\n        inputs = inputs.to(device)\n        labels = labels.to(device)\n\n        optimizer.zero_grad()\n\n        outputs = net(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n\n        training_phase_loss.append(loss.item())\n        # print(f\"Loss 1: {loss.item()}, loss 2: {loss2.item()}\")\n\n        correct, trained = calculate_accurate_percent(outputs, labels)\n        training_phase_correct += correct\n        training_phase_trained += trained\n\n    training_end_time = time.time() - start_time\n    training_loss_avg = sum(training_phase_loss) / len(training_loader)\n    training_accurate_percent = training_phase_correct / training_phase_trained\n\n    return training_end_time, training_loss_avg, training_accurate_percent\n\n\ndef val_one_epoch(val_loader, net, criterion, device):\n    val_phase_loss = []\n    val_phase_correct = torch.tensor(0, device=device, dtype=torch.int)\n    val_phase_trained = torch.tensor(0, device=device, dtype=torch.int)\n    val_labels = []\n    val_preds = []\n    \n    start_time = time.time()\n    net.eval()\n    for (inputs, labels, _) in val_loader:\n        inputs = inputs.to(device)\n        labels = labels.to(device)\n\n        outputs = net(inputs)\n\n        loss = criterion(outputs, labels)\n        val_phase_loss.append(loss.item())\n\n        correct, trained = calculate_accurate_percent(outputs, labels)\n        val_phase_correct += correct\n        val_phase_trained += trained\n        \n        val_labels.extend(labels)\n        val_preds.extend(torch.argmax(outputs, dim=1))\n\n    val_end_time = time.time() - start_time\n    val_loss_avg = sum(val_phase_loss) / len(val_loader)\n    val_accurate_percent = val_phase_correct / val_phase_trained\n    \n    val_labels = [i.cpu() for i in val_labels]\n    val_preds = [i.cpu() for i in val_preds]\n    \n    return val_end_time, val_loss_avg, val_accurate_percent, val_labels, val_preds\n\n\ndef show_heatmap(labels, preds):\n    fig, ax = plt.subplots(figsize=(8, 8)) \n    cm = confusion_matrix(labels, preds)\n    sns.heatmap(cm, annot=True, cmap=\"Blues\", fmt=\"g\")\n\n    ax.set_xticklabels(shortcut_labels)\n    ax.set_yticklabels(shortcut_labels)\n    plt.show()\n    \n    \ndef train_finetuned_model_no_cv(training_loader, val_loader, epochs, device, use_pretrained=True, is_classifier=True):\n    net = load_model()\n    net = net.to(device)\n    \n    if use_pretrained and not is_classifier:\n        params_1x = [param for name, param in net.named_parameters() if name not in [\"fc.weight\", \"fc.bias\"]]\n        optimizer = torch.optim.Adam([\n            {'params': params_1x},\n            {'params': net.fc.parameters(), 'lr': PARAMS[\"LEARNING_RATE\"] * 10}\n        ], lr=PARAMS[\"LEARNING_RATE\"], weight_decay=PARAMS[\"WEIGHT_DECAY\"])\n    else:\n        optimizer = torch.optim.Adam(net.parameters(), lr=PARAMS[\"LEARNING_RATE\"], weight_decay=PARAMS[\"WEIGHT_DECAY\"])\n\n    scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=PARAMS[\"T_0\"], T_mult=1, eta_min=PARAMS['MIN_LEARNING_RATE'], last_epoch=-1)\n#     early_stopping = EarlyStopping(patience=3, verbose=True)\n    criterion = LabelSmoothingCrossEntropy(PARAMS[\"LS_EPSILON\"], reduction='mean')\n\n    statistic = {\n        \"lr\": [],\n        \"training_loss\": [],\n        \"training_correct_percent\": [],\n        \"val_loss\": [],\n        \"val_correct_percent\": [],\n    }\n\n    for i in range(1, PARAMS[\"EPOCHS\"] + 1):\n        training_end_time, training_loss_avg, training_accurate_percent = train_one_epoch(training_loader, net, criterion, optimizer, device)\n        statistic[\"training_loss\"].append(training_loss_avg)\n        statistic[\"training_correct_percent\"].append(training_accurate_percent)\n        \n        val_end_time, val_loss_avg, val_accurate_percent, val_labels, val_preds = val_one_epoch(val_loader, net, criterion, device)\n        statistic[\"val_loss\"].append(val_loss_avg)\n        statistic[\"val_correct_percent\"].append(val_accurate_percent)\n\n        statistic[\"lr\"].append(optimizer.param_groups[0][\"lr\"])\n        scheduler.step(val_loss_avg)\n#         early_stopping(val_loss_avg, net)\n        \n#         if early_stopping.early_stop:\n#             print(\"Early stopping\")\n#             break\n\n        # calculate accurate percentage\n        print(f\"\\nEpoch: {i}/{epochs}\")\n        print(\"Learning rate: %.8f\" % get_learning_rate(optimizer))\n        print(\"Training loss: %.4f\" % training_loss_avg)\n        print(\"Training accurate percent: %.4f\" % training_accurate_percent)\n        print(\"Training time: %.4f\" % training_end_time)\n        print(\"-------\")\n        print(\"Validation loss: %.4f\" % val_loss_avg)\n        print(\"Validation accurate percent: %.4f\" % val_accurate_percent)\n        print(\"Validation time: %.4f\" % val_end_time)\n        print(\"=\" * 50)\n        \n    print(f\"Saving model\")\n\n    torch.save(net.state_dict(), f\"model.pth\")\n    del net, criterion, optimizer, training_loader, val_loader, scheduler\n    torch.cuda.empty_cache()\n    \n    show_heatmap(val_labels, val_preds)\n        \n    return statistic\n\n\ndef train_finetuned_model_cv(df, device, use_pretrained=True, is_classifier=True):\n    # split dataframe into fols\n    total_folds = PARAMS[\"TOTAL_FOLDS\"]\n    folds = split_df_into_folds(df, total_folds)\n    \n    # train model by each fold\n    for fold_index, (training_loader, val_loader) in enumerate(folds):\n        if fold_index != 0:\n            continue\n            \n        # init model\n        net = load_model()\n        net = net.to(device)\n        \n        if use_pretrained and not is_classifier:\n            params_1x = [param for name, param in net.named_parameters() if name not in [\"fc.weight\", \"fc.bias\"]]\n            optimizer = torch.optim.Adam([\n                {'params': params_1x},\n                {'params': net.fc.parameters(), 'lr': PARAMS[\"LEARNING_RATE\"] * 10}\n            ], lr=PARAMS[\"LEARNING_RATE\"], weight_decay=PARAMS[\"WEIGHT_DECAY\"])\n        else:\n            optimizer = torch.optim.Adam(net.parameters(), lr=PARAMS[\"LEARNING_RATE\"], weight_decay=PARAMS[\"WEIGHT_DECAY\"])\n\n#         init scheduler + criterion + early_stopping + optimizer\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=PARAMS[\"T_0\"], T_mult=1, eta_min=PARAMS[\"MIN_LEARNING_RATE\"], last_epoch=-1)\n#         early_stopping = EarlyStopping(patience=5, verbose=True)\n        criterion = LabelSmoothingCrossEntropy(PARAMS[\"LS_EPSILON\"], reduction=\"mean\")\n        min_val_loss = 999\n        \n#         begin training\n        for i in range(1, PARAMS[\"EPOCHS\"] + 1):\n            training_end_time, training_loss_avg, training_accurate_percent = train_one_epoch(training_loader, net, criterion, optimizer, device)\n            \n            with torch.no_grad():\n                val_end_time, val_loss_avg, val_accurate_percent, val_labels, val_preds = val_one_epoch(val_loader, net, criterion, device)\n            \n            scheduler.step(val_loss_avg)\n#             early_stopping(val_loss_avg, net)\n\n            if min_val_loss > val_loss_avg:\n                torch.save(net.state_dict(), f\"best_model_{PARAMS['MODEL']}_fold_{fold_index + 1}.pth\")\n                min_val_loss = val_loss_avg\n                \n#             if early_stopping.early_stop:\n#                 print(\"Early stopping\")\n#                 break\n\n            # calculate accurate percentage\n            print(f\"\\nEpoch: {i}/{PARAMS['EPOCHS']}\")\n            print(f\"Fold: {fold_index + 1}/{total_folds}\")\n            print(\"Learning rate: %.8f\" % get_learning_rate(optimizer))\n            print(\"Training loss: %.4f\" % training_loss_avg)\n            print(\"Training accurate percent: %.4f\" % training_accurate_percent)\n            print(\"Training time: %.4f\" % training_end_time)\n            print(\"-------\")\n            print(\"Validation loss: %.4f\" % val_loss_avg)\n            print(\"Validation accurate percent: %.4f\" % val_accurate_percent)\n            print(\"Validation time: %.4f\" % val_end_time)\n            print(\"=\" * 50)\n            \n            torch.save(net.state_dict(), f\"model_{PARAMS['MODEL']}_fold_{fold_index + 1}_epoch_{i}.pth\")\n        # show confusion matrix\n            \n#         print(f\"Saving model fold {fold_index}\")\n#         torch.save(net.state_dict(), f\"model_{PARAMS['MODEL']}_fold_{fold_index + 1}.pth\")\n        del net, criterion, optimizer, training_loader, val_loader, scheduler\n        torch.cuda.empty_cache()\n        \n        show_heatmap(val_labels, val_preds)\n\n        \n# statistic = train_finetuned_model_no_cv(\n#     training_loader,\n#     val_loader,\n#     PARAMS[\"EPOCHS\"],\n#     device)\n\ntrain_finetuned_model_cv(df, device)","execution_count":null,"outputs":[]},{"metadata":{"id":"xX-xxPqsxpBN","trusted":true},"cell_type":"code","source":"# show confusion matrix\n# https://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html","execution_count":null,"outputs":[]},{"metadata":{"id":"46pSHIy5I5d9"},"cell_type":"markdown","source":"### V5 changes:\n- Replace torch transform by albumentations\n- Add weight decay (trivial: 1e-6)\n- Add early stopping\n- Use similar learning rates for every layers"}],"metadata":{"accelerator":"GPU","colab":{"name":"greenlabs-test-3.ipynb","provenance":[]},"kernelspec":{"display_name":"kaggle","language":"python","name":"kaggle"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.9"}},"nbformat":4,"nbformat_minor":4}