{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pytorch-warmup\n!pip install albumentations","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os.path as osp\n\n# computation\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport pytorch_warmup as warmup\n\n# data pipeline\nimport imageio\nimport imgaug as ia\nfrom imgaug import augmenters as iaa\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom albumentations import (\n    CenterCrop,\n    CoarseDropout,\n    Compose,\n    Cutout,\n    HorizontalFlip,\n    HueSaturationValue,\n    IAAAdditiveGaussianNoise,\n    ImageOnlyTransform,\n    Normalize,\n    OneOf,\n    RandomBrightness,\n    RandomBrightnessContrast,\n    RandomContrast,\n    RandomCrop,\n    RandomResizedCrop,\n    Resize,\n    Rotate,\n    ShiftScaleRotate,\n    Transpose,\n    VerticalFlip,\n)\nfrom albumentations.pytorch import ToTensorV2\n\n# utils\nfrom tqdm.notebook import tqdm\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nimport torchvision.models as models\n\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pytorch GPU 확인\n\nprint(torch.cuda.is_available())\nprint(torch.cuda.get_device_name(0))\nprint(torch.cuda.device_count())\n\n# Default CUDA device\ncuda = torch.device('cuda:0')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Some constants\nROOT = '/kaggle/input/cassava-leaf-disease-classification'\nTRAIN_DIR = f'{ROOT}/train_images/'\nTRAIN_CSV = f'{ROOT}/train.csv'\nTEST_DIR = f'{ROOT}/test_images/'\nTEST_CSV = f'{ROOT}/sample_submission.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(self, split, transform=None, one_hot=False, label_smoothing=False, label_smoothing_alpha=0.2):\n        assert split in ('train', 'val', 'test')\n        self.split = split\n        self.transform = transform\n        if split in ('train', 'val'):\n            csv = pd.read_csv(TRAIN_CSV)\n\n            csv_val = pd.DataFrame()\n            csv_train = pd.DataFrame()\n            for i in range(5):\n                csv_label = csv[csv['label']==i]\n                csv_label_val = csv_label.sample(n=200, random_state=0)\n                csv_label_train = csv_label[~csv_label.index.isin(csv_label_val.index)]\n                csv_label_train = csv_label_train.sample(n=2000, random_state=0, replace=True)\n                csv_val = pd.concat([csv_val, csv_label_val], axis=0)\n                csv_train = pd.concat([csv_train, csv_label_train], axis=0)\n\n            self.df = (csv_train, csv_val)[0 if split == 'train' else 1].reset_index()\n        else:\n            self.df = pd.read_csv(TEST_CSV)\n            \n        self.one_hot = one_hot\n        self.label_smoothing = label_smoothing\n        self.label_smoothing_alpha = label_smoothing_alpha\n        \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, i: int):\n        base_dir = TRAIN_DIR if self.split in ('train', 'val') else TEST_DIR\n        x = imageio.imread(osp.join(base_dir, self.df['image_id'][i]))\n        y = self.df['label'][i] if self.split in ('train', 'val') else -1\n        if self.transform:\n            x = self.transform(x)\n        \n        #One-hot encoding\n        if self.one_hot:\n            y_oh = np.zeros(5)\n            y_oh[y] = 1\n            y = y_oh\n        \n        # Label smoothing\n        if self.label_smoothing:\n            y_ls = y*(1-self.label_smoothing_alpha) + self.label_smoothing_alpha/5\n            y = y_ls\n        \n        y = y.astype('float32')\n        return (x, y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Implement your data augmentation pipelines"},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO: You'll need some heavy augmentations to get a high score. \n#\n# See \n#     https://github.com/aleju/imgaug \n#     https://imageio.readthedocs.io/en/stable/\n#\n# for detailed `imgaug` references.\n#\n# NOTE: If you choose to normalize the training images, you should normalize the test images as well.\n#\nINPUT_SIZE = 224\n\nTRANSFORMS = {\n    'train': transforms.Compose([\n        iaa.Sequential([\n            iaa.Resize((255, 255)),\n            iaa.CenterCropToFixedSize(INPUT_SIZE,INPUT_SIZE),\n            \n            # Image augmentation\n            iaa.Sometimes(0.25, iaa.convolutional.Sharpen(alpha=(0.0, 0.2))),\n            iaa.Sometimes(0.25, iaa.GaussianBlur(sigma=(0, 3.0))),\n            iaa.Multiply((0.8, 1.2), per_channel=0.2),\n#             iaa.Fliplr(0.5),\n#             iaa.Flipud(0.5),\n#             iaa.Affine(rotate=(-20, 20), mode='symmetric'),\n#             iaa.Sometimes(0.25,\n#                           iaa.OneOf([iaa.Dropout(p=(0, 0.1)),\n#                                      iaa.CoarseDropout(0.1, size_percent=0.5)])),\n#             iaa.AddToHueAndSaturation(value=(-10, 10), per_channel=True)\n            \n            \n        ]).augment_image,\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406],\n                             [0.229, 0.224, 0.225])\n    ]),\n    'val': transforms.Compose([\n        iaa.Sequential([\n            iaa.Resize((255, 255)),\n            iaa.CenterCropToFixedSize(INPUT_SIZE,INPUT_SIZE),\n        ]).augment_image,\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406],\n                             [0.229, 0.224, 0.225])\n    ]),\n    'test': transforms.Compose([\n        iaa.Sequential([\n            iaa.Resize((255, 255)),\n            iaa.CenterCropToFixedSize(INPUT_SIZE,INPUT_SIZE),\n        ]).augment_image,\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406],\n                             [0.229, 0.224, 0.225])\n    ]),\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build your Cassava leaf classifier"},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO: Implement your classifier.\n\nclass multiclass_loss_fn(object):\n    def __init__(self, loss_weight= []):\n        super(multiclass_loss_fn, self).__init__()\n        self.loss_weight =  loss_weight\n        \n        if len(self.loss_weight) > 0:\n            loss_weight = np.array(self.loss_weight, dtype='float32')\n            loss_weight_norm = loss_weight.sum()\n            loss_weight = loss_weight/loss_weight_norm\n            loss_weight = loss_weight.reshape(1,-1)\n            loss_weight = torch.Tensor(loss_weight).cuda()\n            self.loss_weight = loss_weight\n        \n    def __call__(self, y_hat, y):\n        loss_weight = torch.repeat_interleave(self.loss_weight, repeats=y_hat.shape[0], dim=0)\n        y_hat = y_hat * loss_weight\n        \n        res = -nn.LogSoftmax(dim=0)(y_hat)\n        res = res * y\n        res = torch.sum(res) / y.shape[0]\n        return res\n\ndef set_parameter_requires_grad(model, feature_extracting):\n    if feature_extracting:\n        for param in model.parameters():\n            param.requires_grad = False\n        \ndef MixupTraining(batch, alpha, random_state=None):\n    img = batch[0]\n    lbl = batch[1]\n    \n    np.random.seed(random_state)\n    ratio = np.random.beta(alpha, alpha, size = (len(img),1,1,1)).astype('float32')\n    ratioImg = torch.Tensor(ratio)\n    ratioImg = torch.repeat_interleave(ratioImg, repeats=img.shape[1], dim=1)\n    ratioImg = torch.repeat_interleave(ratioImg, repeats=img.shape[2], dim=2)\n    ratioImg = torch.repeat_interleave(ratioImg, repeats=img.shape[3], dim=3)\n    \n    ratio = ratio.reshape(len(lbl),1)\n    ratioLbl = torch.Tensor(ratio)\n    ratioLbl = torch.repeat_interleave(ratioLbl, repeats=lbl.shape[1], dim=1)\n    \n    img2 = torch.flip(img, dims=(0,))\n    lbl2 = torch.flip(lbl, dims=(0,))\n    mixedImg = torch.add(torch.mul(ratioImg, img), torch.mul(torch.sub(1, ratioImg), img2))\n    mixedLbl = torch.add(torch.mul(ratioLbl, lbl), torch.mul(torch.sub(1, ratioLbl), lbl2))\n    \n    mixedBatch = (mixedImg, mixedLbl)\n    return mixedBatch\n\ndef make_confMatrix(Y, Y_hat):\n    confMatrix = np.zeros(shape=(5,5))\n    for y, y_hat in zip(Y_trains, Y_trains_hat):\n        confMatrix[int(y), int(y_hat)] += 1\n    return confMatrix\n\nclass CassavaClassifier(nn.Module):\n    def __init__(self):\n        super(CassavaClassifier, self).__init__()\n        self.resnet = models.wide_resnet50_2(pretrained=True)\n        set_parameter_requires_grad(self.resnet, feature_extracting=True)\n        self.num_ftrs = self.resnet.fc.in_features\n        self.resnet.fc = nn.Linear(self.num_ftrs, 200)\n        self.b1 = nn.BatchNorm1d(num_features=200)\n        self.linear1 = nn.Linear(in_features=200, out_features=5)\n        \n    def forward(self, x):\n        x = self.resnet(x)\n        x = self.b1(x)\n        x = self.linear1(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Implement the training loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO: Define some hyperparameters here.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO: Prepare some components for training your network.\n#\n# See\n#\n#     Optimizer:    https://pytorch.org/docs/stable/optim.html#how-to-use-an-optimizer\n#     LR Scheduler: https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n#     DataLoader:   https://pytorch.org/docs/stable/data.html?highlight=dataloader#torch.utils.data.DataLoader\n#\n# for their actual APIs and detailed usages.\ntrainDataset = CassavaDataset('train', transform=TRANSFORMS['train'], one_hot=True, label_smoothing=False, label_smoothing_alpha=0.3)\nvalDataset = CassavaDataset('val', transform=TRANSFORMS['val'], one_hot=True, label_smoothing=False, label_smoothing_alpha=0.3)\ntrainDataLoader = DataLoader(trainDataset, batch_size=128, num_workers=1, shuffle=True)\nvalDataLoader = DataLoader(valDataset, batch_size=64, shuffle=True)\nmodel = CassavaClassifier().cuda()\nloss_fn = multiclass_loss_fn([0.8,1,1,0.7,1])\n# loss_fn = nn.CrossEntropyLoss()\noptimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9, weight_decay=0.01)\n# optimizer = optim.Adam(model.parameters(), lr=0.0001, weight_decay=1e-5)\nscheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.1, patience=0, verbose=True)\n# scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=10, eta_min=0)\n# warmup_scheduler = warmup.UntunedLinearWarmup(optimizer)\nscore_fn = accuracy_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO: Implement your training loop here.\nhistory = dict()\nhistory['loss'] = dict()\nhistory['acc'] = dict()\n\nhistory['loss']['train'] = []\nhistory['loss']['val'] = []\n\nhistory['acc']['train'] = []\nhistory['acc']['val'] = []\n\nloss_weight = [1,1,1,1,1]\nfor epoch in range(100):\n    losses_train = []\n    correct_train = 0\n    incorrect_train = 0\n    \n    confMatrix = np.zeros(shape=(5,5))\n    loss_fn = multiclass_loss_fn(loss_weight)\n    for i,batch in enumerate(tqdm(trainDataLoader)):\n        \n        # batch = MixupTraining(batch, 0.1, random_state=42)\n        \n        X_train = batch[0].cuda()\n        Y_train = batch[1].reshape(-1,5).cuda()\n        # Y_train = batch[1].long().cuda()\n\n        model.train()\n        Y_train_hat = model(X_train)\n        loss_train = loss_fn(Y_train_hat, Y_train)\n        optimizer.zero_grad()\n        loss_train.backward()\n        optimizer.step()\n        # warmup_scheduler.dampen()\n        \n        losses_train.append(loss_train.item())\n\n        Y_train = np.argmax(np.array(Y_train.tolist()),axis=1)\n        # Y_train = np.array(Y_train.tolist())\n        Y_train_hat = np.argmax(np.array(Y_train_hat.tolist()),axis=1)\n        correct_train += (Y_train == Y_train_hat).sum()\n        incorrect_train += (Y_train != Y_train_hat).sum()\n        \n        \n        confMatrix += make_confMatrix(Y_train, Y_train_hat)\n        \n    loss_weight = [1/confMatrix[i,i] for i in range(5)]\n\n\n            \n    losses_val = []\n    correct_val = 0\n    incorrect_val = 0\n    for j,batch in enumerate(tqdm(valDataLoader)):\n        X_val = batch[0].cuda()\n        Y_val = batch[1].reshape(-1,5).cuda()\n        # Y_val = batch[1].long().cuda()\n\n        model.eval()\n        Y_val_hat = model(X_val)\n        loss_val = loss_fn(Y_val_hat, Y_val)\n        \n        losses_val.append(loss_val.item())\n\n        Y_val = np.argmax(np.array(Y_val.tolist()),axis=1)\n        # Y_val = np.array(Y_val.tolist())\n        Y_val_hat = np.argmax(np.array(Y_val_hat.tolist()),axis=1)\n        correct_val += (Y_val == Y_val_hat).sum()\n        incorrect_val += (Y_val != Y_val_hat).sum()\n        \n    loss_train_epoch = sum(losses_train)/len(losses_train)\n    loss_val_epoch = sum(losses_val)/len(losses_val)\n    acc_train_epoch = correct_train / (correct_train + incorrect_train)\n    acc_val_epoch = correct_val / (correct_val + incorrect_val)\n    \n    history['loss']['train'].append( loss_train_epoch )\n    history['loss']['val'].append( loss_val_epoch )\n    history['acc']['train'].append( acc_train_epoch )\n    history['acc']['val'].append( acc_val_epoch )\n        \n    print('epoch %d' %(epoch+1))\n    print('train >> loss: %.8f \\t acc: %.8f' % (loss_train_epoch,acc_train_epoch))\n    print(' val  >> loss: %.8f \\t acc: %.8f' % (loss_val_epoch,acc_val_epoch))\n\n    if epoch > 1:\n        if acc_val_epoch >= max(history['acc']['val']):\n            print('best model is updated.')\n            torch.save(model, 'best_model.pt')\n\n        if max(history['acc']['val'][-5:]) - min(history['acc']['val'][-5:]) < 0.0001:\n            break\n\n\n    scheduler.step(acc_val_epoch)\n    # scheduler.step()\n\n    plt.figure(figsize=(10,5))\n    plt.subplot(2,1,1)\n    plt.plot(history['loss']['train'], color='black')\n    plt.plot(history['loss']['val'], color='r')\n    plt.xlabel('epoch')\n    plt.ylabel('loss')\n    plt.subplot(2,1,2)\n    plt.plot(history['acc']['train'], color='black', label='train')\n    plt.plot(history['acc']['val'], color='r', label='test')\n    plt.xlabel('epoch')\n    plt.ylabel('acc')\n    plt.legend()\n    plt.show()\n\n\nprint('Finished Training')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\n\nY_trains = np.array([])\nY_trains_hat = np.array([])\nfor i,batch in enumerate(tqdm(trainDataLoader)):\n        \n        # batch = MixupTraining(batch, 0.1, random_state=42)\n        \n        X_train = batch[0].cuda()\n        Y_train = batch[1].reshape(-1,5).cuda()\n        \n        model.eval()\n        Y_train_hat = model(X_train)\n        \n        Y_train = np.argmax(np.array(Y_train.tolist()),axis=1)\n        Y_train_hat = np.argmax(np.array(Y_train_hat.tolist()),axis=1)\n        \n        Y_trains = np.concatenate((Y_trains, Y_train), axis=0)\n        Y_trains_hat = np.concatenate((Y_trains_hat, Y_train_hat), axis=0)\n\n        \n        \nconfMatrix = make_confMatrix(Y_train, Y_train_hat)\n    \nprint(confMatrix.astype(int))\n\nsns.heatmap(confMatrix, cmap='RdBu').invert_yaxis()\nplt.ylabel('Y')\nplt.xlabel('Y_hat')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\n\n\nY_trains = np.array([])\nY_trains_hat = np.array([])\nfor i,batch in enumerate(tqdm(valDataLoader)):\n        \n        # batch = MixupTraining(batch, 0.1, random_state=42)\n        \n        X_train = batch[0].cuda()\n        Y_train = batch[1].reshape(-1,5).cuda()\n        \n        model.eval()\n        Y_train_hat = model(X_train)\n        \n        Y_train = np.argmax(np.array(Y_train.tolist()),axis=1)\n        Y_train_hat = np.argmax(np.array(Y_train_hat.tolist()),axis=1)\n        \n        Y_trains = np.concatenate((Y_trains, Y_train), axis=0)\n        Y_trains_hat = np.concatenate((Y_trains_hat, Y_train_hat), axis=0)\n\n        \n        \nconfMatrix = make_confMatrix(Y_train, Y_train_hat)\n\n\nsns.heatmap(confMatrix, cmap='RdBu').invert_yaxis()\nplt.ylabel('Y')\nplt.xlabel('Y_hat')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save your trained model. Note that it is recommended to save the 'best' checkpoint, rather than just saving the 'last' checkpoint.\ntorch.save(best_mobel, 'model.pt')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Verify that your trained model can be loaded"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = torch.load('/kaggle/working/model.pt')\nmodel.eval()\ntest_dataset = CassavaDataset('test', transform=TRANSFORMS['test'])\ntest_loader = DataLoader(test_dataset, batch_size=32)\ntest_csv = pd.read_csv(TEST_CSV)\n\ny_hats = []\nfor x, _ in test_loader:\n    y_hat = model(x.cuda())\n    y_hat = torch.argmax(y_hat,dim=1)\n    y_hats.extend(y_hat.cpu().detach().numpy().tolist())\n    \n\ntest_csv['label'] = y_hats\ntest_csv[['image_id','label']].to_csv(\"submission.csv\", index=False)\ntest_csv.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}