{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"# installing xla module on kaggle kernel\n!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# torchcontrib module for Stocastic Weight averaging \n!pip install torchcontrib","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# xla imports\n\nimport warnings\nimport torch_xla\nimport torch_xla.debug.metrics as met\nimport torch_xla.distributed.data_parallel as dp\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.utils.utils as xu\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.xla_multiprocessing as xmp","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# required imports\nimport pandas as pd\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# reading train csv data using pandas\ntrain = pd.read_csv(\"../input/cassava-leaf-disease-classification/train.csv\")\nsample_submission = pd.read_csv(\"../input/cassava-leaf-disease-classification/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = train.loc[train['label'] == 3].reset_index(drop=True)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df.head(2000)\ndf.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_train = pd.concat([df, train.loc[train['label'] != 3]]).reset_index(drop=True)\nnew_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# creating stratified k-folds \n\nfrom sklearn import model_selection\n\nnew_train[\"kfold\"] = -1    \nnew_train = new_train.sample(frac=1).reset_index(drop=True)\ny = new_train.label.values\nkf = model_selection.StratifiedKFold(n_splits=5)\n\nfor f, (t_, v_) in enumerate(kf.split(X=new_train, y=y)):\n    new_train.loc[v_, 'kfold'] = f\n    \nnew_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['label'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_train['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import Dataset\nimport cv2\nimport torch\nfrom torchvision import transforms\nimport albumentations\nfrom PIL import Image\n\nclass leaf_classification(Dataset):\n    def __init__(self, ids, image_id, label, mean , std , is_valid):\n        self.ids = ids\n        self.image_id = image_id\n        self.label = label\n        self.is_valid = is_valid\n        if self.is_valid == 1: \n            # transforms for validation images\n            self.aug = albumentations.Compose([\n               albumentations.Normalize(mean , std , always_apply = True) \n            ])\n        else: \n            # transfoms for training images \n            self.aug = albumentations.Compose([\n                albumentations.Normalize(mean , std , always_apply = True),\n                albumentations.ShiftScaleRotate(shift_limit = 0.0625,\n                                                scale_limit = 0.1 ,\n                                                rotate_limit = 5,\n                                                p = 0.9)\n            ]) \n        \n    def __len__(self):\n        return len(self.ids)\n    \n    def __getitem__(self, index):\n        # converting jpg format of images to numpy array\n        img = np.array(Image.open('../input/cassava-leaf-disease-classification/train_images/' + self.image_id[index])) \n        \n        img = cv2.resize(img, dsize=(228, 228), interpolation=cv2.INTER_CUBIC)\n        img = self.aug(image = img)['image']\n        img = np.transpose(img , (2,0,1)).astype(np.float32) # 2,0,1 because pytorch excepts image channel first then dimension of image\n        \n       \n        return {\n            'image' : torch.tensor(img, dtype = torch.float) , \n            'label' : torch.tensor(self.label[index], dtype = torch.float)\n        }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fold = 0\n\ndf_train = new_train[new_train.kfold != fold].reset_index(drop=True)\ndf_valid = new_train[new_train.kfold == fold].reset_index(drop=True)\n\n# prepare transforms standard to MNIST\ntrain_data = leaf_classification(ids = [i for i in range(len(df_train))], \n                                 image_id = df_train['image_id'], \n                                 label = df_train['label'],\n                                 mean = (0.485, 0.456, 0.406),\n                                 std = (0.229, 0.224, 0.225), \n                                 is_valid = 0)\n\nval_data = leaf_classification(ids = [i for i in range(len(df_valid))], \n                               image_id = df_valid['image_id'],\n                               label = df_valid['label'],\n                               mean = (0.485, 0.456, 0.406),\n                               std = (0.229, 0.224, 0.225), \n                               is_valid = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#dry run \n\nimport matplotlib.pyplot as plt\n\nidx = 115 \nimg = train_data[idx]['image']\n\nprint(train_data[idx]['label'])\nnpimg = img.numpy()\nplt.imshow(np.transpose(npimg, (1,2,0)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sampler = torch.utils.data.distributed.DistributedSampler(\n          train_data,\n          num_replicas=xm.xrt_world_size(),\n          rank=xm.get_ordinal(),\n          shuffle=True)\n\nvalid_sampler = torch.utils.data.distributed.DistributedSampler(\n          val_data,\n          num_replicas=xm.xrt_world_size(),\n          rank=xm.get_ordinal(),\n          shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_BATCH_SIZE = 32\n\nfrom torch.utils.data import DataLoader\n\ntraining_dataloader = DataLoader(train_data,\n                        num_workers=4,\n                        batch_size=TRAIN_BATCH_SIZE,\n                        sampler=train_sampler,\n                        drop_last=True\n                       )\n\nval_dataloader = DataLoader(val_data,\n                        num_workers=4,\n                        batch_size=TRAIN_BATCH_SIZE,\n                        sampler=valid_sampler,\n                        drop_last=False\n                       )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = xm.xla_device()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install efficientnet_pytorch\n\nimport efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.nn as nn\nimport torch.nn.functional as F\n\n# increasing few layers in our model\nclass EfficientNet_b4(nn.Module):\n    def __init__(self):\n        super(EfficientNet_b4, self).__init__()\n        self.model = efficientnet_pytorch.EfficientNet.from_pretrained('efficientnet-b4')\n        self.dropout = nn.Dropout(0.1)\n        self.final_layer = nn.Linear(1792 , 5)\n        \n    def forward(self, inputs):\n        batch_size, _, _, _ = inputs.shape\n        \n        x = self.model.extract_features(inputs)\n\n        # Pooling and final linear layer\n        x = self.model._avg_pooling(x)\n        \n        x = F.adaptive_avg_pool2d(x, 1).reshape(batch_size, -1)\n        outputs = self.final_layer(self.dropout(x))\n\n        return outputs\n    \nmodel = EfficientNet_b4()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install torch-summary","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torchsummary import summary\nsummary(model, (3, 224, 224))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#for Stochastic Weight Averaging in PyTorch\nfrom torchcontrib.optim import SWA\n\nEPOCHS = 5\nnum_train_steps = int(len(train_data) / TRAIN_BATCH_SIZE / xm.xrt_world_size() * EPOCHS)\n\n# printing the no of training steps for each epoch of our training dataloader  \nxm.master_print(f'num_train_steps = {num_train_steps}, world_size={xm.xrt_world_size()}')\n\nmodel = model.to(device)\nparams = list(model.final_layer.parameters())\n\nbase_optimizer = torch.optim.Adam(params, lr=1e-4* xm.xrt_world_size())\n\n\n\noptimizer = SWA(base_optimizer, swa_start=5, swa_freq=5, swa_lr=0.05)\n\nloss_fn = torch.nn.CrossEntropyLoss()\n\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience = 5, verbose = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!git clone https://github.com/ecs-vlc/FMix.git","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from FMix.fmix import sample_mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lambda_value, mask = sample_mask(1.0, 3.0, (224, 224), 0.0, False)\ntype(lambda_value)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# defining the training loop\ndef train_loop_fn(data_loader, model, optimizer, device, scheduler=None):\n    running_loss = 0.0\n    running_corrects = 0\n    \n    model.train()\n    \n    alpha, decay_power = 1.0, 3.0\n    \n    for batch_index,dataset in enumerate(data_loader):\n        image = dataset[\"image\"]\n        label = dataset[\"label\"]\n        \n        \n        lambda_value, mask = sample_mask(alpha, decay_power, (224, 224), 0.0, False)\n        \n        mask = torch.tensor(mask, dtype = torch.float).to(device) \n        perm = torch.randperm(image.size(0))\n\n        x1, x2 = image * mask, image[perm] * (1 - mask)\n        image = x1 + x2\n\n        sample_and_apply(x, alpha, decay_power, (224, 224), max_soft, reformulate)\n        \n        image = image.to(device, dtype=torch.float)\n        label = label.to(device, dtype=torch.float)\n        \n        optimizer.zero_grad()\n\n        outputs = model(image)\n        \n        loss = loss_fn(outputs, label) * lambda_value + loss_fn(outputs, label[perm]) * (1 - lambda_value)\n#         loss = loss_fn(outputs, label)\n\n        loss.backward()\n        xm.optimizer_step(optimizer)\n\n        running_loss += loss.item()\n            \n    train_loss = running_loss / float(len(train_data))\n    scheduler.step(train_loss)\n    \n    return train_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def eval_loop_fn(data_loader, model, device):\n    running_loss = 0.0\n    running_corrects = 0\n    \n    model.eval()\n    \n    for batch_index,dataset in enumerate(data_loader):\n        image = dataset[\"image\"]\n        label = dataset[\"label\"]\n        \n        image = image.to(device, dtype=torch.float)\n        label = label.to(device, dtype=torch.float)\n        \n        outputs = model(image)\n        _, preds = torch.max(outputs, 1)\n        \n        loss = loss_fn(outputs , label)\n\n        running_loss += loss.item()\n        running_corrects += torch.sum(preds == label.data)\n    \n    valid_loss = running_loss / float(len(val_data))\n    valid_acc = running_corrects.double() / float(len(val_data))\n    \n    return valid_loss , valid_acc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fold_counter(fold):\n    a_string = \"*\" * 20\n\n    print(a_string, \" FOLD NUMBER \", fold, a_string)\n\n    df_train = new_train[new_train.kfold != fold].reset_index(drop=True)\n    df_valid = new_train[new_train.kfold == fold].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _run():\n    \n    for i in range(5):\n        fold_counter(i)\n        \n        for param in model.parameters():\n            param.requires_grad = False\n\n        for param in params:\n            param.requires_grad = True\n            \n        evaluation_accuracies = []\n        for epoch in range(EPOCHS):\n            xm.master_print(f\"Epoch --> {epoch+1} / {EPOCHS}\")\n            xm.master_print(f\"-------------------------------\")\n            para_loader = pl.ParallelLoader(training_dataloader, [device])\n            training_loss = train_loop_fn(para_loader.per_device_loader(device), model, optimizer, device, scheduler)\n            xm.master_print('training Loss: {:.4f} '.format(training_loss))\n\n            para_loader = pl.ParallelLoader(val_dataloader, [device])\n            evaluation_loss, evaluation_accuracy = eval_loop_fn(para_loader.per_device_loader(device), model, device)\n            xm.master_print('validation Loss: {:.4f} & validation accuracy : {:.4f}'.format(evaluation_loss , evaluation_accuracy))\n            evaluation_accuracies.append(evaluation_accuracy)\n            \n            \n        xm.master_print('')    \n        xm.master_print('Best accuracy for this cv fold is: {:.4f} '.format(max(evaluation_accuracies)))\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _mp_fn(rank, flags):\n    torch.set_default_tensor_type('torch.FloatTensor')\n    a = _run()\n    optimizer.swap_swa_sgd()\n    \n# applying multiprocessing so that images get paralley trained in different cores of kaggle-tpu\nFLAGS={}\nxmp.spawn(_mp_fn, args=(FLAGS,), nprocs=1, start_method='fork')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}