{"cells":[{"metadata":{},"cell_type":"markdown","source":"### About\n\nI use Pytorch Lightning for building a model \n* [Triple Stratified 192x192 JPEG images from Chris Deotte](#dataloading)\n* [simple augmentations](#augmentations)\n* [Uses Efficientnet](#efficientnet)\n* [Uses BinaryCrossEntropyWithLogits as the Loss function](#lossfunction)\n* [5 fold CV](#folding)\n* [AdamW optimizer with ReduceLROnPlateau scheduler](#optimscheduler)\n* [GPU training with 12 epochs per fold, lr=1e-4](#training)\n* [Simple avergae of 5 fold output for Submission](#submission)\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# !curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n# !python pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import torch_xla.core.xla_model as xm\n# import torch_xla.distributed.parallel_loader as pl\n# import torch_xla.distributed.xla_multiprocessing as xmp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pytorch_lightning\n# !pip uninstall -q typing --yes\n# !pip install https://github.com/PytorchLightning/pytorch-lightning/archive/master.zip --upgrade\n# !pip install git+https://github.com/PytorchLightning/pytorch-lightning.git@master --upgrade\n\n# Install pytorcuh-Efficientnet\n!pip install git+https://github.com/krisho007/EfficientNet-PyTorch\n\n# !pip install efficientnet-pytorch\n\n!pip install https://github.com/lessw2020/Ranger-Deep-Learning-Optimizer","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"scrolled":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport pandas as pd\nimport numpy as np\nfrom torch.utils.data import Dataset,DataLoader\nimport pytorch_lightning as ptl\nfrom efficientnet_pytorch import EfficientNet\nfrom pytorch_lightning.metrics.classification import AUROC\nfrom pytorch_lightning.callbacks import EarlyStopping\nimport torch.nn.functional as Functional\nfrom PIL import Image\nimport random\nimport os\nimport shutil\nfrom glob import glob\nimport cv2\nfrom torch.optim.lr_scheduler import ExponentialLR\nfrom pytorch_lightning import loggers\nfrom pytorch_lightning import _logger as log\nimport albumentations as A\nimport math\nfrom albumentations.pytorch.transforms import ToTensorV2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nseed_everything()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Load Data <a id='dataloading' />","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/jpeg-melanoma-192x192/train.csv\")\n\ntest = pd.read_csv(\"../input/jpeg-melanoma-192x192/test.csv\")\n\n# Creating a new column to be populated later for submission\ntest['target'] = 0\n\nsubmission = pd.read_csv(\"../input/jpeg-melanoma-192x192/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Records with tfrecord = -1 => duplicate. Getrid of them\ntrain_data = train[train.tfrecord != -1].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Augmentations <a id=\"augmentations\"></a>","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"mean = (0.485, 0.456, 0.406)\nstd = (0.229, 0.224, 0.225)\n\ndef get_train_transforms():\n    return A.Compose([\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.GaussianBlur(p=0.3),\n            A.Normalize(mean, std, max_pixel_value=255, always_apply=True),\n            ToTensorV2(),\n        ], p=1.0)\n\ndef get_valid_transforms():\n    return A.Compose([\n            A.Normalize(mean, std, max_pixel_value=255, always_apply=True),\n            ToTensorV2(),\n        ], p=1.0)\n\ndef get_tta_transforms():\n    return A.Compose([\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.Normalize(mean, std, max_pixel_value=255, always_apply=True),\n            ToTensorV2(),\n        ], p=1.0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataset\n","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"\nclass melanomaDataset(Dataset):\n    def __init__(self, data, is_testing = False, image_folder = '../input/jpeg-melanoma-192x192/train', transforms=None):\n        self.data = data\n        self.is_testing = is_testing\n        self.image_folder = image_folder\n        self.transforms = transforms\n        \n    def __len__(self):\n        return self.data.shape[0]\n    \n    def __getitem__(self, index):\n        image_path = f\"{self.image_folder}/{self.data.iloc[index]['image_name']}.jpg\"\n        target = self.data.iloc[index]['target']\n        \n        image = cv2.imread(image_path, cv2.IMREAD_COLOR)\n        \n        if self.transforms:\n            sample = self.transforms(image=image)\n            image  = sample['image']\n            \n        if self.is_testing:\n            sample =  {\n                \"image_name\": self.data.iloc[index]['image_name'],\n                \"image\": image\n            } \n        else:        \n            sample = {\n                \"image_name\": self.data.iloc[index]['image_name'],\n                \"image\": image,\n                \"target\": torch.tensor(target, dtype = torch.float32)\n            }\n            \n        return sample\n        ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model <a id='efficientnet'/><a id='lossfunction'/><a id='optimscheduler'/><a id='folding'/>","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"class melanomaModel(ptl.LightningModule):\n    def __init__(self, hparams):\n        super(melanomaModel, self).__init__()\n        self.hparams = hparams\n        self.model = EfficientNet.from_pretrained('efficientnet-b5', num_classes=1)        \n        \n    def forward(self, x):\n        return torch.squeeze(self.model(x[\"image\"]))\n        \n\n    def getLoss(self, prediction, actual):\n        loss_function = Functional.binary_cross_entropy_with_logits\n        loss = loss_function(prediction, actual)\n        return loss\n\n    def prepare_data(self):\n        fold = self.hparams.fold\n        complete_range = list(range(15))\n        validation_start_index = fold * 3\n        validation_end_index = validation_start_index + 3\n        validation_range = complete_range[validation_start_index:validation_end_index]\n        \n        df_train = train_data[~train_data.tfrecord.isin(validation_range)].reset_index(drop=True)\n        df_valid = train_data[train_data.tfrecord.isin(validation_range)].reset_index(drop=True)\n        df_test = test\n\n        # Datasets\n        self.train_dataset = melanomaDataset(df_train, transforms=get_train_transforms())\n        self.valid_dataset = melanomaDataset(df_valid, transforms=get_valid_transforms())\n        self.test_dataset = melanomaDataset(df_test, image_folder = '../input/jpeg-melanoma-192x192/test', transforms=get_tta_transforms()) \n\n    def train_dataloader(self):               \n        training_loader = DataLoader(\n            self.train_dataset, batch_size=32, num_workers=4, shuffle=True\n        )        \n        log.info(\"Training data loaded.\")\n        return training_loader    \n    \n    def training_step(self, batch, batch_index):\n        # Find current output\n        batch_prediction = self(batch)        \n        # Find loss\n        loss = self.getLoss(batch_prediction, batch[\"target\"])\n        \n        return {\"loss\": loss}\n    \n\n    def val_dataloader(self):        \n        valid_loader = DataLoader(\n            self.valid_dataset, batch_size=16, num_workers=4, shuffle=False\n        )\n        log.info(\"Validation data loaded.\")\n        return valid_loader    \n\n    def validation_step(self, batch, batch_index):\n        # Find current output\n        batch_prediction = self(batch)\n        # Find loss\n        loss = self.getLoss(batch_prediction, batch[\"target\"])\n        return {\"val_loss\": loss,\n                \"y\" : batch[\"target\"].detach(),\n                \"y_hat\": batch_prediction.detach()}\n    \n    def validation_epoch_end(self, outputs):\n        val_loss_mean = torch.stack([x[\"val_loss\"] for x in outputs]).mean()\n        y = torch.cat([x['y'] for x in outputs])\n        y_hat = torch.cat([x['y_hat'] for x in outputs])\n        auc = AUROC()(pred=y_hat, target=y) if y.float().mean() > 0 else 0.5 # skip sanity check\n        \n        # rounded with a threhold of 0.5 and compared with GT for accuracy\n        acc = (y_hat.round() == y).float().mean().item()\n        \n        print(f\"Fold: {self.hparams.fold} Epoch {self.current_epoch} auc:{auc}\")\n        return {'avg_val_loss': val_loss_mean,\n                'val_auc': auc, 'val_acc': acc}    \n    \n    def test_dataloader(self):\n        return DataLoader(self.test_dataset, batch_size=16, num_workers=4,\n                          drop_last=False, shuffle=False, pin_memory=False)      \n    \n    def test_step(self, batch, batch_nb):\n        y_hat = self(batch).flatten()\n        return {'y_hat': y_hat}\n\n    def test_epoch_end(self, outputs):\n#         import pdb; pdb.set_trace()        \n        # outputs has all the output for test data \n        y_hat = torch.cat([x['y_hat'] for x in outputs])\n        \n        #Below line will fail if it is a fast_dev_run=True, as outputs has only one batch\n        test['target'] = y_hat.tolist()\n        \n        # Two required columns into submission csv\n        header = [\"image_name\",\"target\"]\n        test.to_csv(f'submission{self.hparams.fold}.csv', columns = header, index=False)\n        \n#         return y_hat\n\n\n    def configure_optimizers(self):\n        optim = torch.optim.AdamW(self.parameters(), lr=self.hparams['lr'])\n#         optim = Ranger(self.parameters(), lr=self.hparams['lr'])\n        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n            optim,\n            patience=3,\n            threshold=0.001,\n            mode=\"max\"\n        )\n    \n        gen_sched = {\n            \"scheduler\": scheduler,  # Explore other schedulers\n            \"interval\": \"step\",  # can be 'epoch' as well. step=>batch\n            \"frequency\": 1,\n        }  # called after each training step.If not mentioned, scheduler is called after every epoch\n        return {\"optimizer\": optim, \"scheduler\": gen_sched}  # Run scheduler","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train & Test <a id='training' />","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# Define a function to initialize and train a model\n\ndef train(fold):\n\n    # Checkpoints\n    if not os.path.exists('Checkpoints'):\n        os.makedirs('Checkpoints')   \n\n    # Hyper parameters\n    hparams = {\"fold\":fold, \"lr\":1e-3}\n    model = melanomaModel(hparams)\n    checkpoint_callback = ptl.callbacks.ModelCheckpoint(\"Checkpoints/{fold:02d}_{epoch:02d}_{val_auc:.4f}\",\n                                                   save_top_k=1, monitor='val_auc', mode='max')    \n    \n    early_stop_callback = EarlyStopping(\n       monitor='avg_val_loss',\n       min_delta=0.00,\n       patience=3,\n       verbose=True,\n       mode='min'\n    )    \n    \n#     trainer = ptl.Trainer(tpu_cores=1, precision=16, max_epochs=1, fast_dev_run=False\n#                           , checkpoint_callback=checkpoint_callback\n# #                           , early_stop_callback=early_stop_callback\n#                          )    \n\n    trainer = ptl.Trainer(gpus=-1, max_epochs=5, fast_dev_run=False, checkpoint_callback=checkpoint_callback)       \n    \n    trainer.fit(model)\n    trainer.test()    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"trainer = train(0)\ntrainer = train(1)\ntrainer = train(2)\ntrainer = train(3)\ntrainer = train(4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Submission file <a id='submission' />","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Simple Average the folds\nimport pandas as pd\nSubmission0 = pd.read_csv('./submission0.csv')\nSubmission1 = pd.read_csv('./submission1.csv')\nSubmission2 = pd.read_csv('./submission2.csv')\nSubmission3 = pd.read_csv('./submission3.csv')\nSubmission4 = pd.read_csv('./submission4.csv')\n\nSubmission = pd.concat([Submission0, Submission1, Submission2, Submission3, Submission4]).groupby('image_name').mean().reset_index()\nheader = [\"image_name\",\"target\"]\nSubmission.to_csv(f'submission.csv', columns = header, index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}