{"cells":[{"metadata":{},"cell_type":"markdown","source":"### <font color='blue'>If you like this notebook, please upvote ;)</font>  ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip --quiet install pytorch-lightning","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nsns.set(style=\"darkgrid\")\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score, accuracy_score, classification_report, confusion_matrix, roc_curve, auc\n\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.optim as torch_optim\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom pytorch_lightning.core.lightning import LightningModule\nfrom pytorch_lightning import Trainer\nfrom pytorch_lightning.callbacks.early_stopping import EarlyStopping\nfrom pytorch_lightning.metrics.functional import auroc, accuracy\n\nfrom pytorch_lightning.trainer import seed_everything","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed_everything(seed=123)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load train tabular dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"col_types = {\"image_name\": str,\n            \"patient_id\": str,\n            \"sex\": str,\n            \"age_approx\": np.float16,\n            \"anatom_site_general_challenge\": str,\n            \"diagnosis\": str,\n            \"benign_malignant\": str,\n            \"target\": np.uint8}\ndf = pd.read_csv(\"../input/siim-isic-melanoma-classification/train.csv\", dtype=col_types)\ndf.reset_index(drop=True, inplace=True)\ndf.head(3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Splitting dataset into train / validation and test partitions\n\n#### Pytorch dataset class (images saved in pytorch .pt files)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class MelanomaImageDataset(Dataset):\n    def __init__(self, path, df):\n        self.df = df\n        if \"target\" in self.df.columns:\n            self.target = torch.from_numpy(df.target.values).type(torch.FloatTensor)\n        self.path = path\n\n    def __getitem__(self, idx):\n        image_name = self.df.image_name[idx]\n        image = torch.load(os.path.join(self.path, image_name + '.pt'))\n        if \"target\" in self.df.columns:\n            label = self.target[idx]\n            return image, label\n        else:\n            return image\n                        \n    def __len__(self):\n        return self.df.shape[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"In order to speed up training, JPEG images have been previously saved to tensor pytorch files. The dataset is available here: \n* [dataset images 64x64 grayscale](http://www.kaggle.com/dataset/47c2ca3a3c6c78afa4bce815570256cba79c6879366d5b9b33393097e4293312) \n\nThe transformation applied to JPEG images is simply the following one:\n```\ncustom_transform = transforms.Compose([transforms.Resize((64, 64)),\n                                      transforms.Grayscale(num_output_channels=1),\n                                      transforms.ToTensor(),\n                                     ])\n```","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH_PT_FILE = '../input/melanoma-grayscale-64x64/images64x64.pt/train'\nBATCH_SIZE = 128\n\n# split train/val/test\ntrain_df, val_df = train_test_split(df, stratify=df.target, test_size=0.20)\ntrain_df.reset_index(drop=True, inplace=True)\nval_df.reset_index(drop=True, inplace=True)\n\nval_df, test_df = train_test_split(val_df, stratify=val_df.target, test_size=0.5)\nval_df.reset_index(drop=True, inplace=True)\ntest_df.reset_index(drop=True, inplace=True)\n\n#creating train, valid and test datasets\ntrain_ds = MelanomaImageDataset(PATH_PT_FILE, train_df)\nvalid_ds = MelanomaImageDataset(PATH_PT_FILE, val_df)\ntest_ds = MelanomaImageDataset(PATH_PT_FILE, test_df)\n#creating train, valid and test dataloaders\ntrain_dl = DataLoader(train_ds, batch_size=BATCH_SIZE, num_workers=32, shuffle=True)\nvalid_dl = DataLoader(valid_ds, batch_size=BATCH_SIZE, num_workers=32, shuffle=False)\ntest_dl = DataLoader(test_ds, batch_size=BATCH_SIZE, num_workers=32, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_df.shape, val_df.shape, test_df.shape)\npd.DataFrame({'train': train_df.target.value_counts(), 'val': val_df.target.value_counts(), 'test': test_df.target.value_counts()})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Pytorch Lightning model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class MelanomaModel(LightningModule):\n\n    def __init__(self):\n        super().__init__()\n        # conv kernels\n        self.conv1 = nn.Conv2d(in_channels=1, out_channels=6, kernel_size=3)\n        self.conv2 = nn.Conv2d(6, 16, 3)\n        self.conv3 = nn.Conv2d(16, 32, 3)\n        # dense layers: an affine operation: y = Wx + b\n        self.fc1 = nn.Linear(32 * 6 * 6, 120)  # 6*6 from image dimension\n        self.fc2 = nn.Linear(120, 84)\n        self.fc3 = nn.Linear(84, 1)\n        self.sigmoid = nn.Sigmoid() \n        \n        # callback metrics\n        self.metrics = {'train_loss': [], 'train_acc': [], 'train_aucroc': [],\n                        'val_loss': [], 'val_acc': [], 'val_aucroc': []}\n\n    def forward(self, x):\n        # Max pooling over a (2, 2) window\n        x = F.max_pool2d(F.relu(self.conv1(x)), (2, 2))\n        # If the size is a square we can only specify a single number\n        x = F.max_pool2d(F.relu(self.conv2(x)), 2)\n        x = F.max_pool2d(F.relu(self.conv3(x)), 2)\n        x = torch.flatten(x, start_dim=1) # except the batch size dim\n        x = F.relu(self.fc1(x))\n        x = F.relu(self.fc2(x))\n        x = self.fc3(x)\n        return self.sigmoid(x).flatten()\n        \n    def training_step(self, batch, batch_idx):\n        x, y = batch\n        y_hat = self(x)\n        loss = F.binary_cross_entropy(y_hat, y)\n        return {'loss': loss, \"y_hat\": y_hat, \"y\": y}\n    \n    def training_epoch_end(self, outputs):\n        # concat or stack batchs outputs\n        avg_loss = torch.stack([x['loss'] for x in outputs]).mean()\n        train_y_hat = torch.cat([x['y_hat'] for x in outputs], dim=-1)\n        train_y = torch.cat([x['y'] for x in outputs], dim=-1)\n        # compute accuracy and roc auc\n        train_bin_y_hat = (train_y_hat > 0.5).float() * 1\n        acc = accuracy(train_bin_y_hat, train_y)\n        aucroc = auroc(train_y_hat, train_y)\n#         print(\"train_loss {0:.6f} || train_acc {1:.6f} || train_aucroc {2:.6f}\".format(avg_loss, acc, aucroc))\n        # Record metrics\n        self.metrics['train_loss'].append(avg_loss)\n        self.metrics['train_acc'].append(acc)\n        self.metrics['train_aucroc'].append(aucroc)\n        return {'train_loss': avg_loss}\n    \n    def configure_optimizers(self):\n        optimizer = torch.optim.Adam(self.parameters(), lr=0.001)\n        return optimizer\n    \n    def validation_step(self, batch, batch_idx):\n        x, y = batch\n        y_hat = self(x)\n        return {'val_loss': F.binary_cross_entropy(y_hat, y), \"y_hat\": y_hat, \"y\": y}\n\n    def validation_epoch_end(self, outputs):\n        # concat or stack batchs outputs\n        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()\n        val_y_hat = torch.cat([x['y_hat'] for x in outputs], dim=-1)\n        val_y = torch.cat([x['y'] for x in outputs], dim=-1)\n        # compute accuracy and roc auc\n        val_bin_y_hat = (val_y_hat > 0.5).float() * 1\n        acc = accuracy(val_bin_y_hat, val_y)\n        aucroc = auroc(val_y_hat, val_y)\n#         print(\"val_loss {0:.6f} || val_acc {1:.6f} || val_aucroc {2:.6f}\".format(avg_loss, acc, aucroc))\n        # Record metrics\n        self.metrics['val_loss'].append(avg_loss)\n        self.metrics['val_acc'].append(acc)\n        self.metrics['val_aucroc'].append(aucroc)\n        return {'val_loss': avg_loss, 'val_auc': aucroc} ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = MelanomaModel()\nmodel","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Test on a small batch","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tensor_images = torch.randn(5, 1, 64, 64)\noutputs = model(tensor_images)\nprint(outputs)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training phase ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nearly_stop_callback = EarlyStopping(monitor='val_auc',\n                                    min_delta=0.00,\n                                    patience=5,\n                                    verbose=True,\n                                    mode='max')\ntrainer = Trainer(max_epochs=100,\n                  gpus=1,\n                  check_val_every_n_epoch=1,\n                  num_sanity_val_steps=0,\n                  early_stop_callback=early_stop_callback)\ntrainer.fit(model, train_dl, valid_dl)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Convert callback metrics: from tensor to numpy ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"for key in model.metrics:\n    model.metrics[key] = np.array([val.detach().cpu().numpy() for val in model.metrics[key]])\n\ndf_metrics = pd.DataFrame.from_dict(model.metrics)\ndf_metrics[\"epochs\"] = np.arange(0, len(df_metrics))\ndf_metrics.head(3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Plots","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.set(font_scale=1.)\nplt.figure(figsize=(25,7))\nplt.subplot(131)\nplt.title(\"Loss\")\nsns.lineplot(x=\"epochs\", y=\"train_loss\", data=df_metrics, label=\"train_loss\")\nsns.lineplot(x=\"epochs\", y=\"val_loss\", data=df_metrics, label=\"val_loss\")\nplt.subplot(132)\nplt.title(\"Accuracy\")\nsns.lineplot(x=\"epochs\", y=\"train_acc\", data=df_metrics, label=\"train_acc\")\nsns.lineplot(x=\"epochs\", y=\"val_acc\", data=df_metrics, label=\"val_acc\")\nplt.subplot(133)\nplt.title(\"AUC ROC\")\nsns.lineplot(x=\"epochs\", y=\"train_aucroc\", data=df_metrics, label=\"train_aucroc\")\nsns.lineplot(x=\"epochs\", y=\"val_aucroc\", data=df_metrics, label=\"val_aucroc\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Performances on test dataset\n\n#### Collect predictions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = []\nfor image, label in test_dl:\n    preds.append(model(image).detach())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred_prob = torch.cat(preds, dim=-1).cpu().numpy()\ny_pred = np.where(y_pred_prob >= .5, 1, 0)\ny_true = test_df.target.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,6))\n\n## CONFUSION MATRIX\nplt.subplot(121)\n# Set up the labels for in the confusion matrix\ncm = confusion_matrix(y_true, y_pred)\nnames = ['True Negatives', 'False Positives', 'False Negatives', 'True Positives']\ncounts = ['{0:0.0f}'.format(value) for value in cm.flatten()]\npercentages = ['{0:.2%}'.format(value) for value in cm.flatten()/np.sum(cm)]\nlabels = [f'{v1}\\n{v2}' for v1, v2 in zip(names, percentages)]\nlabels = np.asarray(labels).reshape(2,2)\nticklabels = ['Normal', 'Pneumonia']\n\n# Create confusion matrix as heatmap\nsns.set(font_scale = 1.4)\nax = sns.heatmap(cm, annot=labels, fmt='', cmap='Oranges', xticklabels=ticklabels, yticklabels=ticklabels )\nplt.xticks(size=12)\nplt.yticks(size=12)\nplt.title(\"Confusion Matrix\") #plt.title(\"Confusion Matrix\\n\", fontsize=10)\nplt.xlabel(\"Predicted\", size=14)\nplt.ylabel(\"Actual\", size=14) \n#plt.savefig('cm.png', transparent=True) \n\n## ROC CURVE\nplt.subplot(122)\nfpr, tpr, thresholds = roc_curve(y_true, y_pred_prob)\nauc = roc_auc_score(y_true, y_pred_prob)\nplt.title('ROC Curve')\nplt.plot([0, 1], [0, 1], 'k--', label = \"Random (AUC = 50%)\")\nplt.plot(fpr, tpr, label='CNN (AUC = {:.2f}%)'.format(auc*100))\nplt.xlabel('False Positive Rate', size=14)\nplt.ylabel('True Positive Rate', size=14)\nplt.legend(loc='best')\n#plt.savefig('roc.png', bbox_inches='tight', pad_inches=1)\n\n## END PLOTS\nplt.tight_layout()\n\n## Summary Statistics\nTN, FP, FN, TP = cm.ravel() # cm[0,0], cm[0, 1], cm[1, 0], cm[1, 1]\nacc = (TP + TN) / np.sum(cm) # % positive out of all predicted positives\nprecision = TP / (TP+FP) # % positive out of all predicted positives\nrecall =  TP / (TP+FN) # % positive out of all supposed to be positives\nspecificity = TN / (TN+FP) # % negative out of all supposed to be negatives\nf1 = 2*precision*recall / (precision + recall)\nstats_summary = '[Summary Statistics]\\nAccuracy = {:.2%} | Precision = {:.2%} | Recall = {:.2%} | Specificity = {:.2%} | F1 Score = {:.2%}'.format(acc, precision, recall, specificity, f1)\nprint(stats_summary)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')\ndf_test.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH_TEST_PT_FILES = '../input/melanoma-grayscale-64x64/images64x64.pt/test'\nBATCH_SIZE = 128\n\ntest_ds = MelanomaImageDataset(PATH_TEST_PT_FILES, df_test)\ntest_dl = DataLoader(test_ds, batch_size=BATCH_SIZE, num_workers=32, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = []\nfor image in test_dl:\n    preds.append(model(image).detach())\ny_pred_prob = torch.cat(preds, dim=-1).cpu().numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../input/siim-isic-melanoma-classification/sample_submission.csv')\nsub['target'] = y_pred_prob\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}