{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Predicting melanoma in pytorch using Resnet18","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\n%config InlineBackend.figure_format = 'retina'\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport os\nfrom collections import OrderedDict\nimport torch\nfrom torch import nn\nfrom torch import optim\nimport torch.nn.functional as F\nfrom torchvision import datasets, transforms, models\n\n#import custom_models\n\n#python packages\nfrom PIL import Image\nfrom tqdm import tqdm\n#from tqdm.notebook import tqdm\nimport gc\nimport datetime\nimport copy\nimport matplotlib.pyplot as plt\nimport time\nfrom skimage import io\n#torch\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader, random_split\n#torchvision\nimport torchvision\nfrom torchvision import datasets, models, transforms\nprint(\"PyTorch Version: \",torch.__version__)\nprint(\"Torchvision Version: \",torchvision.__version__)\n\nimport random\nimport cv2\nimport warnings\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Most of the pretrained models require the input to be 224x224 images. Also, we'll need to match the normalization used when the models were trained. Each color channel was normalized separately, the means are `[0.485, 0.456, 0.406]` and the standard deviations are `[0.229, 0.224, 0.225]`.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#download the pretrained model\nimport torchvision.models as models\nmodel = models.resnet18(pretrained = False)\nmodel\n\n#switch device to gpu if available\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class AdvancedHairAugmentation:\n    \"\"\"\n    Impose an image of a hair to the target image\n\n    Args:\n        hairs (int): maximum number of hairs to impose\n        hairs_folder (str): path to the folder with hairs images\n    \"\"\"\n\n    def __init__(self, hairs: int = 5, hairs_folder: str = \"\"):\n        self.hairs = hairs\n        self.hairs_folder = hairs_folder\n\n    def __call__(self, img_path):\n        \"\"\"\n        Args:\n            img (PIL Image): Image to draw hairs on.\n\n        Returns:\n            PIL Image: Image with drawn hairs.\n        \"\"\"\n        img = cv2.imread(img_path)\n        n_hairs = random.randint(1, self.hairs)\n        \n        if not n_hairs:\n            return img\n        \n        height, width, _ = img.shape  # target image width and height\n        hair_images = [im for im in os.listdir(self.hairs_folder) if 'png' in im]\n        \n        for _ in range(n_hairs):\n            hair = cv2.imread(os.path.join(self.hairs_folder, random.choice(hair_images)))\n            hair = cv2.flip(hair, random.choice([-1, 0, 1]))\n            hair = cv2.rotate(hair, random.choice([0, 1, 2]))\n\n            h_height, h_width, _ = hair.shape  # hair image width and height\n            roi_ho = random.randint(0, img.shape[0] - hair.shape[0])\n            roi_wo = random.randint(0, img.shape[1] - hair.shape[1])\n            roi = img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]\n\n            # Creating a mask and inverse mask\n            img2gray = cv2.cvtColor(hair, cv2.COLOR_BGR2GRAY)\n            ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)\n            mask_inv = cv2.bitwise_not(mask)\n\n            # Now black-out the area of hair in ROI\n            img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)\n\n            # Take only region of hair from hair image.\n            hair_fg = cv2.bitwise_and(hair, hair, mask=mask)\n\n            # Put hair in ROI and modify the target image\n            dst = cv2.add(img_bg, hair_fg)\n\n            img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)       \n        return img\n\n    def __repr__(self):\n        return f'{self.__class__.__name__}(hairs={self.hairs}, hairs_folder=\"{self.hairs_folder}\")'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader, ConcatDataset\nfrom PIL import Image\nimport torchvision\nclass MultimodalDataset(Dataset):\n    \"\"\"\n    Custom dataset definition\n    \"\"\"\n    def __init__(self, csv_path, img_path, mode='train', transform=None):\n        \"\"\"\n        \"\"\"\n        self.df = pd.read_csv(csv_path)\n        self.img_path = img_path\n        self.mode= mode\n        self.transform = transform\n        \n            \n    def __getitem__(self, index):\n        \"\"\"\n        \"\"\"\n        img_name = self.df.iloc[index][\"image_name\"] + '.jpg'\n        img_path = os.path.join(self.img_path, img_name)\n        image = Image.open(img_path)\n        \n        dtype = torch.cuda.FloatTensor if torch.cuda.is_available() else torch.FloatTensor # ???\n        \n        if self.mode == 'train':\n            if self.df.iloc[index][\"augmented\"]==1:\n                image = AdvancedHairAugmentation(hairs_folder=\"../input/melanoma-hairs\")(img_path)\n                image = Image.fromarray(image, 'RGB')\n            elif self.df.iloc[index][\"augmented\"]==2:\n                image = AdvancedHairAugmentation(hairs_folder=\"../input/melanoma-hairs\")(img_path)\n                image = Image.fromarray(image, 'RGB')\n            else:  \n                image = image.convert(\"RGB\")\n                \n            image = np.asarray(image)\n            if self.transform is not None:\n                image = self.transform(image)\n            labels = self.df.iloc[index][\"target\"]\n            return image, labels\n            \n        elif self.mode == 'val':\n            image = np.asarray(image)\n            if self.transform is not None:\n                image = self.transform(image)\n            labels = self.df.iloc[index][\"target\"]\n            return image, labels\n        \n        else: #when self.mode=='test'\n            image = np.asarray(image)\n            if self.transform is not None:\n                image = self.transform(image)\n            return image, self.df.iloc[index][\"image_name\"]\n\n    def __len__(self):\n        return len(self.df)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataloaders(input_size, batch_size, augment=False, shuffle = True):\n    # How to transform the image when you are loading them.\n    # you'll likely want to mess with the transforms on the training set.\n    \n    # For now, we resize/crop the image to the correct input size for our network,\n    # then convert it to a [C,H,W] tensor, then normalize it to values with a given mean/stdev. These normalization constants\n    # are derived from aggregating lots of data and happen to produce better results.\n    data_transforms = {\n        'train': transforms.Compose([\n            transforms.ToPILImage(),\n            transforms.Resize(input_size),\n            transforms.CenterCrop(input_size),\n            transforms.ToTensor(),\n            transforms.Normalize([0.5], [0.225])\n        ]),\n        'val': transforms.Compose([\n            transforms.ToPILImage(),\n            transforms.Resize(input_size),\n            transforms.CenterCrop(input_size),\n            transforms.ToTensor(),\n            transforms.Normalize([0.5], [0.225])\n        ]),\n        'test': transforms.Compose([\n            transforms.ToPILImage(),\n            transforms.Resize(input_size),\n            transforms.CenterCrop(input_size),\n            transforms.ToTensor(),\n            transforms.Normalize([0.5], [0.225])\n        ])\n    }\n    \n    data_subsets = {x: MultimodalDataset(csv_path=\"../input/melanoma/\"+x+\".csv\", \n                                         img_path = image_path_dict[x],\n                                         mode = x,\n                                         transform=data_transforms[x]) for x in data_transforms.keys()}\n    # Create training and validation dataloaders\n    # Never shuffle the test set\n    dataloaders_dict = {x: DataLoader(data_subsets[x], batch_size=batch_size, shuffle=False if x != 'train' else shuffle, num_workers=4) for x in data_transforms.keys()}\n    return dataloaders_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_path_dict = {'train': \"../input/siim-isic-melanoma-classification/jpeg/train\",\n                  'val': \"../input/siim-isic-melanoma-classification/jpeg/train\" ,\n                  'test': \"../input/siim-isic-melanoma-classification/jpeg/test\"}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataloaders = get_dataloaders(input_size=224, batch_size=64, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_loader = dataloaders['train']\nval_loader = dataloaders['val']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_loader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_loader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":false},"cell_type":"markdown","source":"samples, labels = iter(trainloader).next()\nplt.figure(figsize=(16,24))\ngrid_imgs = torchvision.utils.make_grid(samples[:24])\nnp_grid_imgs = grid_imgs.numpy()\n'''in tensor, image is (batch, width, height), so you have to transpose it to (width, height, batch) in numpy to show it.'''\nplt.imshow(np.transpose(np_grid_imgs, (1,2,0)))","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Training Only the Classifier","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#Freeze the parameters \n#for param in model.parameters():\n    #param.requires_grad = False ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Classifier architecture to put on top of resnet18\nfc = nn.Sequential(OrderedDict([\n    ('fc1', nn.Linear(512,100)),\n    ('relu', nn.ReLU()),\n    ('fc2', nn.Linear(100,2)),\n    ('output', nn.LogSoftmax(dim=1))\n]))\n\nmodel.fc = fc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#shifting model to gpu\nmodel.to(device)\nmodel","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_epochs = 75\nnum_classes = 2\nbatch_size = 32\nlearning_rate = 0.01","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train():\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n    criterion = nn.CrossEntropyLoss()\n    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n\n    total_step = len(train_loader)\n    \n    #take out the following code if just starting to train\n    checkpoint = torch.load('../input/shuffled-img-conf-matrix-models/model10.pth')\n    model.load_state_dict(checkpoint['state_dict'])\n    optimizer.load_state_dict(checkpoint['optimizer'])\n    epoch_before = checkpoint['epoch']\n    \n    for epoch in range(num_epochs):\n        running_loss = 0.0\n        for i, (inputs, labels) in enumerate(tqdm(train_loader), 1):\n            inputs = inputs.to(device)\n            labels = labels.to(device)\n            optimizer.zero_grad()\n            \n            outputs = model(inputs)\n            outputs = torch.squeeze(outputs)\n            \n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            running_loss += loss.item()\n        \n        #'epoch' = epoch if just starting to train\n        #'epoch' = epoch+epoch_before+1 afterwards\n        model_file = { 'epoch': epoch+epoch_before+1,\n                      'state_dict': model.state_dict(),\n                      'optimizer' : optimizer.state_dict()}\n\n        torch.save(model_file, \"model\" + str(epoch+epoch_before+1) + '.pth')  \n        #str(epoch) if just starting to train \n        #str(epoch+epoch_before+1) afterwards\n\n        model.eval()\n        \n        train_real = []\n        train_pred = []\n        train_correct = 0\n        train_total = 0\n        with torch.no_grad():\n            for data in tqdm(train_loader):\n                images, labels = data\n                \n                images = images.to(device)\n                labels = labels.to(device)\n\n                outputs = model(images)\n                _, predicted = torch.max(outputs.data, 1)\n\n                train_total += labels.size(0)\n\n                train_correct += (predicted == labels).sum().item()\n                train_real.append(labels)\n                train_pred.append(predicted)\n        #str(epoch) if just starting to train\n        #str(epoch+epoch_before+1) afterwards\n        print(\"epoch: \" + str(epoch+epoch_before+1))\n        print('Top One Error of the network on train images: %d %%' % (\n                100 * (1 - train_correct / train_total)))\n        print(\"train_real\", train_real)\n        print(\"train_pred\", train_pred)\n\n        val_real = []\n        val_pred = []\n        val_correct = 0\n        val_total = 0\n        with torch.no_grad():\n            for data in tqdm(val_loader):\n                images, labels = data\n\n                images = images.to(device)\n                labels = labels.to(device)\n\n                outputs = model(images)\n                _, predicted = torch.max(outputs.data, 1)\n\n                val_total += labels.size(0)\n\n                val_correct += (predicted == labels).sum().item()\n                val_real.append(labels)\n                val_pred.append(predicted)\n        #str(epoch) if just starting to train\n        #str(epoch+epoch_before+1) afterwards\n        print(\"epoch: \" + str(epoch+epoch_before+1))\n        print('Top One Error of the network on validation images: %d %%' % (\n                100 * (1 - val_correct / val_total)))\n        print(\"val_real\", val_real)\n        print(\"val_pred\", val_pred)\n        \n        gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#photo only resnet18 pretrained=false, for the confusion matrix to compare with resnet+metadata","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"test_loader = dataloaders['test']","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Prediction on Testset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"markdown","source":"checkpoint = torch.load('../input/trained-model/model39.pth')\nmodel.load_state_dict(checkpoint['state_dict'])\n\nmodel.eval()\nfn_list = []\npred_list = []\nfor x, fn in test_loader:\n    x = x.to(device)\n    output = model(x)\n    pred = torch.argmax(output, dim=1)\n    fn_list += fn\n    pred_list += [p.item() for p in pred]\n\nsubmission = pd.DataFrame({\"image_name\":fn_list, \"target\":pred_list})\nsubmission.to_csv('preds_resnet18.csv', index=False)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Sample some images and their predicted labels","execution_count":null},{"metadata":{"trusted":true},"cell_type":"markdown","source":"samples, _ = iter(testloader).next()\nsamples = samples.to(device)\nfig = plt.figure(figsize=(24, 16))\nfig.tight_layout()\noutput = model(samples[:24])\npred = torch.argmax(output, dim=1)\npred = [p.item() for p in pred]\nad = {0:'benign', 1:'malignant'}\nfor num, sample in enumerate(samples[:24]):\n    plt.subplot(4,6,num+1)\n    plt.title(ad[pred[num]])\n    plt.axis('off')\n    sample = sample.cpu().numpy()\n    plt.imshow(np.transpose(sample, (1,2,0)))","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"References:<br>\n[densenet](https://www.kaggle.com/jaeboklee/pytorch-cat-vs-dog) <br>\n[Udacity Transfer Learning](https://classroom.udacity.com/courses/ud188/lessons/c5706f76-0e30-4b48-b74e-c19fafc33a75/concepts/c33dec4c-ff16-465f-88e9-e95365e7b522)","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}