{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Prediting Melanoma in Pytorch (using images and metadata)","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nfrom skimage import io\n\nimport copy\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader, random_split\n\nimport torchvision\nfrom torchvision import datasets, models, transforms\n\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\nimport gc\nimport cv2\nimport time\nimport datetime\nimport warnings\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm\n%matplotlib inline\n\nfrom PIL import Image\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_CNN(model_name, num_classes):\n    # Initialize these variables which will be set in this if statement. Each of these variables is model specific.\n    # The model (nn.Module) to return\n    model_ft = None\n    # The input image is expected to be (input_size, input_size)\n    input_size = 0\n    \n    # You may NOT use pretrained models!! This is because these models have been \n    # trained on classifing images such as keyboard, mouse, pencil, animals, and such (NOT skin lesion pics)\n    use_pretrained = False\n    \n    # By default, all parameters will be trained (useful when you're starting from scratch)\n    # Within this function you can set .requires_grad = False for various parameters, if you don't want to learn them\n\n    if model_name == \"resnet\":\n        \"\"\" Resnet18\n        \"\"\"\n        model_ft = models.resnet18(pretrained=use_pretrained)\n        num_ftrs = model_ft.fc.in_features\n        model_ft.fc = nn.Linear(num_ftrs, 1)\n        input_size = 224\n        \n    elif model_name == \"vgg16\":\n        \"\"\" VGG16_bn\n        \"\"\"\n        model_ft = models.vgg16_bn(pretrained=use_pretrained)\n        num_ftrs = model_ft.classifier[6].in_features\n        model_ft.classifier[6] = nn.Linear(num_ftrs, 1)\n        input_size = 224\n\n    elif model_name == \"densenet\": #other model to potentially try, but we couldn't in our paper #broke student #no gpu access\n        \"\"\" Densenet\n        \"\"\"\n        model_ft = models.densenet121(pretrained=use_pretrained)\n        num_ftrs = model_ft.classifier.in_features\n        model_ft.classifier = nn.Linear(num_ftrs, 1)\n        input_size = 224\n        \n    else:\n        raise Exception(\"Invalid model name!\")\n    \n    return model_ft, input_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Melanoma_Net(nn.Module):\n    \"\"\"\n    fc1: number of neurons in the hidden fully connected layer\n    \"\"\"\n    def __init__(self, cnn_model_name, num_classes, num_multimodal_features=9, fc1_out=32):\n        #num_classes = 1\n        #num_multimodal_features= 9\n        super(Melanoma_Net, self).__init__()\n        self.cnn, self.input_size = make_CNN(cnn_model_name, num_classes)\n        #define output layers\n        self.fc1 = nn.Linear(num_classes + num_multimodal_features, fc1_out) #takes in input of CNN and multimodal input\n        self.fc2 = nn.Linear(fc1_out, num_classes)\n        \n    def forward(self, image, data):\n        x1 = self.cnn(image)\n        #print(\"x1\", x1.shape)\n        x2 = data\n        #print(\"x2\", x2.shape)\n        #print(\"x1: \", x1, type(x1))\n        #print(\"x2: \", x2, type(x2))\n        #x = torch.cat((x1, x2), dim=1)  \n        x = torch.cat((x1.float(), x2.float()), dim=1)\n        #print(\"concat\", x.shape)\n        x = F.relu(self.fc1(x))\n        #print(\"relu\", x.shape)\n        x = self.fc2(x)\n        #print('forward output: ', x)\n        #print(\"fc2\", x.shape)\n        return x.double()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AdvancedHairAugmentation:\n    \"\"\"\n    Impose an image of a hair to the target image\n\n    Args:\n        hairs (int): maximum number of hairs to impose\n        hairs_folder (str): path to the folder with hairs images\n    \"\"\"\n\n    def __init__(self, hairs: int = 5, hairs_folder: str = \"\"):\n        self.hairs = hairs\n        self.hairs_folder = hairs_folder\n\n    def __call__(self, img_path):\n        \"\"\"\n        Args:\n            img (PIL Image): Image to draw hairs on.\n\n        Returns:\n            PIL Image: Image with drawn hairs.\n        \"\"\"\n        img = cv2.imread(img_path)\n        n_hairs = random.randint(1, self.hairs) # takes random number of img hairs and adds to image\n        \n        if not n_hairs:\n            return img\n        \n        height, width, _ = img.shape  # target image width and height\n        hair_images = [im for im in os.listdir(self.hairs_folder) if 'png' in im]\n        \n        for _ in range(n_hairs):\n            hair = cv2.imread(os.path.join(self.hairs_folder, random.choice(hair_images)))\n            \n            # Randomized flip/rotation of hair strand\n            hair = cv2.flip(hair, random.choice([-1, 0, 1]))\n            hair = cv2.rotate(hair, random.choice([0, 1, 2]))\n\n            h_height, h_width, _ = hair.shape  # hair image width and height\n            roi_ho = random.randint(0, img.shape[0] - hair.shape[0])\n            roi_wo = random.randint(0, img.shape[1] - hair.shape[1])\n            roi = img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]\n\n            # Creating a mask and inverse mask\n            img2gray = cv2.cvtColor(hair, cv2.COLOR_BGR2GRAY)\n            ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)\n            mask_inv = cv2.bitwise_not(mask)\n\n            # Now black-out the area of hair in ROI\n            img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)\n\n            # Take only region of hair from hair image.\n            hair_fg = cv2.bitwise_and(hair, hair, mask=mask)\n\n            # Put hair in ROI and modify the target image\n            dst = cv2.add(img_bg, hair_fg)\n\n            img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) # converts image color back to regular color (instead of producing blue tinted image)        \n        return img\n\n    def __repr__(self):\n        return f'{self.__class__.__name__}(hairs={self.hairs}, hairs_folder=\"{self.hairs_folder}\")'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MultimodalDataset(Dataset):\n    \"\"\"\n    Custom dataset definition\n    \"\"\"\n    def __init__(self, csv_path, img_path, mode='train', transform=None):\n        \"\"\"\n        \"\"\"\n        self.df = pd.read_csv(csv_path)\n        self.img_path = img_path\n        self.mode= mode\n        self.transform = transform\n        \n            \n    def __getitem__(self, index):\n        \"\"\"\n        \"\"\"\n        img_name = self.df.iloc[index][\"image_name\"] + \".jpg\"\n        img_path = os.path.join(self.img_path, img_name)\n        image = Image.open(img_path)\n\n        dtype = torch.cuda.FloatTensor if torch.cuda.is_available() else torch.FloatTensor # ???\n        \n        if self.mode == 'train':\n            #augments malignant images with hair twice\n            if self.df.iloc[index][\"augmented\"]==1:\n                image = AdvancedHairAugmentation(hairs_folder=\"../input/melanoma-hairs\")(img_path)\n                image = Image.fromarray(image, 'RGB')\n            elif self.df.iloc[index][\"augmented\"]==2:\n                image = AdvancedHairAugmentation(hairs_folder=\"../input/melanoma-hairs\")(img_path)\n                image = Image.fromarray(image, 'RGB')\n            else:  \n                image = image.convert(\"RGB\")\n                \n            image = np.asarray(image)\n            if self.transform is not None:\n                image = self.transform(image)\n            labels = torch.tensor(self.df.iloc[index][\"target\"], dtype = torch.float64)\n            \n            features = np.fromstring(self.df.iloc[index][\"features\"][1:-1], sep=\",\") #turns features into an array\n            features = torch.from_numpy(features.astype(\"float\")) #turns the features array into a vector\n            return image, features, labels\n            \n        elif self.mode == 'val':\n            image = np.asarray(image)\n            if self.transform is not None:\n                image = self.transform(image)\n            labels = torch.tensor(self.df.iloc[index][\"target\"], dtype = torch.float64)\n            \n            features = np.fromstring(self.df.iloc[index][\"features\"][1:-1], sep=\",\") #turns features into an array\n            features = torch.from_numpy(features.astype(\"float\")) #turns the features array into a vector\n            return image, features, labels\n        \n        else: #when self.mode=='test'\n            image = np.asarray(image)\n            if self.transform is not None:\n                image = self.transform(image)\n            features = np.fromstring(self.df.iloc[index][\"features\"][1:-1], sep=\",\") #turns features into an array\n            features = torch.from_numpy(features.astype(\"float\")) #turns the features array into a vector\n            return image, features, self.df.iloc[index][\"image_name\"]\n\n    def __len__(self):\n        return len(self.df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_path_dict = {'train': \"../input/siim-isic-melanoma-classification/jpeg/train\",\n                  'val': \"../input/siim-isic-melanoma-classification/jpeg/train\" ,\n                  'test': \"../input/siim-isic-melanoma-classification/jpeg/test\"}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataloaders(input_size, batch_size, augment=False, shuffle = True):\n    # How to transform the image when you are loading them.\n    # Maybe want to mess with the transforms on the training set?\n    \n    # For now, we resize/crop the image to the correct input size for our network,\n    # then convert it to a [C,H,W] tensor, then normalize it to values with a given mean/stdev. These normalization constants\n    # are derived from aggregating lots of data and happen to produce better results.\n    data_transforms = {\n        'train': transforms.Compose([\n            transforms.ToPILImage(),\n            transforms.Resize(input_size),\n            transforms.CenterCrop(input_size),\n            transforms.ToTensor(),\n            transforms.Normalize([0.5], [0.225])\n        ]),\n        'val': transforms.Compose([\n            transforms.ToPILImage(),\n            transforms.Resize(input_size),\n            transforms.CenterCrop(input_size),\n            transforms.ToTensor(),\n            transforms.Normalize([0.5], [0.225])\n        ]),\n        'test': transforms.Compose([\n            transforms.ToPILImage(),\n            transforms.Resize(input_size),\n            transforms.CenterCrop(input_size),\n            transforms.ToTensor(),\n            transforms.Normalize([0.5], [0.225])\n        ])\n    }\n    # Create training and validation datasets\n    data_subsets = {x: MultimodalDataset(csv_path=\"../input/melanoma/\" + x + \".csv\", \n                                         img_path = image_path_dict[x],\n                                         mode = x,\n                                         transform=data_transforms[x]) for x in data_transforms.keys()}\n    \n    # Create training and validation dataloaders\n    # Never shuffle the test set\n    dataloaders_dict = {x: DataLoader(data_subsets[x], batch_size=batch_size, shuffle=False if x != 'train' else shuffle, num_workers=4) for x in data_transforms.keys()}\n    return dataloaders_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(model, dataloaders, criterion, optimizer, \n                scheduler, model_name=str(datetime.datetime.now()), \n                save_dir = None, num_epochs=25):\n    '''\n    model: The NN to train\n    dataloaders: A dictionary containing at least the keys \n                 'train','val' that maps to Pytorch data loaders for the dataset\n    criterion: The Loss function\n    optimizer: The algorithm to update weights \n               (Variations on gradient descent)\n    num_epochs: How many epochs to train for\n    save_dir: Where to save the best model weights that are found, \n              as they are found. Will save to save_dir/weights_best.pt\n              Using None will not write anything to disk\n    '''\n    since = time.time()\n\n    val_acc_history = []\n    \n    #take out if just starting to train\n    #checkpoint = torch.load(\"INSERT PREVIOUS MODEL PATH\")   for ex: torch.load(\"../input/vgg16-models/vgg16_weights_14.pt\")\n    #model.load_state_dict(checkpoint['state_dict'])\n    #optimizer.load_state_dict(checkpoint['optimizer'])\n    #epoch_before = checkpoint['epoch']\n    \n    best_acc = 0.0\n\n    for epoch in range(num_epochs):\n        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n        print('-' * 10)\n\n        # Each epoch has a training and validation phase\n        for phase in ['train', 'val']:\n            val_pred = []\n            train_pred = []\n            if phase == 'train':\n                model.train()  # Set model to training mode\n            else:\n                model.eval()   # Set model to evaluate mode\n\n            running_loss = 0.0\n            running_corrects = 0\n\n            # Iterate over data.\n            # TQDM has nice progress bars\n            for inputs, features, labels in tqdm(dataloaders[phase]):\n                inputs = inputs.to(device)\n                features = features.to(device)\n                labels = labels.to(device)\n\n                # zero the parameter gradients\n                optimizer.zero_grad()\n\n                # forward\n                # track history if only in train\n                with torch.set_grad_enabled(phase == 'train'):\n                    # Get model outputs and calculate loss\n                    outputs = model(inputs, features)\n                    outputs = torch.squeeze(outputs)\n                    labels = torch.squeeze(labels)\n                    #print(\"model outputs: \", outputs, outputs.size())\n                    #print(\"model labels: \", labels.size())\n                    loss = criterion(outputs, labels)\n\n                    # torch.max outputs the maximum value, and its index\n                    # Since the input is batched, we take the max along axis 1\n                    # (the meaningful outputs)\n                    # print(\"outputs: \", outputs)\n                    preds = (outputs > 0).type(torch.float64)\n                    \n                    # backprop + optimize only if in training phase\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n                        train_pred.append(preds)\n                        \n                    if phase == 'val':\n                        val_pred.append(preds)\n                        \n                #statistics\n                #print(\"loss: \", loss.item())\n                #print(\"inputs: \", inputs.size(0), inputs.size())\n                \n                running_loss += loss.item() * inputs.size(0)\n                #print(\"running loss: \", running_loss)\n                running_corrects += torch.sum(preds == labels.data)\n                #print(\"running corrects: \", running_corrects)\n                \n            #print(\"train_pred\", train_pred)    \n            #print(\"val_pred\", val_pred)\n            epoch_loss = running_loss / len(dataloaders[phase].dataset)\n            #is the accuracy calculated correctly?\n            #print(\"running_corrects: \", running_corrects.double(), running_corrects.size())\n            #print(\"dataloaders len: \", len(dataloaders[phase].dataset))\n            \n            epoch_acc = 100* running_corrects.double() / (len(dataloaders[phase].dataset))\n            epoch_acc = epoch_acc.item()\n\n            print('{} Loss: {:.4f} Acc: {:.4f}%'.format(phase, epoch_loss, epoch_acc))\n            \n            #copy the model\n            if phase == 'val':\n                model_file = { 'epoch': epoch,\n                      'state_dict': model.state_dict(),\n                      'optimizer' : optimizer.state_dict()}\n                torch.save(model_file, \"{}_weights_{}.pt\".format(model_name, epoch))\n                #epoch if just started training\n                #epoch + epoch_before + 1 afterwards\n                val_acc_history.append(epoch_acc)\n            if phase == 'val' and epoch_acc > best_acc:\n                best_acc = epoch_acc\n                \n        print()\n        scheduler.step()\n    time_elapsed = time.time() - since\n    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n    print('Best Val Acc: {:4f}'.format(best_acc))\n\n    #return model, val_acc_history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_optimizer(model):\n    #Get all the parameters\n    params_to_update = model.parameters()\n    print(\"Params to learn:\")\n    for name, param in model.named_parameters():\n        if param.requires_grad == True:\n            print(\"\\t\",name)\n\n    #Use SGD\n    optimizer = optim.SGD(params_to_update, lr=0.01, momentum=0.9)\n    return optimizer\n\ndef get_loss(num_classes,device):\n    #Create an instance of the loss function\n    #We could set weights to account for unbalanced data but we have augmented the malignant data to have around the same number of benign/malig\n    \n    pos_weight = torch.tensor(np.ones(num_classes))\n    pos_weight = pos_weight.to(device)\n\n    criterion = nn.BCEWithLogitsLoss()\n    return criterion","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Models to choose from [resnet, vgg16, densenet]\n# You can add your own, or modify these however you wish!\nmodel_name = 'vgg16'\n\n# Number of classes in the dataset\nnum_classes = 1\n\n# Batch size for training (change depending on how much memory you have)\n# You should use a power of 2.\nbatch_size = 32\n\n# Shuffle the input data?\nshuffle_datasets = True\n\n# Number of epochs to train for \nnum_epochs = 10\n\n#Directory to save weights to\n#save_dir = \"weights\"\n#os.makedirs(save_dir, exist_ok=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train!","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Initialize the model for this run\nmodel = Melanoma_Net(cnn_model_name = model_name, num_classes = num_classes)\ninput_size = model.input_size\n    \ndataloaders = get_dataloaders(input_size, batch_size, shuffle = shuffle_datasets)\ncriterion = get_loss(num_classes=num_classes,device=device)\n\n#Move the model to the gpu if needed\nmodel = model.to(device)\n\noptimizer = make_optimizer(model)\nscheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1)\n#scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[5,10],gamma=0.1)\n\n#Train the model!\ntrain_model(model=model,\n            dataloaders=dataloaders, criterion=criterion, optimizer=optimizer,\n            scheduler=scheduler, model_name=model_name, num_epochs=num_epochs)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predict on test csv","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Melanoma_Net(cnn_model_name = model_name, num_classes = num_classes)\ncheckpoint = torch.load('MODEL PATH HERE')\nmodel.load_state_dict(checkpoint['state_dict'])\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\ninput_size = model.input_size\ndataloaders = get_dataloaders(input_size, batch_size, shuffle = shuffle_datasets)\ntest_loader = dataloaders['test']\n\nif torch.cuda.is_available():\n    model.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.eval()\n\nfn_list = [] \npred_list = []\n\nfor inputs, features, fn in test_loader: \n    inputs = inputs.to(device) \n    features = features.to(device) \n    output = model(inputs, features) \n    pred = (output > 0).type(torch.float64) \n    fn_list += fn \n    pred_list += [p.item() for p in pred]\n\nsubmission = pd.DataFrame({\"image_name\":fn_list, \"target\":pred_list}) \nsubmission.to_csv('prediction.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}