{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"I would like to experiment many CNN architectures on the skin cancer images dataset in order to distinguish which model is appropriate for this kind of images.\nI basically followed this tutorial from Pytorch:\nhttps://pytorch.org/tutorials/beginner/finetuning_torchvision_models_tutorial.html","metadata":{}},{"cell_type":"code","source":"!pip install torchviz","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Importing the necessary libraries","metadata":{}},{"cell_type":"code","source":"from __future__ import print_function\nfrom __future__ import division\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport numpy as np\nimport pandas as pd\nimport torchvision\nfrom torch.utils.data import Dataset\nfrom torchviz import make_dot\nfrom PIL import Image\nimport matplotlib.image as mpimg\nfrom torch.utils.data import DataLoader\nfrom torchvision import datasets, models, transforms\nimport matplotlib.pyplot as plt\nimport time\nimport os\nimport copy\nprint(\"PyTorch Version: \",torch.__version__)\nprint(\"Torchvision Version: \",torchvision.__version__)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Defining the dataset source path, selected model archtecture and parameters","metadata":{}},{"cell_type":"code","source":"# Top level data directory. Here we assume the format of the directory conforms\n#   to the ImageFolder structure\ndata_dir = \"../input/siim-isic-melanoma-classification/jpeg/train/\"\ndata_csv_file = \"../input/siim-isic-melanoma-classification/train.csv\"\n\n# Models to choose from [resnet, alexnet, vgg, squeezenet, densenet, inception]\nmodel_name = \"inception\"\n\n# Number of classes in the dataset\nnum_classes = 2\n\n# Batch size for training (change depending on how much memory you have)\nbatch_size = 8\n\n# Number of epochs to train for\nnum_epochs = 10\n\n# Flag for feature extracting. When False, we finetune the whole model,\n# when True we only update the last layer weights (classification layer)\nfeature_extract = False\n\n# Detect if we have a GPU available\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Defining the training function","metadata":{}},{"cell_type":"code","source":"def train_model(model, dataloaders, criterion, optimizer, num_epochs=25, is_inception=False):\n    since = time.time()\n\n    val_acc_history = []\n\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n\n    for epoch in range(num_epochs):\n        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n        print('-' * 10)\n\n        # Each epoch has a training and validation phase\n        for phase in ['train', 'val']:\n            if phase == 'train':\n                model.train()  # Set model to training mode\n            else:\n                model.eval()   # Set model to evaluate mode\n\n            running_loss = 0.0\n            running_corrects = 0\n\n            # Iterate over data.\n            for inputs, labels in dataloaders[phase]:\n                inputs = inputs.to(device)\n                labels = labels.to(device)\n\n                # zero the parameter gradients\n                optimizer.zero_grad()\n\n                # forward\n                # track history if only in train\n                with torch.set_grad_enabled(phase == 'train'):\n                    # Get model outputs and calculate loss\n                    # Special case for inception because in training it has an auxiliary output. In train\n                    #   mode we calculate the loss by summing the final output and the auxiliary output\n                    #   but in testing we only consider the final output.\n                    if is_inception and phase == 'train':\n                        # From https://discuss.pytorch.org/t/how-to-optimize-inception-model-with-auxiliary-classifiers/7958\n                        outputs, aux_outputs = model(inputs)\n                        loss1 = criterion(outputs, labels)\n                        loss2 = criterion(aux_outputs, labels)\n                        loss = loss1 + 0.4*loss2\n                    else:\n                        outputs = model(inputs)\n                        loss = criterion(outputs, labels)\n\n                    _, preds = torch.max(outputs, 1)\n\n                    # backward + optimize only if in training phase\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n\n                # statistics\n                running_loss += loss.item() * inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data)\n\n            epoch_loss = running_loss / len(dataloaders[phase].dataset)\n            epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)\n\n            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n\n            # deep copy the model\n            if phase == 'val' and epoch_acc > best_acc:\n                best_acc = epoch_acc\n                best_model_wts = copy.deepcopy(model.state_dict())\n            if phase == 'val':\n                val_acc_history.append(epoch_acc)\n\n        print()\n\n    time_elapsed = time.time() - since\n    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n    print('Best val Acc: {:4f}'.format(best_acc))\n\n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    return model, val_acc_history","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_parameter_requires_grad(model, feature_extracting):\n    if feature_extracting:\n        for param in model.parameters():\n            param.requires_grad = False","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def initialize_model(model_name, num_classes, feature_extract, use_pretrained=True):\n    # Initialize these variables which will be set in this if statement. Each of these\n    #   variables is model specific.\n    model_ft = None\n    input_size = 0\n\n    if model_name == \"resnet\":\n        \"\"\" Resnet18\n        \"\"\"\n        model_ft = models.resnet18(pretrained=use_pretrained)\n        set_parameter_requires_grad(model_ft, feature_extract)\n        num_ftrs = model_ft.fc.in_features\n        model_ft.fc = nn.Linear(num_ftrs, num_classes)\n        input_size = 224\n\n    elif model_name == \"alexnet\":\n        \"\"\" Alexnet\n        \"\"\"\n        model_ft = models.alexnet(pretrained=use_pretrained)\n        set_parameter_requires_grad(model_ft, feature_extract)\n        num_ftrs = model_ft.classifier[6].in_features\n        model_ft.classifier[6] = nn.Linear(num_ftrs,num_classes)\n        input_size = 224\n\n    elif model_name == \"vgg\":\n        \"\"\" VGG11_bn\n        \"\"\"\n        model_ft = models.vgg11_bn(pretrained=use_pretrained)\n        set_parameter_requires_grad(model_ft, feature_extract)\n        num_ftrs = model_ft.classifier[6].in_features\n        model_ft.classifier[6] = nn.Linear(num_ftrs,num_classes)\n        input_size = 224\n\n    elif model_name == \"squeezenet\":\n        \"\"\" Squeezenet\n        \"\"\"\n        model_ft = models.squeezenet1_0(pretrained=use_pretrained)\n        set_parameter_requires_grad(model_ft, feature_extract)\n        model_ft.classifier[1] = nn.Conv2d(512, num_classes, kernel_size=(1,1), stride=(1,1))\n        model_ft.num_classes = num_classes\n        input_size = 224\n\n    elif model_name == \"densenet\":\n        \"\"\" Densenet\n        \"\"\"\n        model_ft = models.densenet121(pretrained=use_pretrained)\n        set_parameter_requires_grad(model_ft, feature_extract)\n        num_ftrs = model_ft.classifier.in_features\n        model_ft.classifier = nn.Linear(num_ftrs, num_classes)\n        input_size = 224\n\n    elif model_name == \"inception\":\n        \"\"\" Inception v3\n        Be careful, expects (299,299) sized images and has auxiliary output\n        \"\"\"\n        model_ft = models.inception_v3(pretrained=use_pretrained)\n        set_parameter_requires_grad(model_ft, feature_extract)\n        # Handle the auxilary net\n        num_ftrs = model_ft.AuxLogits.fc.in_features\n        model_ft.AuxLogits.fc = nn.Linear(num_ftrs, num_classes)\n        # Handle the primary net\n        num_ftrs = model_ft.fc.in_features\n        #model_ft.fc = nn.Linear(num_ftrs,num_classes)\n        model_ft.fc = nn.Sequential(\n                      nn.Linear(num_ftrs, 256), \n                      nn.ReLU(), \n                      nn.Dropout(0.4),\n                      nn.Linear(256, num_classes))  \n        input_size = 299\n\n    else:\n        print(\"Invalid model name, exiting...\")\n        exit()\n\n    return model_ft, input_size","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Let's observe our selected model","metadata":{}},{"cell_type":"code","source":"# Initialize the model for this run\nmodel_ft, input_size = initialize_model(model_name, num_classes, feature_extract, use_pretrained=True)\n\n# If we want to examine the various layers and blocks building \n# Print the model we just instantiated\nprint(model_ft)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We'll see the parameters that the model will track and optimize, as we set the extract features to False we'll get all the parameters as the model will optimize all layers parameters.","metadata":{}},{"cell_type":"code","source":"# Send the model to GPU\nmodel_ft = model_ft.to(device)\n\n#  Gather the parameters to be optimized/updated in this run. If we are\n#  finetuning we will be updating all parameters. However, if we are\n#  doing feature extract method, we will only update the parameters\n#  that we have just initialized, i.e. the parameters with requires_grad\n#  is True.\nparams_to_update = model_ft.parameters()\nprint(\"Params to learn:\")\nif feature_extract:\n    params_to_update = []\n    for name,param in model_ft.named_parameters():\n        if param.requires_grad == True:\n            params_to_update.append(param)\n            print(\"\\t\",name)\nelse:\n    for name,param in model_ft.named_parameters():\n        if param.requires_grad == True:\n            print(\"\\t\",name)\n\n# Observe that all parameters are being optimized\noptimizer_ft = optim.SGD(params_to_update, lr=0.001, momentum=0.9)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare dataloaders (Train, Validation)","metadata":{}},{"cell_type":"code","source":"class SkinCancerDataset(Dataset):\n    def __init__(self, csv_file, root_dir, transform=None):\n        self.annotations = pd.read_csv(csv_file).sample(5000) # We will just use random 1000 images from the training images\n        self.root_dir = root_dir\n        self.transform = transform\n    def __len__(self):\n        return len(self.annotations)\n    def __getitem__(self, index):\n        img_path = os.path.join(self.root_dir, self.annotations.iloc[index, 0]+\".jpg\")\n        #image = io.imread(img_path)\n        image = Image.open(img_path)\n        y_label = torch.tensor(int(self.annotations.iloc[index, 7]))\n\n        if self.transform:\n            image = self.transform(image)\n        return image, y_label\n        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I used here only 1000 images from the dataset to demonstrate how well the cnn architectures (ResNet, InceptionNet, VGG...) can be beneficial for us.\nYou can experiment it with large amount of data and see the performance of the model.","metadata":{}},{"cell_type":"code","source":"# Data augmentation and normalization for training\n# Just normalization for validation\ndata_transforms = {\n    'train': transforms.Compose([\n        transforms.Grayscale(num_output_channels=3),\n        transforms.RandomResizedCrop(input_size),\n        transforms.RandomRotation(degrees=25),\n        transforms.ColorJitter(),\n        transforms.RandomHorizontalFlip(),\n        transforms.ToTensor()\n    ]),\n    'val': transforms.Compose([\n        transforms.Grayscale(num_output_channels=3),\n        transforms.Resize(input_size),\n        transforms.CenterCrop(input_size),\n        transforms.ToTensor()\n    ]),\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Initializing Datasets and Dataloaders...\")\n\n# Creating our dataset\ntrain_dataset = SkinCancerDataset(csv_file=data_csv_file, root_dir=data_dir, transform=data_transforms['train'])\nprint(len(train_dataset))\n\ntrain_dataset, val_dataset = torch.utils.data.random_split(train_dataset, [4000, 1000])\n\n# Dataloader iterators, make sure to shuffle\ntrain_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\nval_dataloader = DataLoader(val_dataset, batch_size=batch_size, shuffle=True)\n\n# Create training and validation dataloaders\ndataloaders_dict = {'train': train_dataloader, 'val': val_dataloader}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# More sophisticated visualization when applying to a batch","metadata":{}},{"cell_type":"code","source":"# For more interesting view we can visualize the architecture thanks to torchviz\nbatch = next(iter(train_dataloader))\nyhat = model_ft(batch[0].to(device)) \nmake_dot(yhat)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train the model","metadata":{}},{"cell_type":"code","source":"# Setup the loss fxn\ncriterion = nn.CrossEntropyLoss()\n\n# Train and evaluate\nmodel_ft, hist = train_model(model_ft, \n                             dataloaders_dict, \n                             criterion, \n                             optimizer_ft, \n                             num_epochs=num_epochs, \n                             is_inception=(model_name==\"inception\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"# After training\n# Save our model\ncheckpoint_path = \"./InceptionV3_Skin_Cancer_classification_model.pth\"\ntorch.save(model_ft, checkpoint_path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict directly on an image","metadata":{}},{"cell_type":"code","source":"# Inference model\ncheckpoint_path = \"./InceptionV3_Skin_Cancer_classification_model.pth\"\n\n# Load in the model\nmodel_ft = torch.load(checkpoint_path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define a predict function for an unseen image\ndef predict(model, image_skin):\n    model.eval()\n    image = Image.open(image_skin)\n    img = mpimg.imread(image_skin)\n    imgplot = plt.imshow(img)\n    plt.show()\n    image = data_transforms['val'](image)\n    image = image.unsqueeze(0).to(device)\n    out = model(image)\n    _, pred = torch.max(out, dim=1)\n    print(pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# I will use a sample from the dataset here but you can dowload any other Skin Cancer image and apply the model on it\nim = os.path.join(data_dir, os.listdir(data_dir)[10])\nprint(im)\npredict(model_ft, im)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}