{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Step1: Unzipping File and Showing Statistics","metadata":{}},{"cell_type":"code","source":"import shutil\nshutil.unpack_archive('../input/melanoma-partial-augmentation-for-balance-512x512/Melanoma-JPEG-512.zip', './')\n\n\nimport os\nN_data = 53\n\npath, dirs, files = next(os.walk(\"./validation/benign\"))\nprint('Benign samples in validation set:',len(files))\n\npath, dirs, files = next(os.walk(\"./validation/malignant\"))\nprint('Malignant samples in validation set:',len(files))\n\nbenign_training_samples = []\nmalignant_training_samples = []\n\nfor iter_data in range(N_data):\n    path = './tr' + str(iter_data) + '/benign'\n    path, dirs, files = next(os.walk(path))\n    benign_training_samples.append(len(files))\n    #print('Benign samples in training set'+str(iter_data)+':',len(files))\n    \n    path = './tr' + str(iter_data) + '/malignant'\n    path, dirs, files = next(os.walk(path))\n    malignant_training_samples.append(len(files))\n    #print('Malignant samples in training set'+str(iter_data)+':',len(files))\n\nprint('Benign samples in training set:',benign_training_samples)\nprint('Malignant samples in training set:',malignant_training_samples)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T02:59:49.474717Z","iopub.execute_input":"2021-12-27T02:59:49.475055Z","iopub.status.idle":"2021-12-27T03:00:15.547611Z","shell.execute_reply.started":"2021-12-27T02:59:49.474975Z","shell.execute_reply":"2021-12-27T03:00:15.546036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Step2: DataLoader","metadata":{}},{"cell_type":"code","source":"from __future__ import print_function, division\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport numpy as np\nimport torchvision\nfrom torchvision import datasets, models, transforms\nimport matplotlib.pyplot as plt\nimport time\nimport copy\n\n\n\n\nplt.ion()   # interactive mode\n\n# Data augmentation and normalization for training\n# Just normalization for validation\ndata_transforms = {\n    0: transforms.Compose([\n        transforms.Resize(512),\n        transforms.Resize((560,560)),\n        transforms.RandomRotation(15,),\n        transforms.RandomResizedCrop(512),\n        transforms.RandomGrayscale(p=0.1),\n        transforms.RandomHorizontalFlip(),\n        transforms.RandomVerticalFlip(),\n        transforms.RandomPerspective(),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.507, 0.487, 0.441], std=[0.267, 0.256, 0.276])\n    ]),\n    1: transforms.Compose([\n        transforms.Resize(512),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.507, 0.487, 0.441], std=[0.267, 0.256, 0.276])\n    ]),\n}\n\ndata_dir = './'\ndata_folders = []\n\nfor iter_data in range(N_data):\n    data_folders.append('./tr' + str(iter_data))\ndata_folders.append('validation')\n    \nprint(data_folders)\n\nimage_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x),\n                                          data_transforms[x == 'validation'])\n                  for x in data_folders}\ndataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=14,\n                                             shuffle=True, num_workers=0)\n              for x in data_folders}\ndataset_sizes = {x: len(image_datasets[x]) for x in data_folders}\nclass_names = image_datasets['validation'].classes\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\ndef imshow(inp, title=None):\n    \"\"\"Imshow for Tensor.\"\"\"\n    inp = inp.numpy().transpose((1, 2, 0))\n    mean = np.array([0.485, 0.456, 0.406])\n    std = np.array([0.229, 0.224, 0.225])\n    inp = std * inp + mean\n    inp = np.clip(inp, 0, 1)\n    plt.imshow(inp)\n    if title is not None:\n        plt.title(title)\n    #plt.savefig('test.pdf', dpi = 300)\n    plt.pause(0.001)  # pause a bit so that plots are updated","metadata":{"execution":{"iopub.status.busy":"2021-12-27T03:00:15.550221Z","iopub.execute_input":"2021-12-27T03:00:15.550487Z","iopub.status.idle":"2021-12-27T03:00:17.667013Z","shell.execute_reply.started":"2021-12-27T03:00:15.550460Z","shell.execute_reply":"2021-12-27T03:00:17.666141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Step3: Observing Augmentation for Training and Validation Data ","metadata":{}},{"cell_type":"code","source":"# Get a batch of training data\ninputs, classes = next(iter(dataloaders[data_folders[10]]))\n\n# Make a grid from batch\nout = torchvision.utils.make_grid(inputs)\n\nimshow(out)#title=[class_names[x] for x in classes])","metadata":{"execution":{"iopub.status.busy":"2021-12-27T03:00:17.668835Z","iopub.execute_input":"2021-12-27T03:00:17.669300Z","iopub.status.idle":"2021-12-27T03:00:20.178816Z","shell.execute_reply.started":"2021-12-27T03:00:17.669264Z","shell.execute_reply":"2021-12-27T03:00:20.177548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get a batch of training data\ninputs, classes = next(iter(dataloaders['validation'])) #53rd element in data_folders\n\n# Make a grid from batch\nout = torchvision.utils.make_grid(inputs)\n\nimshow(out)# title=[class_names[x] for x in classes])","metadata":{"execution":{"iopub.status.busy":"2021-12-27T03:00:20.180731Z","iopub.execute_input":"2021-12-27T03:00:20.181215Z","iopub.status.idle":"2021-12-27T03:00:22.366597Z","shell.execute_reply.started":"2021-12-27T03:00:20.181145Z","shell.execute_reply":"2021-12-27T03:00:22.365825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Step4: Declaring Neural Networks\n\n#### Declaring several NNs. However, we are using one of the combinations. We may use another combination for investigating a different pre-trained NN or for investigating a different FC.","metadata":{}},{"cell_type":"code","source":"#model_name = 'vgg19_bn'\nmodel_ft = models.vgg19_bn(pretrained=True)\nnum_ftrs = model_ft.classifier[0].in_features\nlayer_width = 512 #Small for Resnet, large for VGG\n\n\n#model_ft = models.wide_resnet101_2(pretrained=True)\n#num_ftrs = model_ft.fc.in_features\n#layer_width = 20 #Small for Resnet, large for VGG\n\n\nhalf_in_size = round(num_ftrs/2)\n\nNum_class=2\n\nclass SpinalNet(nn.Module):\n    def __init__(self):\n        super(SpinalNet, self).__init__()\n        \n        self.fc_spinal_layer1 = nn.Sequential(\n            nn.Linear(half_in_size, layer_width),\n            nn.ReLU(inplace=True),)\n        self.fc_spinal_layer2 = nn.Sequential(\n            nn.Linear(half_in_size+layer_width, layer_width),\n            nn.ReLU(inplace=True),)\n        self.fc_spinal_layer3 = nn.Sequential(\n            nn.Linear(half_in_size+layer_width, layer_width),\n            nn.ReLU(inplace=True),)\n        self.fc_spinal_layer4 = nn.Sequential(\n            nn.Linear(half_in_size+layer_width, layer_width),\n            nn.ReLU(inplace=True),)\n        self.fc_out = nn.Sequential(\n            nn.Linear(layer_width*4, Num_class),)\n        \n    def forward(self, x):\n        x1 = self.fc_spinal_layer1(x[:, 0:half_in_size])\n        x2 = self.fc_spinal_layer2(torch.cat([ x[:,half_in_size:2*half_in_size], x1], dim=1))\n        x3 = self.fc_spinal_layer3(torch.cat([ x[:,0:half_in_size], x2], dim=1))\n        x4 = self.fc_spinal_layer4(torch.cat([ x[:,half_in_size:2*half_in_size], x3], dim=1))\n        \n        x = torch.cat([x1, x2], dim=1)\n        x = torch.cat([x, x3], dim=1)\n        x = torch.cat([x, x4], dim=1)\n  \n        x = self.fc_out(x)\n        return x\n        \nVGG_fc = nn.Sequential(\n            nn.Linear(num_ftrs, 4096),\n            nn.ReLU(inplace=True),\n            nn.Dropout(),\n            nn.Linear(4096, 4096),\n            nn.ReLU(inplace=True),\n            nn.Dropout(),\n            nn.Linear(4096, Num_class)\n        )\n\n\n'''\nChanging the fully connected layer to SpinalNet or VGG or ResNet\n'''\n\n#model_ft.fc = nn.Linear(num_ftrs, 2) # SpinalNet() # \nmodel_ft.classifier = SpinalNet() # VGG_fc #\nmodel_ft = model_ft.to(device)\ncriterion = nn.CrossEntropyLoss()","metadata":{"execution":{"iopub.status.busy":"2021-12-27T03:00:22.368066Z","iopub.execute_input":"2021-12-27T03:00:22.368417Z","iopub.status.idle":"2021-12-27T03:00:47.839130Z","shell.execute_reply.started":"2021-12-27T03:00:22.368379Z","shell.execute_reply":"2021-12-27T03:00:47.838253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Step5: The Training Function","metadata":{}},{"cell_type":"code","source":"import csv\n\n\ndef train_model_imbalanced (model, criterion, optimizer, scheduler, phases_data):\n    since = time.time()\n\n    best_model_wts = copy.deepcopy(model.state_dict())\n    optimal_acc = 0.0\n    optimal_F1 = 0.0\n    \n    file = open('train_details.csv', 'a+', newline ='')\n    with file:     \n        write = csv.writer(file) \n\n        for phase in phases_data:\n            if phase != 'validation':\n                model.train()  # Set model to training mode\n            else:\n                model.eval()   # Set model to evaluate mode\n\n            running_loss = 0.0\n            running_corrects = 0\n            malignant_phase = 0\n            malignant_correctly_predicted = 0\n            malignant_predicted = 0\n\n\n            # Iterate over data.\n            for inputs, labels in dataloaders[phase]:\n                inputs = inputs.to(device)\n                labels = labels.to(device)\n\n                # zero the parameter gradients\n                optimizer.zero_grad()\n\n                # forward\n                # track history if only in train\n                with torch.set_grad_enabled(phase != 'validation'):\n                    outputs = model(inputs)\n                    _, preds = torch.max(outputs, 1)\n                    loss = criterion(outputs, labels)\n\n                    # backward + optimize only if in training phase\n                    if phase != 'validation':\n                        loss.backward()\n                        optimizer.step()\n                    if phase == 'validation':\n                        for iter_l in range(len(labels)):\n                            if labels[iter_l] == 1: # Malignant\n                                malignant_phase = malignant_phase + 1\n                                if preds[iter_l] == 1: \n                                    malignant_correctly_predicted = malignant_correctly_predicted + 1\n                            if preds[iter_l] == 1: \n                                malignant_predicted = malignant_predicted + 1\n\n\n                # statistics\n                running_loss += loss.item() * inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data)\n            if phase == 'train':\n                scheduler.step()\n\n            epoch_loss = running_loss / dataset_sizes[phase]\n            epoch_acc = running_corrects.double() / dataset_sizes[phase]\n\n            print('{} Loss: {:.4f} Acc: {:.4f}'.format(\n                phase, epoch_loss, epoch_acc))\n            \n            output_format = [phase, epoch_loss, epoch_acc]\n            write.writerows([output_format]) \n\n            # deep copy the model\n            if phase == 'validation': \n                precision = malignant_correctly_predicted/malignant_predicted\n                recall = malignant_correctly_predicted/malignant_phase\n                F1_score = 2/(1/precision+1/recall)\n                if F1_score + epoch_acc < optimal_F1 + optimal_acc: #considering both factors equally\n                    print('F1 score: {:.4f}'.format(F1_score))\n                    model.load_state_dict(best_model_wts) # if no improvement, start from previous best model\n                    continue\n                optimal_F1 = F1_score\n                optimal_acc = epoch_acc\n                best_model_wts = copy.deepcopy(model.state_dict())\n                time_elapsed = time.time() - since\n                print('Time from Start {:.0f}m {:.0f}s, F1 score: {:.4f}'.format(\n                    time_elapsed // 60, time_elapsed % 60, F1_score))\n            #print()\n\n        time_elapsed = time.time() - since\n        print('Training complete in {:.0f}m {:.0f}s'.format(\n            time_elapsed // 60, time_elapsed % 60))\n        print('Optimal val Acc: {:4f}'.format(optimal_acc))\n\n        # load best model weights\n\n        model.load_state_dict(best_model_wts)\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-12-27T03:00:47.840601Z","iopub.execute_input":"2021-12-27T03:00:47.840949Z","iopub.status.idle":"2021-12-27T03:00:47.855891Z","shell.execute_reply.started":"2021-12-27T03:00:47.840912Z","shell.execute_reply":"2021-12-27T03:00:47.854746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Step6: Training","metadata":{}},{"cell_type":"code","source":"phases = []\ninterval_val = 5 \n\nfor iter_data in range(54):\n    phases.append('./tr' + str(iter_data%N_data)) # calling all traiing folders\n    if (iter_data%interval_val) == (interval_val-1) or iter_data > 47:\n        phases.append('validation') # calling the validation data at a certain interval\n                                    # Also, calling frequently, near the end of training\nprint('Phases:', phases)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T03:00:47.871329Z","iopub.execute_input":"2021-12-27T03:00:47.871661Z","iopub.status.idle":"2021-12-27T03:00:47.878653Z","shell.execute_reply.started":"2021-12-27T03:00:47.871631Z","shell.execute_reply":"2021-12-27T03:00:47.877781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)\n\n# Decay LR by a factor of 0.1 every 7 epochs\nexp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)\n\nmodel_ft = train_model_imbalanced(model_ft, criterion, optimizer_ft, exp_lr_scheduler, phases)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-14T12:01:03.435025Z","iopub.execute_input":"2021-06-14T12:01:03.440217Z","iopub.status.idle":"2021-06-14T12:05:02.083635Z","shell.execute_reply.started":"2021-06-14T12:01:03.439735Z","shell.execute_reply":"2021-06-14T12:05:02.082588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Step7: Confusion Matrix","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import classification_report\nimport seaborn as sn\nimport pandas as pd\n\ny_pred = []\ny_true = []\n\n# iterate over test data\nfor inputs, labels in dataloaders['validation']:\n        inputs = inputs.to(device)\n        labels = labels.to(device)\n        \n        output = model_ft(inputs) # Feed Network\n\n        output = (torch.max(torch.exp(output), 1)[1]).data.cpu().numpy()\n        y_pred.extend(output) # Save Prediction\n        \n        labels = labels.data.cpu().numpy()\n        y_true.extend(labels) # Save Truth\n\n# constant for classes\nclasses = ('benign', 'malignant')\n\n# Build confusion matrix\ncf_matrix = confusion_matrix(y_true, y_pred)\ndf_cm = pd.DataFrame(cf_matrix, index = [i for i in classes],\n                     columns = [i for i in classes])\n\n\n\nplt.figure(figsize = (4,2),dpi=100)\nplt.rcParams['font.size'] = '16'\nsn.heatmap(df_cm, annot=True, fmt=\".0f\")\n\nprint(classification_report(y_true, y_pred, target_names = ['benign', 'malignant']))","metadata":{"execution":{"iopub.status.busy":"2021-12-27T03:10:13.390543Z","iopub.execute_input":"2021-12-27T03:10:13.390874Z","iopub.status.idle":"2021-12-27T03:10:20.998787Z","shell.execute_reply.started":"2021-12-27T03:10:13.390842Z","shell.execute_reply":"2021-12-27T03:10:20.998017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Step8: Saving Model","metadata":{}},{"cell_type":"code","source":"best_net = model_ft.classifier\n\nPATH = \"./best_model.pt\"\ntorch.save(best_net.state_dict(), PATH)\n\n\n'''\n#Guideline for loading model in future \n\nimport sys\nsys.modules[__name__].__dict__.clear()\n\ndevice = torch.device('cpu')\n\nmodel_ft = models.wide_resnet101_2(pretrained=True)\nmodel_fc = nn.Linear(num_ftrs, 2)\n\nmodel_fc.load_state_dict(torch.load(PATH, map_location=device))\nmodel_ft.fc = model_fc\n\n'''","metadata":{"execution":{"iopub.status.busy":"2021-06-14T12:05:06.494349Z","iopub.execute_input":"2021-06-14T12:05:06.494845Z","iopub.status.idle":"2021-06-14T12:05:06.50548Z","shell.execute_reply.started":"2021-06-14T12:05:06.494797Z","shell.execute_reply":"2021-06-14T12:05:06.504106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Deleting Unzipped files for smaller output size\nimport shutil\nfor name_folders in data_folders:\n    shutil.rmtree(name_folders)\n\n#plt.savefig('output.pdf')","metadata":{"execution":{"iopub.status.busy":"2021-06-14T12:05:06.507251Z","iopub.execute_input":"2021-06-14T12:05:06.507757Z","iopub.status.idle":"2021-06-14T12:05:08.541216Z","shell.execute_reply.started":"2021-06-14T12:05:06.507706Z","shell.execute_reply":"2021-06-14T12:05:08.540127Z"},"trusted":true},"execution_count":null,"outputs":[]}]}