{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Import the essentials","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing\nimport matplotlib.pyplot as plt # Plotting\nimport seaborn as sns # Plotting\n\n# Import Image Libraries - Pillow and OpenCV\nfrom PIL import Image\nimport cv2\n\n# Import PyTorch and useful fuctions\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import TensorDataset, DataLoader, Dataset,random_split\nimport torchvision.transforms as transforms\nfrom torch.utils.data.sampler import SubsetRandomSampler\nimport torchvision\nimport torch.optim as optim\nimport torchvision.models as models # Pre-Trained models\n\n# Import useful sklearn functions\nimport sklearn\nfrom sklearn.metrics import cohen_kappa_score, accuracy_score\n\nimport time\nfrom tqdm import tqdm_notebook\n\nimport os\nprint(os.listdir(\"../input\"))\nbase_dir = \"../input/aptos2019-blindness-detection/\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading Data + EDA","execution_count":null},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_csv = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\ntest_csv = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Train Size = {}'.format(len(train_csv)))\nprint('Public Test Size = {}'.format(len(test_csv)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Displaying the contents of the training and the test dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_csv.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"counts = train_csv['diagnosis'].value_counts()\nclass_list = ['No DR', 'Mild', 'Moderate', 'Severe', 'Proliferate']\nfor i,x in enumerate(class_list):\n    counts[x] = counts.pop(i)\n\nplt.figure(figsize=(12,6))\nsns.barplot(counts.index, counts.values, alpha=1, palette='deep')\nplt.title('Distribution of Output Classes')\nplt.ylabel('Number of Occurrences', fontsize=14)\nplt.xlabel('Target Classes', fontsize=14)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualizing Training Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(30, 16))\n# display 20 images\ntrain_imgs = os.listdir(base_dir+\"/train_images\")\nfor idx, img in enumerate(np.random.choice(train_imgs, 20)):\n    ax = fig.add_subplot(2, 20//2, idx+1, xticks=[], yticks=[])\n    im = Image.open(base_dir+\"/train_images/\" + img)\n    plt.imshow(im)\n    lab = train_csv.loc[train_csv['id_code'] == img.split('.')[0], 'diagnosis'].values[0]\n    ax.set_title('Severity: %s'%lab)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualizing Test Set","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(30, 16))\n# display 20 images\ntest_imgs = os.listdir(base_dir+\"/test_images\")\nfor idx, img in enumerate(np.random.choice(test_imgs, 20)):\n    ax = fig.add_subplot(2, 20//2, idx+1, xticks=[], yticks=[])\n    im = Image.open(base_dir+\"/test_images/\" + img)\n    plt.imshow(im)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Processing","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Our own custom class for datasets\nclass CreateDataset(Dataset):\n    def __init__(self, df_data, data_dir = '../input/', transform=None):\n        super().__init__()\n        self.df = df_data.values\n        self.data_dir = data_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        img_name,label = self.df[index]\n        img_path = os.path.join(self.data_dir, img_name+'.png')\n        image = cv2.imread(img_path)\n        if self.transform is not None:\n            image = self.transform(image)\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms = transforms.Compose([\n    transforms.ToPILImage(mode='RGB'),\n    transforms.Resize((224, 224)),\n    transforms.RandomHorizontalFlip(p=0.5),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225))\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = \"../input/aptos2019-blindness-detection/train_images/\"\ntest_path = \"../input/aptos2019-blindness-detection/test_images/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" train_data = CreateDataset(df_data=train_csv, data_dir=train_path, transform=transforms)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Set Batch Size\nbatch_size = 32\n\n# Percentage of training set to use as validation\nvalid_size = 0.15\n\n# obtain training indices that will be used for validation\nnum_train = len(train_data)\nindices = list(range(num_train))\nsplit = int(np.floor(valid_size * num_train))\ntrain_idx, valid_idx = indices[split:], indices[:split]\n\n# Create Samplers\ntrain_sampler = SubsetRandomSampler(train_idx)\nvalid_sampler = SubsetRandomSampler(valid_idx)\n\n# prepare data loaders (combine dataset and sampler)\ntrain_loader = DataLoader(train_data, batch_size=batch_size, sampler=train_sampler)\nvalid_loader = DataLoader(train_data, batch_size=batch_size, sampler=valid_sampler)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define Model Architecture","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model = models.resnet152(pretrained=False)\nmodel.load_state_dict(torch.load(\"../input/pretrained-model-weights-pytorch/resnet152-b121ed2d.pth\"))\n# for param in model.parameters():\n#     param.requires_grad = False\nmodel.avg_pool = nn.AdaptiveAvgPool2d(output_size=(1,1))\nmodel.fc = nn.Sequential(\n                nn.Linear(in_features=2048, out_features=1024, bias=True),\n                nn.ReLU(),\n                nn.Linear(in_features=1024, out_features=512, bias=True),\n                nn.ReLU(),\n                nn.Linear(in_features=512, out_features=256, bias=True),\n                nn.ReLU(),\n                nn.Linear(in_features=256, out_features=128, bias=True),\n                nn.ReLU(),\n                nn.Linear(in_features=128, out_features=32, bias=True),\n                nn.ReLU(),\n                nn.Linear(in_features=32, out_features=1, bias=True), \n            )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check if CUDA is available\ntrain_on_gpu = torch.cuda.is_available()\n\nif not train_on_gpu:\n    print('CUDA is not available.  Training on CPU ...')\nelse:\n    print('CUDA is available!  Training on GPU ...')\n    model.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Trainable Parameters\npytorch_total_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\nprint(\"Number of trainable parameters: \\n{}\".format(pytorch_total_params))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training (Fine Tuning) and Validation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# specify loss function (categorical cross-entropy loss)\ncriterion = nn.SmoothL1Loss()\n\n# specify optimizer\noptimizer = optim.Adam(model.parameters(), lr=0.00015)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# number of epochs to train the model\nn_epochs = 7\n\nvalid_loss_min = np.Inf\n\n# keeping track of losses as it happen\ntrain_losses = []\nvalid_losses = []\nval_kappa = []\ntest_accuracies = []\nvalid_accuracies = []\nkappa_epoch = []\nbatch = 0\n\nfor epoch in range(1, n_epochs+1):\n\n    # keep track of training and validation loss\n    train_loss = 0.0\n    valid_loss = 0.0\n    ###################\n    # train the model #\n    ###################\n    model.train()\n    for data, target in tqdm_notebook(train_loader):\n        # move tensors to GPU if CUDA is available\n        if train_on_gpu:\n            data, target = data.cuda(), target.cuda().float()\n        target = target.view(-1, 1)\n        # clear the gradients of all optimized variables\n        optimizer.zero_grad()\n        with torch.set_grad_enabled(True):\n            # forward pass: compute predicted outputs by passing inputs to the model\n            output = model(data)\n            # calculate the batch loss\n            loss = criterion(output, target)\n            # backward pass: compute gradient of the loss with respect to model parameters\n            loss.backward()\n            # perform a single optimization step (parameter update)\n            optimizer.step()\n        # Update Train loss and accuracies\n        train_loss += loss.item()*data.size(0)\n        \n    ######################    \n    # validate the model #\n    ######################\n    model.eval()\n    for data, target in tqdm_notebook(valid_loader):\n        # move tensors to GPU if CUDA is available\n        if train_on_gpu:\n            data, target = data.cuda(), target.cuda().float()\n        # forward pass: compute predicted outputs by passing inputs to the model\n        target = target.view(-1, 1)\n        with torch.set_grad_enabled(True):\n            output = model(data)\n            # calculate the batch loss\n            loss = criterion(output, target)\n        # update average validation loss \n        valid_loss += loss.item()*data.size(0)\n        #output = output.cohen_kappa_score_kappa_score)\n        y_actual = target.data.cpu().numpy()\n        y_pred = output[:,-1].detach().cpu().numpy()\n        val_kappa.append(cohen_kappa_score(y_actual, y_pred.round()))        \n    \n    # calculate average losses\n    train_loss = train_loss/len(train_loader.sampler)\n    valid_loss = valid_loss/len(valid_loader.sampler)\n    valid_kappa = np.mean(val_kappa)\n    kappa_epoch.append(np.mean(val_kappa))\n    train_losses.append(train_loss)\n    valid_losses.append(valid_loss)\n        \n    # print training/validation statistics \n    print('Epoch: {} | Training Loss: {:.6f} | Val. Loss: {:.6f} | Val. Kappa Score: {:.4f}'.format(\n        epoch, train_loss, valid_loss, valid_kappa))\n    \n    ##################\n    # Early Stopping #\n    ##################\n    if valid_loss <= valid_loss_min:\n        print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(\n        valid_loss_min,\n        valid_loss))\n        torch.save(model.state_dict(), 'best_model.pt')\n        valid_loss_min = valid_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\n%config InlineBackend.figure_format = 'retina'\n\nplt.plot(train_losses, label='Training loss')\nplt.plot(valid_losses, label='Validation loss')\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Loss\")\nplt.legend(frameon=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\n%config InlineBackend.figure_format = 'retina'\n\nplt.plot(kappa_epoch, label='Val Kappa Score/Epochs')\nplt.legend(\"\")\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Kappa Score\")\nplt.legend(frameon=False)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"model.load_state_dict(torch.load('best_model.pt'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_transforms = torchvision.transforms.Compose([\n    torchvision.transforms.ToPILImage(),\n    torchvision.transforms.Resize((224, 224)),\n    #torchvision.transforms.ColorJitter(brightness=2, contrast=2),\n    torchvision.transforms.RandomHorizontalFlip(p=0.5),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225))\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_csv['diagnosis'] = -1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data = CreateDataset(df_data=test_csv, data_dir=test_path, transform=test_transforms)\ntest_loader = DataLoader(test_data, batch_size=batch_size, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" def round_off_preds(preds, coef=[0.5, 1.5, 2.5, 3.5]):\n    for i, pred in enumerate(preds):\n            if pred < coef[0]:\n                preds[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                preds[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                preds[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                preds[i] = 3\n            else:\n                preds[i] = 4\n    return preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(testloader):\n    '''Function used to make predictions on the test set'''\n    model.eval()\n    preds = []\n    for batch_i, (data, target) in enumerate(testloader):\n        data, target = data.cuda(), target.cuda()\n        output = model(data)\n        pr = output.detach().cpu().numpy()\n        for i in pr:\n            preds.append(i.item())\n            \n    return preds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### TTA (Test Time Augmentation)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"preds1 = np.array(predict(testloader=test_loader))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds2 = np.array(predict(testloader=test_loader))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds3 = np.array(predict(testloader=test_loader))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds4 = np.array(predict(testloader=test_loader))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds5 = np.array(predict(testloader=test_loader))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds6 = np.array(predict(testloader=test_loader))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds7 = np.array(predict(testloader=test_loader))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds8 = np.array(predict(testloader=test_loader))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = (preds1 + preds2 + preds3 + preds4 + \n         preds5 + preds6 + preds7 + preds8)/8.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = round_off_preds(preds)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Generating Submission File","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_sub = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_sub.diagnosis = preds\nsample_sub.diagnosis = sample_sub['diagnosis'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install jovian --upgrade","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import jovian","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"jovian.commit(project='course-project-version3')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}