{"cells":[{"metadata":{"_uuid":"802241568d06a5d9f7a9cd932e07a907a1a241f6"},"cell_type":"markdown","source":"# Khoi's attempt to detect diabetic retinopathy"},{"metadata":{"_uuid":"26e2364fa9b012db80c9f2d585c625470cf25802"},"cell_type":"markdown","source":"## Initializing dataset class"},{"metadata":{"trusted":true,"_uuid":"9da7cd27774e39db4d52e61280da9eb2b2e7b3c7"},"cell_type":"code","source":"import os\nimport torch\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\nfrom PIL import Image\nimport pandas as pd\nimport numpy as np\nfrom matplotlib import pyplot as plt\n\n# custom dataset wrapper\nclass RetinasDataset(Dataset):\n    def __init__(self, csv_file, root_dir, transform=None):\n        self.retina_df = pd.read_csv(csv_file)\n        self.root_dir = root_dir\n        self.transform = transform\n        \n    def __getitem__(self, idx):\n        img_name = os.path.join(self.root_dir, str(self.retina_df.iloc[idx, 0]) + '.jpeg')\n        if not os.path.isfile(img_name):\n            return None\n        \n        image = Image.open(img_name)\n        label = int(self.retina_df.iloc[idx, 1])\n        if self.transform:\n            image = self.transform(image)\n            \n        sample = {'image': image, 'label': label}\n        return sample\n        \n    def __len__(self):\n        return len(self.retina_df)\n\n\ntransform_viz = transforms.Compose([\n    transforms.Resize((256, 256)),\n])\nviz = RetinasDataset(csv_file='../input/diabetic-retinopathy-detection/trainLabels.csv',\n                         root_dir='../input/diabetic-retinopathy-detection', transform=transform_viz)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b8becfb9b02ccbcec0808f979b014c1ad8d13ae8"},"cell_type":"markdown","source":"## Some sample pics"},{"metadata":{"trusted":true,"_uuid":"3aff3f1551c968f466d2a81813e91e10e3283e40"},"cell_type":"code","source":"rows = 4\ncols = 4\nfig = plt.figure(figsize=(8, 8))\n\nfor i in range(1, rows * cols + 1):\n    fig.add_subplot(rows, cols, i)\n    plt.imshow(viz[i]['image'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2b9767a9e7e2369bf2c564a10aee2444ecc2cc3f"},"cell_type":"markdown","source":"## Split into training and valition set"},{"metadata":{"trusted":true,"_uuid":"1c64013918b1ba3fdee56ab6a25dd6986daa4d21"},"cell_type":"code","source":"from torch.utils.data import DataLoader\nfrom torch.utils.data.sampler import SubsetRandomSampler\n\nnormalize = transforms.Compose([\n    transforms.Resize((256, 256)),\n    transforms.ToTensor(),\n                           # mean and std for 3 channels\n    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n])\ndataset = RetinasDataset(csv_file='../input/diabetic-retinopathy-detection/trainLabels.csv',\n                         root_dir='../input/diabetic-retinopathy-detection', transform=normalize)\n\nnp.random.seed(42)\nbatch_size = 400\nval_split = .2\ndata_sz = len(dataset)\n\n# splitting the data into training and validation sets\nindices = list(range(data_sz))\nsplit = int(np.floor(val_split * data_sz))\nnp.random.shuffle(indices)\ntrain_indices, val_indices = indices[split:], indices[:split]\n\n# random batch samplers\ntrain_sampler = SubsetRandomSampler(train_indices)\nval_sampler = SubsetRandomSampler(val_indices)\n\n# some images that are in trainLabels.csv do not exist :(\ndef rm_na(batch): \n    batch = list(filter(lambda x : x is not None, batch))\n    data = [item['image'] for item in batch]\n    \n    if len(data) > 0:\n        data = torch.stack(data).cuda()\n        target = torch.Tensor([item['label'] for item in batch]).long().cuda()\n        return [data, target]\n    \n    return None\n    \n\n# batch feeders\ntrain_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=train_sampler, collate_fn=rm_na)\nval_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=val_sampler, collate_fn=rm_na)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"87ced1264e345c4def0bbb369b97ca674f2c92c1"},"cell_type":"markdown","source":"## Toy CNN"},{"metadata":{"trusted":true,"_uuid":"09813a6998a506293e690437d2c70a2bcf529542"},"cell_type":"code","source":"import torch.nn as nn\nimport torch.nn.functional as F\nfrom functools import reduce\n\nclass ToyNet(nn.Module):\n    def __init__(self):\n        super(ToyNet, self).__init__()\n        self.conv1 = nn.Conv2d(3, 6, 5)\n        self.pool = nn.MaxPool2d(2, 2)\n        self.conv2 = nn.Conv2d(6, 16, 5)\n        self.fc1 = nn.Linear(16 * 61 * 61, 120)\n        self.fc2 = nn.Linear(120, 84)\n        self.fc3 = nn.Linear(84, 5)\n\n    def forward(self, x):\n        x = self.pool(F.relu(self.conv1(x)))\n        x = self.pool(F.relu(self.conv2(x)))\n        x = x.view(-1, 16 * 61 * 61)\n        x = F.relu(self.fc1(x))\n        x = F.relu(self.fc2(x))\n        x = self.fc3(x)\n        return x\n\n\nnet = ToyNet()\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nnet.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"729d5f39a48c98bbdc5824329d2f7137e07cefa4"},"cell_type":"markdown","source":"## Time to train!"},{"metadata":{"trusted":true,"_uuid":"e15a073f15eb040d26575cd748a2e35d7a1743b4","scrolled":false},"cell_type":"code","source":"import torch.optim as optim\n\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.SGD(net.parameters(), lr=0.0008, momentum=0.07)\n\nfor epoch in range(5):  # loop over the dataset multiple times\n    running_loss = 0.0\n    for i, data in enumerate(train_loader, 0):\n        # get the inputs\n        if data != None:\n            inputs, labels = data\n        else:\n            continue\n\n        # zero the parameter gradients\n        optimizer.zero_grad()\n\n        # forward pass, backprop, update weights\n        outputs = net(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n\n        # print statistics\n        running_loss += loss.item()\n        if i % 10 == 9:    # print every 10 mini-batches\n            print('[Epoch %d, Batch %d] Loss: %.3f' % (epoch + 1, i + 1, running_loss / 10))\n            running_loss = 0.0\n\nprint('Finished Training')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f1c551d89c3a1c1d9bd1468bf4cf71be6975cfe1"},"cell_type":"code","source":"torch.save(net.state_dict(), '/kaggle/working/../input/toy-cnn.pt')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7231cea673866fc4bca938f5a1d90294d69eba77"},"cell_type":"markdown","source":"From the loss alone above, we can see that the gradient descent with momentum \"overshoots\" quite a bit in the end, so we can try to reduce momentum for it to settle in the minimum."},{"metadata":{"_uuid":"90264ce6e703b857d5fef8a2e1ee9ad10557d624"},"cell_type":"markdown","source":"## Trying the model on test data"},{"metadata":{"trusted":true,"_uuid":"fe0e786346f5959d144ff58e512220113a04314e"},"cell_type":"code","source":"import os\nprint(os.listdir(\"/kaggle/working/../input/testset\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea9350a3e5d94259b6e6b628f5e03e12a6e2736b"},"cell_type":"code","source":"from sklearn.metrics import f1_score\n\ndef gen_answers(test_csv, root_dir='../input', transform=None):\n    testset = RetinasDataset(csv_file=test_csv, root_dir=root_dir, transform=transform)\n    test_loader = torch.utils.data.DataLoader(testset, batch_size=batch_size, sampler=train_sampler, collate_fn=rm_na)\n    \n    with torch.no_grad():\n        for data in test_loader:\n            if data != None: \n                inputs, labels = data\n                outputs = net(inputs)\n                print(labels)\n            else:\n                print(\"hi\", end=\" \") # hm, Kaggle doesn't provide test images in the 1000 imgs :(\n        \nraw_test_pred = gen_answers('../input/testset/retinopathy_solution.csv', transform=normalize)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9bd6577e791339ea585f54ce8678be93ca010ad8"},"cell_type":"markdown","source":"Since we do not use the validation set to indirectly tweak our CNN, let's use it for test set as it doesn't exist. Ideally, we would use validation set to try to generalize our model and then measure the performance on unseen data based on the actual test set. We could also divide the training data into train, validation, and test sets, but I thought then the training set would be too small to train on, and the test set would be too small to tell what's the performance (for example, if we were to split 60%, 20%, 20%). Remember: there are only 1000 data in this environment."},{"metadata":{"_uuid":"b82541a7ef2c45542dd0f0f84bd53fb2c55cb876"},"cell_type":"markdown","source":"## Validation set performance"},{"metadata":{"trusted":true,"_uuid":"c36a2d0263b5538c24e452fdf367fffe2ebffcbf"},"cell_type":"code","source":"all_labels = torch.Tensor().cuda()\nall_pred = torch.Tensor().cuda()\n\nwith torch.no_grad():\n    for data in val_loader:\n        inputs, labels = data\n        outputs = net(inputs)\n        values, pred_labels = outputs.max(1)\n        \n        all_labels = torch.cat((all_labels.long(), labels)).cuda()\n        all_pred = torch.cat((all_pred.long(), pred_labels)).cuda()\n        \n        print(\"Actual labels:\", labels)\n        print(\"Predicted labels:\", pred_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4b3afb57bbd3bc535ae3c77577ce77623071e285"},"cell_type":"code","source":"print(f1_score(all_labels.cpu(), all_pred.cpu(), average='micro'))\n# calculate metrics globally by counting the total true positives, false negatives and false positives.\nprint(f1_score(all_labels.cpu(), all_pred.cpu(), average='weighted'))\n# calculate metrics for each label, and find their average weighted by support (the number of true instances for each label).","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8e03c895b85774db54669123aef1b64e03b0046e"},"cell_type":"markdown","source":"In real life, these labels are not really categorical but rather discrete but on the same scale. Therefore, it might be useful to see the average distance by which we're \"off.\" After all, predicting that someone is stage 3 of retinopathy when it is actually 4 is better than saying 0."},{"metadata":{"trusted":true,"_uuid":"631a5c829f8490aa1de9e539b0e5c3b35b79c210"},"cell_type":"code","source":"print(np.mean(np.array(abs(all_labels - all_pred).cpu())))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0942531386931f1253d849fc0ea4623894e2095f"},"cell_type":"markdown","source":"## Model evalutation\n\nHorrible. Always predicts 0. This network has high bias because it is very shallow. This is why it's not able to extract subtle features that distinguish different classes. Another factor is that our dataset is unbalanced, having predominantly 0, causing even less learning for labels 1-4.\n\nThen, we should try something deeper with more capacity to build more complex combination of features. Although it is tempting to train deeper CNNs, we cannot do so due to limited data (in this kernel, there are only 1000 data, and in the competition, around 35k), then we would bump into a problem of overfitting (high variance).\n\nThus, a feasible solution would be to utilize transfer learning and retrain only the last (few) fully-connected layer(s) with the limited data that we have. In addition, we can do data augmentation by:\n1. Rotation (by 90, 180, 270 deg.) to account for inverted images\n2. Different brigthness/saturation/contrast/hue settings to account for different apparati and eyes (and random errors, such as the amount of light)\n3. Different crops to account for different distances at which the eyes are being shot"},{"metadata":{"_uuid":"6696cfd797c538720f0cd37350ea6f34317f0177"},"cell_type":"markdown","source":"## Transfer learning scheme (the unworking draft)"},{"metadata":{"trusted":true,"_uuid":"f18100818264e8e0453849c0929a5f74ea2f50b5"},"cell_type":"code","source":"def train_model(model, criterion, optimizer, scheduler, num_epochs=25):\n    since = time.time()\n\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n\n    for epoch in range(num_epochs):\n        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n        print('-' * 10)\n\n        # Each epoch has a training and validation phase\n        for phase in ['train', 'val']:\n            if phase == 'train':\n                scheduler.step()\n                model.train()  # Set model to training mode\n            else:\n                model.eval()   # Set model to evaluate mode\n\n            running_loss = 0.0\n            running_corrects = 0\n\n            # Iterate over data.\n            for inputs, labels in dataloaders[phase]:\n                inputs = inputs.to(device)\n                labels = labels.to(device)\n\n                # zero the parameter gradients\n                optimizer.zero_grad()\n\n                # forward\n                # track history if only in train\n                with torch.set_grad_enabled(phase == 'train'):\n                    outputs = model(inputs)\n                    _, preds = torch.max(outputs, 1)\n                    loss = criterion(outputs, labels)\n\n                    # backward + optimize only if in training phase\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n\n                # statistics\n                running_loss += loss.item() * inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data)\n\n            epoch_loss = running_loss / dataset_sizes[phase]\n            epoch_acc = running_corrects.double() / dataset_sizes[phase]\n\n            print('{} Loss: {:.4f} Acc: {:.4f}'.format(\n                phase, epoch_loss, epoch_acc))\n\n            # deep copy the model\n            if phase == 'val' and epoch_acc > best_acc:\n                best_acc = epoch_acc\n                best_model_wts = copy.deepcopy(model.state_dict())\n\n        print()\n\n    time_elapsed = time.time() - since\n    print('Training complete in {:.0f}m {:.0f}s'.format(\n        time_elapsed // 60, time_elapsed % 60))\n    print('Best val Acc: {:4f}'.format(best_acc))\n\n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"67168dd38d768bb6b523bdd5592c188de6adb37b"},"cell_type":"markdown","source":"## Importing ResNet saved weights"},{"metadata":{"trusted":true,"_uuid":"9c5783f33851ee0b414236c61cb5cc2796f35de4"},"cell_type":"code","source":"from torchvision import models\n\nmodel_ft = models.resnet18(pretrained=True)\nnum_ftrs = model_ft.fc.in_features\nmodel_ft.fc = nn.Linear(num_ftrs, 2)\n\nmodel_ft = model_ft.to(device)\n\ncriterion = nn.CrossEntropyLoss()\noptimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)\n\n# Decay LR by a factor of 0.1 every 7 epochs\nexp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8feaecd7d3c08769738a8588fcb8aff7cae8e0bb"},"cell_type":"markdown","source":"## Retrain the last FC layer"},{"metadata":{"trusted":true,"_uuid":"ca873b7d36b8c589b698121fc97b18226d1f27ec"},"cell_type":"code","source":"model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler, num_epochs=25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd2b8557d6d5d2d0cfed26d4df87dc83a0ef15e2"},"cell_type":"code","source":"import os\nprint(os.path.isfile('../input/diabetic-retinopathy-detection/1_left.jpeg'))\nprint(os.path.isfile('../input/diabetic-retinopathy-detection/10_left.jpeg'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eafdc4d86afdf5f081fe064a6c025e2ca67fa8bc"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}