{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Some notes\nThis notebook contains a pre-trained Inceptionv3 with large learning rate, TTA and Grad-CAM.<br>\n<br>**Likbez on topic:**\n+ **Anatomy of the organ of vision:** <br>[[rus] 50 min video](https://www.youtube.com/watch?v=0OECPht72hA&list=LLDzSJMVSU9zgR9SJDynuxAA&index=6&t=0s) or [[rus] 5 min video](https://www.youtube.com/watch?v=TJN_9P8yQJU&list=LLDzSJMVSU9zgR9SJDynuxAA&index=11&t=0s)\n+ **Classification of diabetic retinopathy:**\n<br>\n[[en] 16 min video](https://www.youtube.com/watch?v=IWspTG9wIsU&list=LLDzSJMVSU9zgR9SJDynuxAA&index=3&t=752s) or [[en] 13 min video](https://www.youtube.com/watch?v=VIrkurR446s&list=LLDzSJMVSU9zgR9SJDynuxAA&index=2&t=11s) or [1 min video](https://www.youtube.com/watch?v=mb0hGpo6LK4&list=LLDzSJMVSU9zgR9SJDynuxAA&index=4&t=0s) or [[en] 7 min text](https://nei.nih.gov/health/diabetic/retinopathy)\n+ **Baseline from youtube channel \"DevPRO\":**<br>\n[[rus] 38 min video](https://www.youtube.com/watch?v=jOsPYvRDUpE)\n\nAlso, I wrote a [**\"bot-ophthalmologist\"**](https://t.me/MedEyeBot/), including based on data from these competitions, and if you are interested in creating some kind of interface for other people to interact with your ml-models, then my [**repository**](https://github.com/OldBonhart/MedEyeService) can become a starting point, it contains the bot code and some notes for deployment on heroku, there you can see an example of a bunch of **api telegram** + **pythorch** + **heroku**."},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nprint(os.listdir(\"../input\"))\n\nimport numpy as np\nimport pandas as pd\n\nimport cv2\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\ntest_df = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\nsub = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\nprint('Dimensions:', train_df.shape, test_df.shape, sub.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['diagnosis'] = 0\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = ['No DR', 'Mild', 'Moderate', 'Severe', 'Proliferative DR']\nexplode = (0.1, 0, 0, 0, 0.1)\n\nfig, ax = plt.subplots(figsize=(7,7))\nax.pie(train_df['diagnosis'].value_counts(), explode=explode, labels=labels,\n      autopct='%1.1f%%',shadow=True, startangle=90);\n\nax.set_title('Distibution the presence of diabetic retinopathu in each image on a scale of 0 to 4',\n            fontdict={\n                'fontsize':15\n            });","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(15, 10))\nfor label in sorted(train_df['diagnosis'].unique()):\n    for i, (idx, row) in enumerate(train_df.loc[train_df['diagnosis'] == label].sample(5).iterrows()):\n        ax = fig.add_subplot(5, 5, label * 5 + i + 1, xticks=[], yticks=[])\n        img = cv2.imread(f\"../input/aptos2019-blindness-detection/train_images/{row['id_code']}.png\")\n        plt.imshow(img[...,[2,1,0]])\n        ax.set_title(f'Label: {label}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data preprocessing"},{"metadata":{"trusted":true},"cell_type":"code","source":"# code from https://www.kaggle.com/ratthachat/aptos-updatedv14-preprocessing-ben-s-cropping\n# Image processing\ndef crop_image_from_gray(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n            img = np.stack([img1,img2,img3],axis=-1)\n        return img\n    \n    \ndef load_ben_color(path, sigmaX=10 ):\n    image = cv2.imread(path)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = crop_image_from_gray(image)\n    image = cv2.resize(image, (512, 512))\n    image=cv2.addWeighted ( image,4, cv2.GaussianBlur( image , (0,0) , sigmaX) ,-4 ,128)\n        \n    return image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset and dataloader"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader \n\nimport torchvision\nfrom torchvision.transforms import transforms","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Dataset\nclass RetinaDataset(Dataset):\n    def __init__(self, df, img_dir, transforms):\n        self.df = df\n        self.img_dir = img_dir\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        img_name = os.path.join(self.img_dir,\n                               self.df.iloc[idx, 0] + '.png')\n        image = load_ben_color(img_name)\n        image = self.transforms(image)\n        label = self.df.iloc[idx, 1]\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Augmentations for train/test data\ntrain_aug = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomRotation(100),\n    transforms.RandomVerticalFlip(),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                 std=[0.229, 0.224, 0.225]),\n])\ntest_aug = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                 std=[0.229, 0.224, 0.225])\n])\n\n# train/test dataset & dataloader\ntrain_img_dir = '../input/aptos2019-blindness-detection/train_images/'\ntest_img_dir =  '../input/aptos2019-blindness-detection/test_images/'\n\ntrain_dataset = RetinaDataset(df=train_df, img_dir=train_img_dir, transforms=train_aug)\ntest_dataset = RetinaDataset(df=test_df, img_dir=test_img_dir, transforms=test_aug)\n\ntrain_loader = DataLoader(dataset=train_dataset, batch_size=16,shuffle=True)\ntest_loader = DataLoader(dataset=test_dataset, batch_size=16, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Checking data"},{"metadata":{"trusted":true},"cell_type":"code","source":"i, l = next(iter(train_loader))\ni.shape, l.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Checking aug for tta\ndef show_aug(inp, title=None):\n    \"\"\"Imshow for Tensor.\"\"\"\n    inp = inp.numpy().transpose((1, 2, 0))\n    mean = np.array([0.485, 0.456, 0.406])\n    std = np.array([0.229, 0.224, 0.225])\n    inp = std * inp + mean\n    inp = np.clip(inp, 0, 1)\n    plt.figure(figsize=(20,15))\n    plt.imshow(inp)\n    if title is not None:\n        plt.title(title)\n    plt.pause(0.001)  # pause a bit so that plots are updated\n\n    \n# Get a batch of training data\ninputs, _ = next(iter(train_loader))\n\n# Make a grid from batch\nout = torchvision.utils.make_grid(inputs,4)  \n\nshow_aug(out)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Hyper parameters\nnum_epochs = 8\nnum_classes = 5\nlr = 0.001","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Model initialization\ndevice = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\nf = '../input/inceptionv3-6epoch/inception3_6epoch.pt'\n\nmodel = torchvision.models.inception_v3(pretrained=False, aux_logits = False)\nmodel.fc = torch.nn.Linear(model.fc.in_features, num_classes)\nmodel.load_state_dict(torch.load(f, map_location='cuda:0'))\nmodel = model.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Loss and Optimizer\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adamax(model.parameters(), lr=lr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Training model\ntotal_step = len(train_loader)\nfor epoch in range(num_epochs):\n    for batch_i, (data, target) in enumerate(train_loader):\n        data, target = data.to(device), target.to(device)\n        \n        # Forward pass\n        outputs = model(data)\n        loss = criterion(outputs, target)\n        \n        # Backward and optimizer\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        \n        if (batch_i+1) % 100 == 0:\n            print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'\n                 .format(epoch+1, num_epochs, batch_i+1, total_step, loss.item()))\ntorch.save(model.state_dict(), 'model.pt')            ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TTA"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Augmentation data generators\n\naug1 = transforms.Compose([\n       transforms.ToPILImage(),\n       transforms.ToTensor(),\n       transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                            std=[0.229, 0.224, 0.225])\n])\n\n\naug2 = transforms.Compose([\n        transforms.ToPILImage(),\n        transforms.RandomChoice([\n            transforms.RandomRotation((0,0)),\n            transforms.RandomHorizontalFlip(p=1),\n            transforms.RandomVerticalFlip(p=1),\n            transforms.RandomRotation((90,90)),\n            transforms.RandomRotation((180,180)),\n            transforms.RandomRotation((270,270)),\n        ]),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                             std=[0.229, 0.224, 0.225])\n])\n\n\naug3 = transforms.Compose([\n        transforms.ToPILImage(),\n        transforms.RandomHorizontalFlip(),\n        transforms.RandomRotation(100),\n        transforms.RandomVerticalFlip(),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                 std=[0.229, 0.224, 0.225])\n])\n\n# Augmentation dataset & data loaders\ntest_dataset1 = RetinaDataset(df=test_df, img_dir=test_img_dir, transforms=aug1)\ntest_dataset2 = RetinaDataset(df=test_df, img_dir=test_img_dir, transforms=aug2)\ntest_dataset3 = RetinaDataset(df=test_df, img_dir=test_img_dir, transforms=aug3)\n\ntl1 = DataLoader(dataset=test_dataset1, batch_size=16, shuffle=False)\ntl2 = DataLoader(dataset=test_dataset2, batch_size=16, shuffle=False)\ntl3 = DataLoader(dataset=test_dataset3, batch_size=16, shuffle=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tta_loaders = [tl1, tl2, tl3]\n\nt1,t2,t3 = [], [], []\npreds = [t1, t2, t3]\nfor i in range(len(tta_loaders)):\n    with torch.no_grad():\n        model.eval()\n        for data, target in tta_loaders[i]:\n            data = data.to(device)\n            target = target.to(device)\n            outputs = model(data)\n            for probs in outputs:\n                #print(prob)\n                preds[i].append(probs.detach().cpu().numpy())\n                \nend = [(a+b+c) / 3 for a,b,c in zip(t1, t2, t3)]\n\npredictions = []\nfor prob in end:   \n    idx = np.argmax(prob)\n    #pred = rectification(prob[idx])\n    predictions.append(idx)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['diagnosis'] =  predictions\nsub.to_csv('submission.csv', index=False)\nsub['diagnosis'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Heatmaps with ROI"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Extract  pretrained activations\nclass SaveFeatures():\n    \"\"\" Extract pretrained activations\"\"\"\n    features=None\n    def __init__(self, m):\n        self.hook = m.register_forward_hook(self.hook_fn)\n    def hook_fn(self, module, input, output):\n        self.features = ((output.cpu()).data).numpy()\n    def remove(self):\n        self.hook.remove()\n        \nfinal_layer = model._modules.get('Mixed_7c')\nactivated_features = SaveFeatures(final_layer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Probabilities & labels for each images\noutput = model(data[:8])# conver to cuda for softmax\nprobabilities = F.softmax(output,dim=1).data.squeeze()\npred_idx = np.argmax(probabilities.cpu().detach().numpy(),axis=1)\nlabels = pred_idx\nactivated_features.remove()\nprint('Probabilities classes: %s \\n Prediction indices %s \\n Labels: %s' % (probabilities, pred_idx, labels))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def getCAM(feature_conv, weight_fc, class_idx):\n    _, nc, h, w = feature_conv.shape\n    cam = weight_fc[class_idx].dot(feature_conv[0,:, :, ].reshape((nc, h*w)))\n    cam = cam.reshape(h, w)\n    cam = cam - np.min(cam)\n    cam_img = cam / np.max(cam)\n    return cam_img\n\nweight_softmax_params = list(model._modules.get('fc').parameters())\nweight_softmax = np.squeeze(weight_softmax_params[0].cpu().data.numpy())\nweight_softmax_params","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Current images & their heatmaps\ncur_images = data.cpu().numpy().transpose((0, 2, 3, 1))\nheatmaps = []\nfor i in pred_idx:\n    img = getCAM(activated_features.features, weight_softmax, i)\n    heatmaps.append(img)\n    \nprint(cur_images.shape, len(heatmaps))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Probability for each images\nproba = []\nfor i in probabilities.cpu().detach().numpy():\n    idx = np.argmax(i)\n    proba.append((str(np.round(i[idx]*100,2)))+'%')\nprint(proba)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig=plt.figure(figsize=(20,15))\nfor i in range(0, len(cur_images[:8])):\n    img = cur_images[i]\n    mask = heatmaps[i]\n    ax = fig.add_subplot(4, 4,i +1,xticks=[], yticks=[])\n    plt.imshow(img)\n    plt.imshow(cv2.resize(mask, (512,512), interpolation=cv2.INTER_LINEAR), alpha=0.5, cmap='jet');\n    ax.set_title('Label %d with %s probability' % (labels[i], proba[i]),fontsize=14)\n    \n#cax = fig.add_axes([0.3, 0.42, 0.4, 0.04]) # place where be map\ncax = fig.add_axes([0.32, 0.42, 0.4, 0.03]) # place where be map\nclb = plt.colorbar(cax=cax, orientation='horizontal',ticks=[0, 0.5, 1])\nclb.ax.set_title('Level of \"attention\" NN in making prediction',fontsize=20)\nclb.ax.set_xticklabels(['low', 'medium', 'high'],fontsize=18)\n\n\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}