{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport torch \nimport torch.nn as nn\nfrom torch.utils.data import TensorDataset, DataLoader,Dataset , SubsetRandomSampler \nimport torch.nn.functional as F\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport cv2\nimport albumentations\nfrom albumentations import torch as AT\nfrom torch.optim import lr_scheduler\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold,StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import cohen_kappa_score\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_dr = \"../input/aptos2019-blindness-detection/\"","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Reading the CSVs\ntrain = pd.read_csv(base_dr+'train.csv')\ntest = pd.read_csv(base_dr+'test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def cohen_k_score(y_true , y_pred):\n    skl = cohen_kappa_score(np.argmax(y_true, axis=1), np.argmax(y_pred, axis=1), weights='quadratic')\n    return skl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"###  Diabetic Ratinopathy\n\n0 - No DR , 1 - Mild , 2 - Moderate , 3 - Severe , 4 - Proliferative DR"},{"metadata":{"trusted":true},"cell_type":"code","source":"#### Label Distribution:\nimport seaborn as sns\nsns.countplot(x='diagnosis' , data=train);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('No of images in the train set:', format(train.shape[0]))\nprint('No of images in the test set:', format(test.shape[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"diagnosis = {'No DR': 0, \n'Mild': 1, \n'Moderate': 2, \n'Severe': 3, \n'Proliferative DR': 4}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### preparing some useful stuff for easy vizualizations later on\nRev = dict((v,k) for k,v in diagnosis.items())  ### need to reverse\ndia = pd.DataFrame(train['diagnosis'].map(Rev))\ndia.rename(columns={'diagnosis': \"diagnosisText\"},inplace=True)\ndia = pd.concat([train,pd.DataFrame(dia)],1)\ndia.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from random import sample\nimport cv2\nimport matplotlib.image as mpimg\n\ndef plotClass(category,N):\n    # credit : https://www.kaggle.com/pheadrus/purepytorchmodels?scriptVersionId=15756053\n    categoryIdx = dia[dia['diagnosisText']==category].index[:30]\n    randIdx = sample(list(categoryIdx),N)\n    jpegName = dia.iloc[randIdx,:]['id_code'].values\n    fig = plt.figure(figsize=(18,14))\n    for i , jpeg in enumerate(list(jpegName)):\n        plt.subplot(1,N ,i+1)\n        imgFile = mpimg.imread('{}/train_images/{}.png'.format(base_dr,jpeg))\n        plt.imshow(imgFile)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plotClass('No DR',4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plotClass('Proliferative DR',4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plotClass('Severe',4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plotClass('Moderate',4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import OneHotEncoder,LabelEncoder\n\ndef prepare_labels(y):\n    values = np.array(y)\n    label_encoder = LabelEncoder()\n    integer_encoded = label_encoder.fit_transform(values)\n\n    onehot_encoder = OneHotEncoder(sparse=False)\n    integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)\n    onehot_encoded = onehot_encoder.fit_transform(integer_encoded)\n\n    y = onehot_encoded\n    return y, label_encoder","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y, le_full = prepare_labels(train['diagnosis'])\ny.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y[0:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.iloc[0:5]['diagnosis']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BlindnessDataset(Dataset):\n    def __init__(self, df, datatype='train', transform = transforms.Compose([transforms.CenterCrop(32),transforms.ToTensor()]), y = None):\n        self.df = df\n        self.datatype = datatype\n        self.image_files_list = [f'../input/aptos2019-blindness-detection/{self.datatype}_images/{i}.png' for i in df['id_code'].values]\n        if self.datatype == 'train':\n            self.labels = y\n        else:\n            self.labels = np.zeros((df.shape[0], 5))\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.image_files_list)\n\n    def __getitem__(self, idx):\n        img_name = self.image_files_list[idx]\n        img = cv2.imread(img_name)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        image = self.transform(image=img)\n        image = image['image']\n\n        img_name_short = self.image_files_list[idx].split('.')[0]\n\n        label = self.labels[idx]\n        if self.datatype == 'test':\n            return image, label, img_name\n        else:\n            return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#data_transforms = albumentations.Compose([\n#    albumentations.Resize(224, 224),\n#    albumentations.HorizontalFlip(),\n#    albumentations.RandomBrightness(),\n#    albumentations.ShiftScaleRotate(rotate_limit=15, scale_limit=0.10),\n#    albumentations.JpegCompression(80),\n#    albumentations.HueSaturationValue(),\n#    albumentations.Normalize(),\n#    AT.ToTensor()\n#    ])\ndata_transforms = albumentations.Compose([\n    albumentations.Resize(224, 224),\n    albumentations.HorizontalFlip(p=0.5),\n    albumentations.Transpose(p=0.5),\n    albumentations.Flip(p=0.0),\n    albumentations.OneOf([\n        albumentations.CLAHE(clip_limit=2), albumentations.IAASharpen(), albumentations.IAAEmboss(), \n        albumentations.RandomBrightness(), albumentations.RandomContrast(),\n        albumentations.JpegCompression(), albumentations.Blur(), albumentations.GaussNoise()], p=0.5), \n    albumentations.HueSaturationValue(p=0.5), \n    albumentations.ShiftScaleRotate(shift_limit=0.15, scale_limit=0.15, rotate_limit=45, p=0.5),\n    albumentations.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225]),\n    AT.ToTensor()\n   ])\ndata_transforms_test = albumentations.Compose([\n    albumentations.Resize(224, 224),\n    albumentations.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225]),\n    #albumentations.Random\n    AT.ToTensor()\n    ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 64\nnum_workers = 8\n\n## 90 - 10 train & validation split \ntr, val = train_test_split(train.diagnosis, stratify=train.diagnosis, test_size=0.1)\n\n# SubsetSampler for train & validation\ntrain_sampler = SubsetRandomSampler(list(tr.index))\nvalid_sampler = SubsetRandomSampler(list(val.index))\n\n# Train Dataset\ndataset = BlindnessDataset(df=train, datatype='train', transform=data_transforms, y=y)\n\n#Test Dataset\ntest_set = BlindnessDataset(df=test, datatype='test', transform=data_transforms_test)\n\n# prepare data loaders (combine dataset and sampler)\ntrain_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, \n                                           sampler=train_sampler, num_workers=num_workers)\nvalid_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, \n                                           sampler=valid_sampler, num_workers=num_workers)\ntest_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, \n                                          num_workers=num_workers)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images_batch, labels_batch = iter(train_loader).next()\nprint(images_batch.shape)\nprint(labels_batch.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = torchvision.models.resnet50()\nmodel.load_state_dict(torch.load(\"../input/resnet50/resnet50.pth\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Freeze model weights\nfor param in model.parameters():\n    param.requires_grad = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_parameters(model):\n    '''\n    Count of trainable weights in a model\n    '''\n    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n\ncount_parameters(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for name, child in model.named_children():\n    print(name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for name, child in model.named_children():\n    if name in ['layer3', 'layer4']:\n        print(name + ' is unfrozen')\n        for param in child.parameters():\n            param.requires_grad = True\n    else:\n        print(name + ' is frozen')\n        for param in child.parameters():\n            param.requires_grad = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_ftrs = model.fc.in_features\nprint(num_ftrs)\n#model.fc = nn.Sequential(\n#                          nn.BatchNorm1d(num_ftrs, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n#                          nn.Dropout(p=0.5),\n#                          nn.Linear(in_features=2048, out_features=1024, bias=True),\n#                          nn.SELU(),\n#                          #nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n#                          #nn.Dropout(p=0.5),\n#                          #nn.Linear(in_features=2048, out_features=1024, bias=True),\n#                          nn.BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n#                          nn.SELU(),\n#                          nn.Dropout(p=0.4),\n#                          nn.Linear(in_features=1024, out_features=100, bias=True),\n#                          nn.SELU(),\n#                          nn.Dropout(p=0.3),\n#                          nn.Linear(in_features=100, out_features=5, bias=True),\n#                         )\n\nmodel.fc =  model.last_linear = nn.Sequential(\n                          nn.BatchNorm1d(num_ftrs, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n                          nn.Dropout(p=0.25),\n                          nn.Linear(in_features=2048, out_features=2048, bias=True),\n                          nn.ReLU(),\n                          nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n                          nn.Dropout(p=0.5),\n                          nn.Linear(in_features=2048, out_features=5, bias=True),\n                         )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.cuda()\ncriterion = nn.BCEWithLogitsLoss()\n\noptimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr= 1e-4)\n#scheduler = lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)\n#scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.5, patience=2)\nscheduler = lr_scheduler.StepLR(optimizer, 5, gamma=0.2)\nscheduler_cosineAL = lr_scheduler.CosineAnnealingLR(optimizer, len(train_loader), eta_min=1e-6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"useCosine = True\nvalid_ck_min = 0\npatience = 10\n# current number of epochs, where validation loss didn't increase\np = 0\n# whether training should be stopped\nstop = False\n\n# number of epochs to train the model\nn_epochs = 70\nfor epoch in range(1, n_epochs+1):\n    print('Epoch:', format(epoch))\n\n    train_loss = []\n    train_ck_score = []\n\n    for batch_i, (data, target) in enumerate(train_loader):\n        \n        model.train()\n\n        data, target = data.cuda(), target.cuda()\n        \n        #print('target:',format(target.shape))\n\n        optimizer.zero_grad()\n        output = model(data)\n        \n        #print('output:',format(output.shape))\n        \n        loss = criterion(output, target.float())\n        train_loss.append(loss.item())\n        \n        a = target.data.cpu().numpy()\n        #b = output[:,-1].detach().cpu().numpy()\n        b = output.detach().cpu().numpy()\n        train_ck_score.append(cohen_k_score(a, b))\n        \n        loss.backward()\n        optimizer.step()\n        \n    model.eval()\n    val_loss = []\n    val_ck_score = []\n    for batch_i, (data, target) in enumerate(valid_loader):\n        data, target = data.cuda(), target.cuda()\n        output = model(data)\n\n        loss = criterion(output, target.float())\n\n        val_loss.append(loss.item()) \n        a = target.data.cpu().numpy()\n        b = output.detach().cpu().numpy()\n        val_ck_score.append(cohen_k_score(a, b))\n\n    # print(f'Epoch {epoch}, train loss: {np.mean(train_loss):.4f}, valid loss: {np.mean(val_loss):.4f}, train auc: {np.mean(train_auc):.4f}, valid auc: {np.mean(val_auc):.4f}')\n    print(f'Epoch {epoch}, train loss: {np.mean(train_loss):.4f}, valid loss: {np.mean(val_loss):.4f}.')\n    print(f'Epoch {epoch}, train cohen: {np.mean(train_ck_score):.4f}, valid cohen: {np.mean(val_ck_score):.4f}.')\n    \n    #valid_loss = np.mean(val_loss)\n    val_ck_score = np.mean(val_ck_score)\n    \n    if useCosine:\n        scheduler_cosineAL.step()\n    else:\n        scheduler.step()\n    \n    #scheduler.step(val_ck_score)\n    if val_ck_score > valid_ck_min:\n        print('Validation CK score increased ({:.6f} --> {:.6f}).  Saving model ...'.format(\n        valid_ck_min,\n        val_ck_score))\n        #torch.save(model_conv.state_dict(), 'model.pt')\n        valid_ck_min = val_ck_score\n        p = 0\n\n    # check if validation loss didn't improve\n    if val_ck_score < valid_ck_min:\n        p += 1\n        print(f'{p} epochs of decreasing val ck score')\n        if p > patience:\n            print('Stopping training : Early Stopping')\n            stop = True\n            break        \n            \n    if stop:\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\n\nmodel.eval()\nfor (data, target, name) in test_loader:\n    data = data.cuda()\n    output = model(data)\n    output = output.cpu().detach().numpy()\n    for i, (e, n) in enumerate(list(zip(output, name))):\n        sub.loc[sub['id_code'] == n.split('/')[-1].split('.')[0], 'diagnosis'] = le_full.inverse_transform([np.argmax(e)])\n        \nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.diagnosis.value_counts(normalize = True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}