{"cells":[{"metadata":{},"cell_type":"markdown","source":"# imports"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import confusion_matrix, cohen_kappa_score\nimport sys\nimport numpy as np\nimport pandas as pd\nimport torchvision\nimport torch.nn as nn\nfrom tqdm import tqdm_notebook as tqdm\nfrom PIL import Image, ImageFile\nfrom torch.utils.data import Dataset\nfrom torch import optim, cuda\nimport torch\nfrom torchvision import transforms\nimport os\nfrom sklearn.model_selection import train_test_split\nfrom torch.utils.data.sampler import SubsetRandomSampler\ndevice = torch.device(\"cuda:0\")\nImageFile.LOAD_TRUNCATED_IMAGES = True\nfrom timeit import default_timer as timer\nimport scipy as sp\nfrom functools import partial\nfrom sklearn import metrics\nfrom collections import Counter\nimport cv2\nimport random\nfrom os.path import isfile","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything(2020)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"learning_rate=1e-5\nepochs=15\nnum_workers=2\nbatch_size=16\nIMG_SIZE    = 256","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocess Image"},{"metadata":{"trusted":true},"cell_type":"code","source":"train2015df = pd.read_csv('../input/retinopathy-train-2015/rescaled_train_896/trainLabels.csv')\ntrain2015df[\"id_code\"]=train2015df[\"image\"]\ntrain2015df[\"diagnosis\"]=train2015df[\"level\"]\ntrain2015df=train2015df.drop(columns=[\"image\",\"level\"])\ntraindf = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\ntraindf2 = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\n_,cvdf=train_test_split(traindf2, test_size=0.25, random_state=2019, stratify=traindf2.diagnosis)\ntestdf = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\ntraindf=traindf.append(train2015df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train      = '../input/aptos2019-blindness-detection/train_images/'\ntest       = '../input/aptos2019-blindness-detection/test_images/'\ntrain_2015 = '../input/retinopathy-train-2015/rescaled_train_896/rescaled_train_896/'\ndef expand_path(p):\n    p = str(p)\n    if isfile(train + p + \".png\"):\n        return train + (p + \".png\")\n    if isfile(train_2015 + p + '.png'):\n        return train_2015 + (p + \".png\")\n    if isfile(test + p + \".png\"):\n        return test + (p + \".png\")\n    return p\n\ndef p_show(imgs, label_name=None, per_row=3):\n    n = len(imgs)\n    rows = (n + per_row - 1)//per_row\n    cols = min(per_row, n)\n    fig, axes = plt.subplots(rows,cols, figsize=(15,15))\n    for ax in axes.flatten(): ax.axis('off')\n    for i,(p, ax) in enumerate(zip(imgs, axes.flatten())): \n        img = Image.open(expand_path(p))\n        ax.imshow(img)\n        ax.set_title(train_df[train_df.id_code == p].diagnosis.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def crop_image1(img,tol=7):\n    # img is image data\n    # tol  is tolerance\n        \n    mask = img>tol\n    return img[np.ix_(mask.any(1),mask.any(0))]\n\ndef crop_image_from_gray(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n    #         print(img1.shape,img2.shape,img3.shape)\n            img = np.stack([img1,img2,img3],axis=-1)\n    #         print(img.shape)\n        return img\n\ndef load_ben_color(image, sigmaX=10 ):\n    #image = cv2.imread(path)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = crop_image_from_gray(image)\n    image = cv2.resize(image, (IMG_SIZE, IMG_SIZE))\n    image=cv2.addWeighted ( image,4, cv2.GaussianBlur( image , (0,0) , sigmaX) ,-4 ,128)\n    return image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# dataset and Data Loader"},{"metadata":{"trusted":true},"cell_type":"code","source":"class RetinopathyDatasetTest(Dataset):\n    def __init__(self, data, transform, dtype='train',mode='train'):\n        self.data = data\n        self.transform = transform\n        self.dtype=dtype\n        self.mode=mode\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        #directory='../input/aptos2019-blindness-detection/'+self.dtype+'_images'\n        #img_name = os.path.join(directory, self.data.loc[idx, 'id_code'] + '.png')\n        img_name=expand_path(self.data.loc[idx, 'id_code'])\n        #print(img_name)\n        image = cv2.imread(img_name)\n        image = load_ben_color(image, sigmaX=10 )\n        image = Image.fromarray(image)\n        image = self.transform(image)\n        label=self.data['diagnosis'][idx]\n        if(self.mode=='test'):\n            return {'image': image}\n        elif(self.mode=='train'):\n            return image,label\n        else :\n            return {'image': image},label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_df, val_df = train_test_split(traindf, test_size=0.25, random_state=2019, stratify=traindf.diagnosis)\ntrain_df=traindf\ntrain_df=train_df.reset_index(drop=True)\n#val_df=val_df.reset_index(drop=True)\ncvdf=cvdf.reset_index(drop=True)\nsample_df=pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\ntransform = transforms.Compose([\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomRotation((-120, 120)),\n    transforms.ToTensor(),\n    transforms.Normalize([0.55387044 ,0.5336158 , 0.5232084 ], [0.2063595 , 0.18827829, 0.15147245])\n])\n\ntest_transform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize([0.55387044 ,0.5336158 , 0.5232084 ], [0.2063595 , 0.18827829, 0.15147245])\n])\ntrain_dataset = RetinopathyDatasetTest(data=train_df,\n                                      transform=transform,dtype='train',mode='train')\n#valid_dataset = RetinopathyDatasetTest(data=val_df,\n#                                      transform=transform,dtype='train',mode='train')\ntest_dataset = RetinopathyDatasetTest(sample_df,\n                                      transform=test_transform,dtype='test',mode='test')\ncv_dataset= RetinopathyDatasetTest(cvdf,\n                                      transform=transform,dtype='train',mode='train')\n#for confusion matrix validation\n#valid_dataset = RetinopathyDatasetTest(csv_file='../input/train.csv',transform=transform,dtype='train',mode='valid')\ntr, val = train_test_split(train_df.diagnosis, stratify=train_df.diagnosis, test_size=0.25)\ntrain_sampler = SubsetRandomSampler(list(tr.index))\nvalid_sampler = SubsetRandomSampler(list(val.index))\n#valid_dataset = torch.utils.data.Subset(dataset, list(val.index))\n#train_dataset=torch.utils.data.Subset(dataset, list(tr.index))\ntrain_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size,sampler=train_sampler, num_workers=num_workers)\nvalid_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size,sampler=valid_sampler, num_workers=num_workers)\ncv_loader=torch.utils.data.DataLoader(cv_dataset, batch_size=batch_size, num_workers=num_workers)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = torch.load(\"../input/pytorch-resnext/complete_modelv50.pth\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fine Tune"},{"metadata":{"trusted":true},"cell_type":"code","source":"def finetune(model,learningRate,n_epochs=epochs):\n    criterion=nn.MSELoss()\n    optimizer=optim.Adam(model.parameters(),lr=learningRate,weight_decay=1e-5)\n    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5)\n    save_file_name='resnetx101.pt'\n    max_epochs_stop=5\n    print_every=1\n    for param in model.parameters():\n                param.requires_grad = True\n    optimizer=optim.Adam(model.parameters(),lr=learningRate,weight_decay=1e-5)\n    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5)\n    epochs_no_improve = 0\n    valid_loss_min = np.Inf\n    valid_max_acc = 0\n    history = []\n    try:\n        print(f'Model has been trained for: {model.epochs} epochs.\\n')\n    except:\n        model.epochs = 0\n        print(f'Starting Training from Scratch.\\n')\n    overall_start = timer()\n    for epoch in (range(n_epochs)):\n        torch.cuda.empty_cache()\n        # keep track of training and validation loss each epoch\n        train_loss = 0.0\n        valid_loss = 0.0\n\n        train_mse = 0\n        valid_mse = 0\n\n        # Set to training\n        model.train()\n        start = timer()\n\n        # Training loop\n        for ii, (data, target) in enumerate((train_loader)):\n            data, target =data.cuda(), target.cuda().float()\n            target=target.unsqueeze(1)\n            # Clear gradients\n            optimizer.zero_grad()\n            # Predicted outputs are log probabilities\n            output = model(data)\n\n            # Loss and backpropagation of gradients\n            loss = criterion(output, target)\n            loss.backward()\n\n            # Update the parameters\n            optimizer.step()\n            # Track train loss by multiplying average loss by number of examples in batch\n            train_loss += loss.item() \n            # Calculate accuracy by finding max log probability\n            #_, pred = torch.max(output, dim=1)\n            #correct_tensor = pred.eq(target.data.view_as(pred))\n            # Need to convert correct tensor from int to float to average\n            #accuracy = torch.mean(correct_tensor.type(torch.FloatTensor))\n            # Multiply average accuracy times the number of examples in batch\n            #train_acc += accuracy.item() * data.size(0)\n\n            # Track training progress\n            print(\n                f'Epoch: {epoch}\\t{100 * (ii + 1) / len(train_loader):.2f}% complete. {timer() - start:.2f} seconds elapsed in epoch.',\n                end='\\r')\n\n        # After training loops ends, start validation\n        else:\n            model.epochs += 1\n                # Don't need to keep track of gradients\n            with torch.no_grad():\n                # Set to evaluation mode\n                model.eval()\n\n                # Validation loop\n                for data, target in cv_loader:\n                    # Tensors to gpu\n                    data, target = data.cuda(), target.cuda().float()\n                    target=target.unsqueeze(1)\n                    # Forward pass\n                    output = model(data)\n\n                    # Validation loss\n                    loss = criterion(output, target)\n                    # Multiply average loss times the number of examples in batch\n                    valid_loss += loss.item()\n                    # Calculate validation accuracy\n                    #_, pred = torch.max(output, dim=1)\n                    #correct_tensor = pred.eq(target.data.view_as(pred))\n                    #accuracy = torch.mean(\n                        #correct_tensor.type(torch.FloatTensor))\n                    # Multiply average accuracy times the number of examples\n                    #valid_acc += accuracy.item() * data.size(0)\n\n                # Calculate average losses\n                train_loss = train_loss / len(train_loader)\n                valid_loss = valid_loss / len(cv_loader)\n\n                # Calculate average accuracy\n                #train_acc = train_acc / len(train_loader.dataset)\n                #valid_acc = valid_acc / len(valid_loader.dataset)\n\n                history.append([train_loss, valid_loss,scheduler.get_lr()[0]])\n\n                # Print training and validation results\n                if (epoch + 1) % print_every == 0:\n                    print(\n                        f'\\nEpoch: {epoch} \\tTraining Loss: {train_loss:.4f} \\tValidation Loss: {valid_loss:.4f}'\n                    )\n                    print('lr:', scheduler.get_lr()[0])\n                    #print(\n                    #    f'\\t\\tTraining Accuracy: {100 * train_acc:.2f}%\\t Validation Accuracy: {100 * valid_acc:.2f}%'\n                    #)\n\n                # Save the model if validation loss decreases\n                if valid_loss < valid_loss_min:\n                    # Save model\n                    torch.save(model.state_dict(), save_file_name)\n                    # Track improvement\n                    epochs_no_improve = 0\n                    valid_loss_min = valid_loss\n                    best_epoch = epoch\n\n                # Otherwise increment count of epochs with no improvement\n                else:\n                    epochs_no_improve += 1\n                    # Trigger early stopping\n                    if epochs_no_improve >= max_epochs_stop:\n                        print(f'\\nEarly Stopping! Total epochs: {epoch}. Best epoch: {best_epoch} with loss: {valid_loss_min:.2f}')\n                        total_time = timer() - overall_start\n                        print(f'{total_time:.2f} total seconds elapsed. {total_time / (epoch+1):.2f} seconds per epoch.')\n\n                        # Load the best state dict\n                        model.load_state_dict(torch.load(save_file_name))\n                        # Attach the optimizer\n                        model.optimizer = optimizer\n\n                        # Format history\n                        history = pd.DataFrame(history,columns=['train_loss', 'valid_loss','Learning_rate'])\n                        return model,history\n        scheduler.step()\n    # Attach the optimizer\n    model.optimizer = optimizer\n    # Record overall time and print out stats\n    total_time = timer() - overall_start\n    print(f'\\nBest epoch: {best_epoch} with loss: {valid_loss_min:.2f} ')\n    print(f'{total_time:.2f} total seconds elapsed. {total_time / (epoch+1):.2f} seconds per epoch.')\n    # Format history\n    history = pd.DataFrame(history,columns=['train_loss', 'valid_loss','Learning_rate'])\n    return model,history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"model,history=finetune(model,learningRate=1e-3,n_epochs=15)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Plot Losses"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(8, 6))\nfor c in ['train_loss', 'valid_loss']:\n    plt.plot(\n        history[c], label=c)\nplt.legend()\nplt.xlabel('Epoch')\nplt.ylabel('MSE Losses')\nplt.title('Training and Validation Losses')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"history"},{"metadata":{},"cell_type":"markdown","source":"## train optimized Cohen kappa"},{"metadata":{"trusted":true},"cell_type":"code","source":"class OptimizedRounder(object):\n    def __init__(self):\n        self.coef_ = 0\n\n    def _kappa_loss(self, coef, X, y):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n\n        ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')\n        return -ll\n\n    def fit(self, X, y):\n        loss_partial = partial(self._kappa_loss, X=X, y=y)\n        initial_coef = [0.5, 1.5, 2.5, 3.5]\n        self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')\n        print(-loss_partial(self.coef_['x']))\n\n    def predict(self, X, coef):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n        return X_p\n\n    def coefficients(self):\n        return self.coef_['x']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def Run_validation():\n    torch.cuda.empty_cache()\n    valid_preds = []\n    labels=[]\n    tk0 = tqdm(cv_loader)\n    for ii, (x_batch,label) in enumerate(tk0):\n         with torch.no_grad():\n            #x_batch = x_batch[\"image\"]\n            pred = model(x_batch.to(device))\n            valid_preds.append(pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1))\n            labels.append(label)\n    predict_valid=valid_preds[0]\n    for i in range(1,len(cv_loader)):\n        predict_valid=np.append(predict_valid,valid_preds[i])\n    label_valid=labels[0].numpy()\n    for i in range(1,len(cv_loader)):\n        label_valid=np.append(label_valid,labels[i].numpy())\n    return predict_valid,label_valid","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def confusionMatrics(preds_valid,label_valid):\n    labels = ['0 - No DR', '1 - Mild', '2 - Moderate', '3 - Severe', '4 - Proliferative DR']\n    cnf_matrix = confusion_matrix(label_valid, preds_valid)\n    cnf_matrix_norm = cnf_matrix.astype('float') / cnf_matrix.sum(axis=1)[:, np.newaxis]\n    df_cm = pd.DataFrame(cnf_matrix_norm, index=labels, columns=labels)\n    plt.figure(figsize=(16, 7))\n    sns.heatmap(df_cm, annot=True, fmt='.2f', cmap=\"Blues\")\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict_valid,label_valid=Run_validation()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\noptR = OptimizedRounder()\noptR.fit(predict_valid,label_valid)\ncoefficients=optR.coefficients()\nprint(coefficients)\npreds_valid=optR.predict(predict_valid,coefficients)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"confusionMatrics(preds_valid,label_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Train Cohen Kappa score: %.3f\" % cohen_kappa_score(preds_valid, label_valid, weights='quadratic'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Validation"},{"metadata":{"trusted":true},"cell_type":"code","source":"predict_valid_1,label_valid_1=Run_validation()\npreds_valid_1=optR.predict(predict_valid_1,coefficients)\nconfusionMatrics(preds_valid_1,label_valid_1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Train Cohen Kappa score: %.3f\" % cohen_kappa_score(preds_valid_1, label_valid_1, weights='quadratic'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Witout Optimizing"},{"metadata":{"trusted":true},"cell_type":"code","source":"coefficients_1=[0.5, 1.5, 2.5, 3.5]\npreds_valid_2=optR.predict(predict_valid_1,coefficients_1)\nconfusionMatrics(preds_valid_2,label_valid_1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Train Cohen Kappa score: %.3f\" % cohen_kappa_score(preds_valid_2, label_valid_1, weights='quadratic'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predict With TTA"},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in tqdm(range(10)):\n    test_preds=[]\n    torch.cuda.empty_cache()\n    test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=4)\n    test_preds1 = np.zeros((len(test_dataset), 1))\n    tk0 = tqdm(test_data_loader)\n    for i, x_batch in enumerate(tk0):\n        with torch.no_grad():\n            x_batch = x_batch[\"image\"]\n            pred = model(x_batch.to(device))\n            test_preds1[i * batch_size:(i + 1) * batch_size] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)\n            test_preds.append(test_preds1)\npredict = np.mean(test_preds, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds=optR.predict(predict,coefficients)\nsample = pd.read_csv(\"../input/aptos2019-blindness-detection/sample_submission.csv\")\nsample.diagnosis = test_preds.astype(int)\nsample.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, ax = plt.subplots(figsize=(14, 8.7))\nax = sns.countplot(x=\"diagnosis\", data=sample, palette=\"GnBu_d\")\nsns.despine()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.save(model,\"final_traindv25.pth\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}