{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import cv2\nimport matplotlib.pyplot as plt\nfrom os.path import isfile\nimport torch\nimport torch.nn as nn\nimport numpy as np\nimport pandas as pd \nimport os\nfrom PIL import Image, ImageFilter\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\nfrom torch.optim import Adam, SGD, RMSprop\nimport time\nfrom torch.autograd import Variable\nimport torch.functional as F\nfrom tqdm import tqdm\nfrom sklearn import metrics\nimport urllib\nimport pickle\nimport cv2\nimport torch.nn.functional as F\nfrom torch.nn.parameter import Parameter\nfrom torchvision import models\nimport seaborn as sns\nimport random\nimport sys\nimport gc","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:57:07.108076Z","iopub.execute_input":"2022-08-16T04:57:07.109974Z","iopub.status.idle":"2022-08-16T04:57:10.758032Z","shell.execute_reply.started":"2022-08-16T04:57:07.109843Z","shell.execute_reply":"2022-08-16T04:57:10.756705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed_everything(123)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:57:10.760819Z","iopub.execute_input":"2022-08-16T04:57:10.762465Z","iopub.status.idle":"2022-08-16T04:57:10.774906Z","shell.execute_reply.started":"2022-08-16T04:57:10.762422Z","shell.execute_reply":"2022-08-16T04:57:10.773248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(f'Device : {device.upper()}')","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:57:10.778471Z","iopub.execute_input":"2022-08-16T04:57:10.780515Z","iopub.status.idle":"2022-08-16T04:57:10.875176Z","shell.execute_reply.started":"2022-08-16T04:57:10.780426Z","shell.execute_reply":"2022-08-16T04:57:10.872925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train      = '../input/aptos2019-blindness-detection/train_images/'\ntest       = '../input/aptos2019-blindness-detection/test_images/'\ntrain_csv  = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\n\ntrain_df, val_df = train_test_split(train_csv, test_size=0.1, stratify=train_csv.diagnosis)\ntrain_df.reset_index(drop=True, inplace=True)\nval_df.reset_index(drop=True, inplace=True)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:57:10.881097Z","iopub.execute_input":"2022-08-16T04:57:10.882869Z","iopub.status.idle":"2022-08-16T04:57:10.932717Z","shell.execute_reply.started":"2022-08-16T04:57:10.882757Z","shell.execute_reply":"2022-08-16T04:57:10.931274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def expand_path(p):\n    p = str(p)\n    if isfile(train + p + \".png\"):\n        return train + (p + \".png\")\n    if isfile(train_2015 + p + '.png'):\n        return train_2015 + (p + \".png\")\n    if isfile(test + p + \".png\"):\n        return test + (p + \".png\")\n    return p\n\ndef p_show(imgs, label_name=None, per_row=3):\n    n = len(imgs)\n    rows = (n + per_row - 1)//per_row\n    cols = min(per_row, n)\n    fig, axes = plt.subplots(rows,cols, figsize=(15,15))\n    for ax in axes.flatten(): ax.axis('off')\n    for i,(p, ax) in enumerate(zip(imgs, axes.flatten())): \n        img = Image.open(expand_path(p))\n        ax.imshow(img)\n        ax.set_title(train_df[train_df.id_code == p].diagnosis.values)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:57:10.934642Z","iopub.execute_input":"2022-08-16T04:57:10.935687Z","iopub.status.idle":"2022-08-16T04:57:10.948070Z","shell.execute_reply.started":"2022-08-16T04:57:10.935647Z","shell.execute_reply":"2022-08-16T04:57:10.946413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs = []\nfor p in train_df.id_code:\n    imgs.append(p)\n    if len(imgs) == 16: break\np_show(imgs)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:57:10.950099Z","iopub.execute_input":"2022-08-16T04:57:10.951840Z","iopub.status.idle":"2022-08-16T04:57:24.257750Z","shell.execute_reply.started":"2022-08-16T04:57:10.951795Z","shell.execute_reply":"2022-08-16T04:57:24.256615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def crop_image1(img,tol=7):\n    # img is image data\n    # tol  is tolerance\n        \n    mask = img>tol\n    return img[np.ix_(mask.any(1),mask.any(0))]\n\ndef crop_image_from_gray(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n    #         print(img1.shape,img2.shape,img3.shape)\n            img = np.stack([img1,img2,img3],axis=-1)\n    #         print(img.shape)\n        return img","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:57:24.258869Z","iopub.execute_input":"2022-08-16T04:57:24.259187Z","iopub.status.idle":"2022-08-16T04:57:24.274600Z","shell.execute_reply.started":"2022-08-16T04:57:24.259144Z","shell.execute_reply":"2022-08-16T04:57:24.273000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_SIZE    = 400\nLR = 1e-4\nBATCH_SIZE = 32\nEPOCHS = 20","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:57:24.276879Z","iopub.execute_input":"2022-08-16T04:57:24.277792Z","iopub.status.idle":"2022-08-16T04:57:24.288380Z","shell.execute_reply.started":"2022-08-16T04:57:24.277753Z","shell.execute_reply":"2022-08-16T04:57:24.286936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MyDataset(Dataset):\n    \n    def __init__(self, dataframe, transform=None):\n        self.df = dataframe\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        \n        label = self.df.diagnosis.values[idx]\n        label = np.expand_dims(label, -1)\n        \n        p = self.df.id_code.values[idx]\n        p_path = expand_path(p)\n        image = cv2.imread(p_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = crop_image_from_gray(image)\n        image = cv2.resize(image, (IMG_SIZE, IMG_SIZE))\n        image = cv2.addWeighted ( image,4, cv2.GaussianBlur( image , (0,0) , 30) ,-4 ,128)\n        image = transforms.ToPILImage()(image)\n        \n        if self.transform:\n            image = self.transform(image)\n        \n        image = np.resize(image, (3,IMG_SIZE, IMG_SIZE))\n        image = torch.tensor(image)\n        label = torch.tensor(label)\n        \n        return image, label","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:57:24.290885Z","iopub.execute_input":"2022-08-16T04:57:24.291862Z","iopub.status.idle":"2022-08-16T04:57:24.306878Z","shell.execute_reply.started":"2022-08-16T04:57:24.291805Z","shell.execute_reply":"2022-08-16T04:57:24.305495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transform = transforms.Compose([\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomRotation((-120, 120)),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])\n\ntrainset     = MyDataset(train_df, transform =train_transform)\ntrain_loader = torch.utils.data.DataLoader(trainset, batch_size=32, shuffle=True, num_workers=2)\nvalset       = MyDataset(val_df, transform   =train_transform)\nval_loader   = torch.utils.data.DataLoader(valset, batch_size=32, shuffle=False, num_workers=2)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:57:24.314438Z","iopub.execute_input":"2022-08-16T04:57:24.316849Z","iopub.status.idle":"2022-08-16T04:57:24.326161Z","shell.execute_reply.started":"2022-08-16T04:57:24.316804Z","shell.execute_reply":"2022-08-16T04:57:24.324794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gem(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM,self).__init__()\n        self.p = Parameter(torch.ones(1)*p)\n        self.eps = eps\n    def forward(self, x):\n        return gem(x, p=self.p, eps=self.eps)       \n    def __repr__(self):\n        return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + ', ' + 'eps=' + str(self.eps) + ')'","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:57:24.329097Z","iopub.execute_input":"2022-08-16T04:57:24.330249Z","iopub.status.idle":"2022-08-16T04:57:24.343519Z","shell.execute_reply.started":"2022-08-16T04:57:24.330204Z","shell.execute_reply":"2022-08-16T04:57:24.341841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = models.efficientnet_b0(pretrained = True)\nmodel.classifier = nn.Linear(in_features=1280, out_features=1, bias=True)\n# model.avgpool = GeM()\nmodel.to(device)\n\ncriterion = nn.MSELoss()\noptimizer = torch.optim.Adam(model.parameters(), lr = LR, weight_decay=1e-5)\nscheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0 = 50, T_mult=1, eta_min=0, last_epoch=- 1, verbose=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:57:24.347217Z","iopub.execute_input":"2022-08-16T04:57:24.347710Z","iopub.status.idle":"2022-08-16T04:57:30.868892Z","shell.execute_reply.started":"2022-08-16T04:57:24.347655Z","shell.execute_reply":"2022-08-16T04:57:30.867569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(1, EPOCHS + 1):\n    train_loss = []\n    val_loss = []\n    torch.cuda.empty_cache()\n    gc.collect()\n    model.train()\n    for idx, (X_batch, y_batch) in enumerate(tqdm(train_loader)):\n        optimizer.zero_grad()\n        X_batch = X_batch.to(device)\n        y_batch = y_batch.to(device)\n        output = model(X_batch)\n        loss = criterion(output.float(), y_batch.float())\n        train_loss.append(loss.item())\n        loss.backward()\n        optimizer.step()\n        scheduler.step()\n    model.eval()\n    torch.cuda.empty_cache()\n    gc.collect()\n    for idx, (X_batch, y_batch) in enumerate(tqdm(val_loader)):\n        X_batch = X_batch.to(device)\n        y_batch = y_batch.to(device)\n        output = model(X_batch)\n        loss = criterion(output.float(), y_batch.float())\n        val_loss.append(loss.item())\n    avg_loss = np.mean(val_loss)\n    if avg_loss < best:\n        best = val_loss\n        torch.save(model.state_dict(), 'model.bin')\n    print(f'Epoch : {i}')\n    print(f'Training loss : {np.mean(train_loss)}')\n    print(f'Valid loss : {avg_loss}')","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:58:22.164249Z","iopub.execute_input":"2022-08-16T04:58:22.164689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()\ny_pred = []\ny_true = []\nwith torch.no_grad():\n    for (X, y) in tqdm(val_loader):\n        X = X.to(device)\n        output = model(X)\n        y_pred.extend(output)\n        y_true.extend(y)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:58:07.930053Z","iopub.status.idle":"2022-08-16T04:58:07.931161Z","shell.execute_reply.started":"2022-08-16T04:58:07.930787Z","shell.execute_reply":"2022-08-16T04:58:07.930818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []\nfor val in y_pred:\n    preds.append(val.detach().tolist()[0])","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:58:07.933058Z","iopub.status.idle":"2022-08-16T04:58:07.934055Z","shell.execute_reply.started":"2022-08-16T04:58:07.933746Z","shell.execute_reply":"2022-08-16T04:58:07.933776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"true = []\nfor val in y_true:\n    true.append(val.detach().tolist()[0])","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:58:07.935988Z","iopub.status.idle":"2022-08-16T04:58:07.937043Z","shell.execute_reply.started":"2022-08-16T04:58:07.936664Z","shell.execute_reply":"2022-08-16T04:58:07.936705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(true), len(preds))","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:58:07.939012Z","iopub.status.idle":"2022-08-16T04:58:07.939992Z","shell.execute_reply.started":"2022-08-16T04:58:07.939661Z","shell.execute_reply":"2022-08-16T04:58:07.939700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = pd.DataFrame(columns = ['y_true', 'y_pred'])\npredictions['y_true'] = true\npredictions['y_pred'] = preds","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:58:07.942016Z","iopub.status.idle":"2022-08-16T04:58:07.943037Z","shell.execute_reply.started":"2022-08-16T04:58:07.942726Z","shell.execute_reply":"2022-08-16T04:58:07.942755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions[predictions['y_true'] == 0].describe().T","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:58:07.944951Z","iopub.status.idle":"2022-08-16T04:58:07.945979Z","shell.execute_reply.started":"2022-08-16T04:58:07.945616Z","shell.execute_reply":"2022-08-16T04:58:07.945645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions[predictions['y_true'] == 1].describe().T","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:58:07.947884Z","iopub.status.idle":"2022-08-16T04:58:07.948958Z","shell.execute_reply.started":"2022-08-16T04:58:07.948521Z","shell.execute_reply":"2022-08-16T04:58:07.948585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions[predictions['y_true'] == 2].describe().T","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:58:07.950847Z","iopub.status.idle":"2022-08-16T04:58:07.951813Z","shell.execute_reply.started":"2022-08-16T04:58:07.951478Z","shell.execute_reply":"2022-08-16T04:58:07.951507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions[predictions['y_true'] == 3].describe().T","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:58:07.953921Z","iopub.status.idle":"2022-08-16T04:58:07.955098Z","shell.execute_reply.started":"2022-08-16T04:58:07.954700Z","shell.execute_reply":"2022-08-16T04:58:07.954763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions[predictions['y_true'] == 4].describe().T","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:58:07.957230Z","iopub.status.idle":"2022-08-16T04:58:07.958380Z","shell.execute_reply.started":"2022-08-16T04:58:07.958042Z","shell.execute_reply":"2022-08-16T04:58:07.958073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"coef = [0.5, 1.4, 2.5, 3.2]\ntest_preds = []\nfor i, pred in enumerate(predictions['y_pred'].values):\n    if pred < coef[0]:\n        test_preds.append(0)\n    elif pred >= coef[0] and pred < coef[1]:\n        test_preds.append(1)\n    elif pred >= coef[1] and pred < coef[2]:\n        test_preds.append(2)\n    elif pred >= coef[2] and pred < coef[3]:\n        test_preds.append(3)\n    else:\n        test_preds.append(4)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:58:07.960366Z","iopub.status.idle":"2022-08-16T04:58:07.961502Z","shell.execute_reply.started":"2022-08-16T04:58:07.961146Z","shell.execute_reply":"2022-08-16T04:58:07.961180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import cohen_kappa_score\n\ncohen_kappa_score(predictions['y_true'], test_preds, weights = 'quadratic')","metadata":{"execution":{"iopub.status.busy":"2022-08-16T04:58:07.963463Z","iopub.status.idle":"2022-08-16T04:58:07.964555Z","shell.execute_reply.started":"2022-08-16T04:58:07.964191Z","shell.execute_reply":"2022-08-16T04:58:07.964237Z"},"trusted":true},"execution_count":null,"outputs":[]}]}