{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import cv2\nimport matplotlib.pyplot as plt\nfrom os.path import isfile\nimport torch\nimport torch.nn as nn\nimport numpy as np\nimport pandas as pd \nimport os\nfrom PIL import Image, ImageFilter\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\nfrom torch.optim import Adam, SGD, RMSprop\nimport time\nfrom torch.autograd import Variable\nimport torch.functional as F\nfrom tqdm import tqdm\nfrom sklearn import metrics\nimport urllib\nimport pickle\nimport cv2\nimport torch.nn.functional as F\nfrom torch.nn.parameter import Parameter\nfrom torchvision import models\nimport seaborn as sns\nimport random\nimport sys\nimport gc\nimport warnings\nfrom torch.cuda.amp import autocast, GradScaler\ntorch.autograd.set_detect_anomaly(False)\ntorch.autograd.profiler.profile(False)\ntorch.autograd.profiler.emit_nvtx(False)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T06:31:37.032208Z","iopub.execute_input":"2022-08-17T06:31:37.033176Z","iopub.status.idle":"2022-08-17T06:31:37.045782Z","shell.execute_reply.started":"2022-08-17T06:31:37.033128Z","shell.execute_reply":"2022-08-17T06:31:37.044632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 123\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\ntorch.cuda.manual_seed(SEED)\ntorch.cuda.manual_seed_all(SEED)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\n\nwarnings.filterwarnings('ignore')\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(f'\\n Device : {device.upper()}')","metadata":{"execution":{"iopub.status.busy":"2022-08-17T06:31:37.051039Z","iopub.execute_input":"2022-08-17T06:31:37.052047Z","iopub.status.idle":"2022-08-17T06:31:37.061372Z","shell.execute_reply.started":"2022-08-17T06:31:37.052005Z","shell.execute_reply":"2022-08-17T06:31:37.059986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_DIR = '../input/diabetic-retinopathy-unziped/main train/main train'\ndf = pd.read_csv('../input/diabetic-retinopathy-unziped/train.csv')\ndf.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-17T06:31:37.064665Z","iopub.execute_input":"2022-08-17T06:31:37.065535Z","iopub.status.idle":"2022-08-17T06:31:37.094204Z","shell.execute_reply.started":"2022-08-17T06:31:37.065497Z","shell.execute_reply":"2022-08-17T06:31:37.093297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"one = df[df['level'] == 1].iloc[:708,:]\ntwo = df[df['level'] == 2].iloc[:708,:]\nthree = df[df['level'] == 3].iloc[:708,:]\nfour = df[df['level'] == 4]\nzero = df[df['level'] == 0].iloc[:708,:]\nreduced_df = pd.concat([one, two, three, four, zero], ignore_index = True, copy = True)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T06:31:37.095799Z","iopub.execute_input":"2022-08-17T06:31:37.096377Z","iopub.status.idle":"2022-08-17T06:31:37.110946Z","shell.execute_reply.started":"2022-08-17T06:31:37.096339Z","shell.execute_reply":"2022-08-17T06:31:37.109706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reduced_df['level'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-17T06:31:37.113318Z","iopub.execute_input":"2022-08-17T06:31:37.114024Z","iopub.status.idle":"2022-08-17T06:31:37.122404Z","shell.execute_reply.started":"2022-08-17T06:31:37.113989Z","shell.execute_reply":"2022-08-17T06:31:37.121305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df, val_df = train_test_split(reduced_df, test_size=0.1, stratify=reduced_df.level)\ntrain_df.reset_index(drop=True, inplace=True)\nval_df.reset_index(drop=True, inplace=True)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-17T06:31:37.297049Z","iopub.execute_input":"2022-08-17T06:31:37.297418Z","iopub.status.idle":"2022-08-17T06:31:37.314986Z","shell.execute_reply.started":"2022-08-17T06:31:37.297387Z","shell.execute_reply":"2022-08-17T06:31:37.313898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def expand_path(p):\n    p = str(p)\n    if isfile(train + p + \".png\"):\n        return train + (p + \".png\")\n    if isfile(train_2015 + p + '.png'):\n        return train_2015 + (p + \".png\")\n    if isfile(test + p + \".png\"):\n        return test + (p + \".png\")\n    return p\n\ndef p_show(imgs, label_name=None, per_row=3):\n    n = len(imgs)\n    rows = (n + per_row - 1)//per_row\n    cols = min(per_row, n)\n    fig, axes = plt.subplots(rows,cols, figsize=(15,15))\n    for ax in axes.flatten(): ax.axis('off')\n    for i,(p, ax) in enumerate(zip(imgs, axes.flatten())): \n        img = Image.open(expand_path(p))\n        ax.imshow(img)\n        ax.set_title(train_df[train_df.id_code == p].diagnosis.values)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T06:31:37.317549Z","iopub.execute_input":"2022-08-17T06:31:37.318406Z","iopub.status.idle":"2022-08-17T06:31:37.327187Z","shell.execute_reply.started":"2022-08-17T06:31:37.318353Z","shell.execute_reply":"2022-08-17T06:31:37.326205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def crop_image1(img,tol=7):\n    # img is image data\n    # tol  is tolerance\n        \n    mask = img>tol\n    return img[np.ix_(mask.any(1),mask.any(0))]\n\ndef crop_image_from_gray(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n    #         print(img1.shape,img2.shape,img3.shape)\n            img = np.stack([img1,img2,img3],axis=-1)\n    #         print(img.shape)\n        return img","metadata":{"execution":{"iopub.status.busy":"2022-08-17T06:31:37.328666Z","iopub.execute_input":"2022-08-17T06:31:37.329320Z","iopub.status.idle":"2022-08-17T06:31:37.341267Z","shell.execute_reply.started":"2022-08-17T06:31:37.329284Z","shell.execute_reply":"2022-08-17T06:31:37.340254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_SIZE = 300\nLR = 1e-4\nBATCH_SIZE = 32\nEPOCHS = 20","metadata":{"execution":{"iopub.status.busy":"2022-08-17T06:31:37.344642Z","iopub.execute_input":"2022-08-17T06:31:37.344946Z","iopub.status.idle":"2022-08-17T06:31:37.350965Z","shell.execute_reply.started":"2022-08-17T06:31:37.344900Z","shell.execute_reply":"2022-08-17T06:31:37.349759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MyDataset(Dataset):\n    \n    def __init__(self, dataframe, transform=None):\n        self.df = dataframe\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        \n        label = self.df.level.values[idx]\n        label = np.expand_dims(label, -1)\n        \n        p = self.df.image.values[idx]\n        p_path = IMG_DIR + '/' + p + '.jpeg'\n        image = cv2.imread(p_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = crop_image_from_gray(image)\n        image = cv2.resize(image, (IMG_SIZE, IMG_SIZE))\n        image = cv2.addWeighted ( image,4, cv2.GaussianBlur( image , (0,0) , 30) ,-4 ,128)\n        image = transforms.ToPILImage()(image)\n        \n        if self.transform:\n            image = self.transform(image)\n        \n        image = np.resize(image, (3,IMG_SIZE, IMG_SIZE))\n        image = torch.tensor(image)\n        label = torch.tensor(label)\n        \n        return image, label","metadata":{"execution":{"iopub.status.busy":"2022-08-17T06:31:37.354146Z","iopub.execute_input":"2022-08-17T06:31:37.354517Z","iopub.status.idle":"2022-08-17T06:31:37.365052Z","shell.execute_reply.started":"2022-08-17T06:31:37.354468Z","shell.execute_reply":"2022-08-17T06:31:37.364148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transform = transforms.Compose([\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomRotation((-120, 120)),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])\n\ntrainset     = MyDataset(train_df, transform =train_transform)\ntrain_loader = torch.utils.data.DataLoader(trainset, batch_size=32, shuffle=True, num_workers=2, pin_memory=True)\nvalset       = MyDataset(val_df, transform   =train_transform)\nval_loader   = torch.utils.data.DataLoader(valset, batch_size=32, shuffle=False, num_workers=2, pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T06:31:37.366306Z","iopub.execute_input":"2022-08-17T06:31:37.366802Z","iopub.status.idle":"2022-08-17T06:31:37.378952Z","shell.execute_reply.started":"2022-08-17T06:31:37.366767Z","shell.execute_reply":"2022-08-17T06:31:37.377974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = models.efficientnet_b0(pretrained = True)\nmodel.classifier = nn.Linear(in_features=1280, out_features=1, bias=True)\n# model.avgpool = GeM()\nmodel.to(device)\nmodel = model\ncriterion = nn.MSELoss()\noptimizer = torch.optim.Adam(model.parameters(), lr = LR, weight_decay=1e-5)\nscheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0 = 50, T_mult=1, eta_min=0, last_epoch=- 1, verbose=False)\n# scaler = GradScaler()","metadata":{"execution":{"iopub.status.busy":"2022-08-17T06:31:37.380626Z","iopub.execute_input":"2022-08-17T06:31:37.381995Z","iopub.status.idle":"2022-08-17T06:31:37.626806Z","shell.execute_reply.started":"2022-08-17T06:31:37.381955Z","shell.execute_reply":"2022-08-17T06:31:37.625659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Cmatrix(rater_a, rater_b, min_rating=None, max_rating=None):\n    assert(len(rater_a) == len(rater_b))\n    if min_rating is None:\n        min_rating = min(rater_a + rater_b)\n    if max_rating is None:\n        max_rating = max(rater_a + rater_b)\n    num_ratings = int(max_rating - min_rating + 1)\n    conf_mat = [[0 for i in range(num_ratings)]\n                for j in range(num_ratings)]\n    for a, b in zip(rater_a, rater_b):\n        conf_mat[a - min_rating][b - min_rating] += 1\n    return conf_mat\n\ndef histogram(ratings, min_rating=None, max_rating=None):\n    if min_rating is None:\n        min_rating = min(ratings)\n    if max_rating is None:\n        max_rating = max(ratings)\n    num_ratings = int(max_rating - min_rating + 1)\n    hist_ratings = [0 for x in range(num_ratings)]\n    for r in ratings:\n        hist_ratings[r - min_rating] += 1\n    return hist_ratings\n\ndef quadratic_weighted_kappa(y, y_pred):\n    rater_a = y\n    rater_b = y_pred\n    min_rating=None\n    max_rating=None\n    rater_a = np.array(rater_a, dtype=int)\n    rater_b = np.array(rater_b, dtype=int)\n    assert(len(rater_a) == len(rater_b))\n    if min_rating is None:\n        min_rating = min(min(rater_a), min(rater_b))\n    if max_rating is None:\n        max_rating = max(max(rater_a), max(rater_b))\n    conf_mat = Cmatrix(rater_a, rater_b,\n                                min_rating, max_rating)\n    num_ratings = len(conf_mat)\n    num_scored_items = float(len(rater_a))\n\n    hist_rater_a = histogram(rater_a, min_rating, max_rating)\n    hist_rater_b = histogram(rater_b, min_rating, max_rating)\n\n    numerator = 0.0\n    denominator = 0.0\n\n    for i in range(num_ratings):\n        for j in range(num_ratings):\n            expected_count = (hist_rater_a[i] * hist_rater_b[j]\n                              / num_scored_items)\n            d = pow(i - j, 2.0) / pow(num_ratings - 1, 2.0)\n            numerator += d * conf_mat[i][j] / num_scored_items\n            denominator += d * expected_count / num_scored_items\n\n    return (1.0 - numerator / denominator)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T06:31:37.629861Z","iopub.execute_input":"2022-08-17T06:31:37.630287Z","iopub.status.idle":"2022-08-17T06:31:37.644102Z","shell.execute_reply.started":"2022-08-17T06:31:37.630249Z","shell.execute_reply":"2022-08-17T06:31:37.642913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\ntorch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2022-08-17T06:31:37.645729Z","iopub.execute_input":"2022-08-17T06:31:37.646114Z","iopub.status.idle":"2022-08-17T06:31:38.139830Z","shell.execute_reply.started":"2022-08-17T06:31:37.646078Z","shell.execute_reply":"2022-08-17T06:31:38.138789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best = 100\nfor i in range(1, EPOCHS + 1):\n    train_loss = []\n    val_loss = []\n    torch.cuda.empty_cache()\n    gc.collect()\n    model.train()\n    for idx, (X_batch, y_batch) in enumerate(tqdm(train_loader)):\n        optimizer.zero_grad()\n        X_batch = X_batch.to(device)\n        y_batch = y_batch.to(device)\n#         with autocast():\n        output = model(X_batch)\n        loss = criterion(output.float(), y_batch.float())\n        train_loss.append(loss.item())\n        loss.backward()\n#         scaler.step()\n        optimizer.step()\n    model.eval()\n    torch.cuda.empty_cache()\n    gc.collect()\n    for idx, (X_batch, y_batch) in enumerate(tqdm(val_loader)):\n        X_batch = X_batch.to(device)\n        y_batch = y_batch.to(device)\n        output = model(X_batch)\n        loss = criterion(output.float(), y_batch.float())\n        val_loss.append(loss.item())\n    avg_loss = np.mean(val_loss)\n    if avg_loss < best:\n        best = avg_loss\n        torch.save(model.state_dict(), 'model.bin')\n    print(f'Epoch : {i}')\n    print(f'Training loss : {np.mean(train_loss)}')\n    print(f'Valid loss : {avg_loss}')","metadata":{"execution":{"iopub.status.busy":"2022-08-17T06:31:38.141424Z","iopub.execute_input":"2022-08-17T06:31:38.141784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}