{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#importing libraries \nimport numpy as np\nimport pandas as pd\n\nfrom PIL import Image, ImageFile\nImageFile.LOAD_TRUNCATED_IMAGES = True\nimport cv2\n\nfrom tqdm import tqdm_notebook as tqdm\nfrom functools import partial\nimport scipy as sp\n\nimport random\nimport time\nimport sys\nimport os\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n\nfrom sklearn import metrics\nfrom sklearn.metrics import confusion_matrix\nimport torch\nimport torchvision\n\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torchvision import transforms, models, datasets\nfrom torch.utils.data import Dataset\nfrom torch.autograd import Variable\n\n!pip install efficientnet_pytorch\nfrom efficientnet_pytorch import EfficientNet\n\nimport warnings\nwarnings.filterwarnings('ignore')\n!mkdir models","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-30T05:15:21.904729Z","iopub.execute_input":"2023-10-30T05:15:21.905271Z","iopub.status.idle":"2023-10-30T05:15:36.538199Z","shell.execute_reply.started":"2023-10-30T05:15:21.905233Z","shell.execute_reply":"2023-10-30T05:15:36.536929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndef seed_everything(seed = 23):\n    # tests\n    assert isinstance(seed, int), 'seed has to be an integer'\n    \n    # randomness\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n'''","metadata":{"execution":{"iopub.status.busy":"2023-10-30T05:16:35.300959Z","iopub.execute_input":"2023-10-30T05:16:35.301428Z","iopub.status.idle":"2023-10-30T05:16:35.311105Z","shell.execute_reply.started":"2023-10-30T05:16:35.301391Z","shell.execute_reply":"2023-10-30T05:16:35.310019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_size = 256\n#IMAGE PREPROCESSING\n\ndef prepare_image(path, \n                  sigmaX         = 10, \n                  do_random_crop = False):\n    \n    '''\n    Preprocess image\n    '''\n    \n    # import imagepre\n    image = cv2.imread(path)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    \n    # perform smart crops\n    image = crop_black(image, tol = 7)\n    if do_random_crop == True:\n        image = random_crop(image, size = (0.9, 1))\n    \n    # resize and color\n    image = cv2.resize(image, (int(image_size), int(image_size)))\n    image = cv2.addWeighted(image, 4, cv2.GaussianBlur(image, (0, 0), sigmaX), -4, 128)\n    \n    # circular crop\n    image = circle_crop(image, sigmaX = sigmaX)\n\n    # convert to tensor    \n    image = torch.tensor(image)\n    image = image.permute(2, 1, 0)\n    return image","metadata":{"execution":{"iopub.status.busy":"2023-10-30T05:16:38.619609Z","iopub.execute_input":"2023-10-30T05:16:38.620058Z","iopub.status.idle":"2023-10-30T05:16:38.629647Z","shell.execute_reply.started":"2023-10-30T05:16:38.620023Z","shell.execute_reply":"2023-10-30T05:16:38.628312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#CROP FUNCTIONS\n\ndef crop_black(img, \n               tol = 7):\n    \n    '''\n    Perform automatic crop of black areas\n    '''\n    \n    if img.ndim == 2:\n        mask = img > tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    \n    elif img.ndim == 3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img > tol\n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        \n        if (check_shape == 0): \n            return img \n        else:\n            img1 = img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2 = img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3 = img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n            img  = np.stack([img1, img2, img3], axis = -1)\n            return img\n","metadata":{"execution":{"iopub.status.busy":"2023-10-30T05:16:43.929574Z","iopub.execute_input":"2023-10-30T05:16:43.929959Z","iopub.status.idle":"2023-10-30T05:16:43.940087Z","shell.execute_reply.started":"2023-10-30T05:16:43.929914Z","shell.execute_reply":"2023-10-30T05:16:43.938858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def circle_crop(img, \n                sigmaX = 10):   \n    \n    '''\n    Perform circular crop around image center\n    '''\n        \n    height, width, depth = img.shape\n    \n    largest_side = np.max((height, width))\n    img = cv2.resize(img, (largest_side, largest_side))\n\n    height, width, depth = img.shape\n    \n    x = int(width / 2)\n    y = int(height / 2)\n    r = np.amin((x,y))\n    \n    circle_img = np.zeros((height, width), np.uint8)\n    cv2.circle(circle_img, (x,y), int(r), 1, thickness = -1)\n    \n    img = cv2.bitwise_and(img, img, mask = circle_img)\n    return img \n","metadata":{"execution":{"iopub.status.busy":"2023-10-30T05:16:46.194895Z","iopub.execute_input":"2023-10-30T05:16:46.195859Z","iopub.status.idle":"2023-10-30T05:16:46.203400Z","shell.execute_reply.started":"2023-10-30T05:16:46.195826Z","shell.execute_reply":"2023-10-30T05:16:46.202272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def random_crop(img, \n                size = (0.9, 1)):\n    \n    '''\n    Random crop\n    '''\n\n    height, width, depth = img.shape\n    \n    cut = 1 - random.uniform(size[0], size[1])\n    \n    i = random.randint(0, int(cut * height))\n    j = random.randint(0, int(cut * width))\n    h = i + int((1 - cut) * height)\n    w = j + int((1 - cut) * width)\n\n    img = img[i:h, j:w, :]    \n    \n    return img","metadata":{"execution":{"iopub.status.busy":"2023-10-30T05:16:52.739319Z","iopub.execute_input":"2023-10-30T05:16:52.739692Z","iopub.status.idle":"2023-10-30T05:16:52.746849Z","shell.execute_reply.started":"2023-10-30T05:16:52.739664Z","shell.execute_reply":"2023-10-30T05:16:52.745742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EyeData(Dataset):\n    \n    # initialize\n    def __init__(self, data, directory, transform = None, do_random_crop = True, itype = '.jpeg'):\n        self.data      = data\n        self.directory = directory\n        self.transform = transform\n        self.do_random_crop = do_random_crop\n        self.itype = itype\n    # length\n    def __len__(self):\n        return len(self.data)\n    \n    # get items    \n    def __getitem__(self, idx):\n        img_name = os.path.join(self.directory, self.data.loc[idx, 'image'] + self.itype)\n        image    = prepare_image(img_name, do_random_crop = self.do_random_crop)\n        image    = self.transform(image)\n        label    = torch.tensor(self.data.loc[idx, 'level'])\n        return {'image': image, 'label': label}","metadata":{"execution":{"iopub.status.busy":"2023-10-30T05:16:56.701058Z","iopub.execute_input":"2023-10-30T05:16:56.701445Z","iopub.status.idle":"2023-10-30T05:16:56.708778Z","shell.execute_reply.started":"2023-10-30T05:16:56.701416Z","shell.execute_reply":"2023-10-30T05:16:56.707873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Data(Dataset):\n    \n    # initialize\n    def __init__(self, data, directory, transform = None, do_random_crop = True, itype = '.jpeg'):\n        self.data      = data\n        self.directory = directory\n        self.transform = transform\n        self.do_random_crop = do_random_crop\n        self.itype = itype\n    # length\n    def __len__(self):\n        return len(self.data)\n    \n    # get items    \n    def __getitem__(self, idx):\n        img_name = os.path.join(self.directory, self.data.loc[idx, 'image'] + self.itype)\n        image = cv2.imread(img_name)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = crop_black(image, tol = 7)\n        image = cv2.resize(image, (int(image_size), int(image_size)))\n        image = circle_crop(image, sigmaX = 10)\n        image = torch.tensor(image)\n        image = image.permute(2, 1, 0)\n        image    = self.transform(image)\n        label    = torch.tensor(self.data.loc[idx, 'level'])\n        return {'image': image, 'label': label}","metadata":{"execution":{"iopub.status.busy":"2023-10-30T05:16:58.899616Z","iopub.execute_input":"2023-10-30T05:16:58.900012Z","iopub.status.idle":"2023-10-30T05:16:58.910661Z","shell.execute_reply.started":"2023-10-30T05:16:58.899982Z","shell.execute_reply":"2023-10-30T05:16:58.909741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def init_model(train= True, \n               trn_layers = 2,\n               model_name = 'enet_b7'):\n    \n    '''\n    Initialize the model\n    '''\n    \n    ### training mode\n    if train == True:\n        \n        # load pre-trained model\n        model = EfficientNet.from_pretrained('efficientnet-b7', num_classes = 5)\n        model.load_state_dict(torch.load('/kaggle/working/models/model_enet_b7.bin'))   \n        \n        # freeze first layers\n        for child in list(model.children())[:-trn_layers]:\n            for param in child.parameters():\n                param.requires_grad = False\n        \n        \n    #inference mode\n    if train == False:\n        \n        # load pre-trained model\n        model = EfficientNet.from_pretrained('efficientnet-b7', num_classes = 5)\n        model.load_state_dict(torch.load('../input/diabetic-retinopathy-pre-training/models/model_{}.bin'.format(model_name, 1)))   \n\n        # freeze all layers\n        for param in model.parameters():\n            param.requires_grad = False\n            \n            \n    ### return model\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-10-30T05:16:58.912468Z","iopub.execute_input":"2023-10-30T05:16:58.912837Z","iopub.status.idle":"2023-10-30T05:16:58.926855Z","shell.execute_reply.started":"2023-10-30T05:16:58.912810Z","shell.execute_reply":"2023-10-30T05:16:58.925800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#GPU CHECK\ntrain_on_gpu = torch.cuda.is_available()\nif not train_on_gpu:\n    print('CUDA is not available. Training on CPU...')\n    device = torch.device('cpu')\nelse:\n    print('CUDA is available. Training on GPU...')\n    device = torch.device('cuda:0')","metadata":{"execution":{"iopub.status.busy":"2023-10-30T05:17:00.939131Z","iopub.execute_input":"2023-10-30T05:17:00.939525Z","iopub.status.idle":"2023-10-30T05:17:00.968784Z","shell.execute_reply.started":"2023-10-30T05:17:00.939484Z","shell.execute_reply":"2023-10-30T05:17:00.967575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#RANDOMNESS\n'''\n\nseed = 23\nseed_everything(seed)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-10-30T05:17:00.970859Z","iopub.execute_input":"2023-10-30T05:17:00.971234Z","iopub.status.idle":"2023-10-30T05:17:00.989830Z","shell.execute_reply.started":"2023-10-30T05:17:00.971198Z","shell.execute_reply":"2023-10-30T05:17:00.988769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/diabetic-retinopathy-resized/trainLabels.csv')","metadata":{"execution":{"iopub.status.busy":"2023-10-30T05:20:25.309763Z","iopub.execute_input":"2023-10-30T05:20:25.310622Z","iopub.status.idle":"2023-10-30T05:20:25.339947Z","shell.execute_reply.started":"2023-10-30T05:20:25.310581Z","shell.execute_reply":"2023-10-30T05:20:25.338892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import data\ntrain = pd.read_csv('../input/diabetic-retinopathy-resized/trainLabels.csv')\ntrain.columns = ['image', 'level']\n#test = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\ntrain = df[:28000]\ntest  = df[28000:]\n\n# check shape\nprint(train.shape, test.shape)\nprint('-' * 15)\nprint(train['level'].value_counts())\nprint('-' * 15)\nprint(test['level'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2023-10-30T05:25:28.154025Z","iopub.execute_input":"2023-10-30T05:25:28.154967Z","iopub.status.idle":"2023-10-30T05:25:28.188605Z","shell.execute_reply.started":"2023-10-30T05:25:28.154930Z","shell.execute_reply":"2023-10-30T05:25:28.187540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CLASS DISTRIBUTION\n\n# plot\nfig = plt.figure(figsize = (15, 5))\nplt.hist(train['level'])\nplt.title('Class Distribution')\nplt.ylabel('Number of examples')\nplt.xlabel('Diagnosis')","metadata":{"execution":{"iopub.status.busy":"2023-10-30T05:25:33.048587Z","iopub.execute_input":"2023-10-30T05:25:33.049003Z","iopub.status.idle":"2023-10-30T05:25:33.509664Z","shell.execute_reply.started":"2023-10-30T05:25:33.048973Z","shell.execute_reply":"2023-10-30T05:25:33.508535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transformations\nsample_trans = transforms.Compose([transforms.ToPILImage(),\n                                   transforms.ToTensor(),\n                                  ])\nsample = Data(data       = train.iloc[0:10], \n                      directory  = '../input/diabetic-retinopathy-resized/resized_train/resized_train',\n                      transform  = sample_trans,\n                      itype ='.jpeg')\n\n# data loader\nsample_loader = torch.utils.data.DataLoader(dataset     = sample, \n                                            batch_size  = 10, \n                                            shuffle     = False, \n                                            num_workers = 4)\n\n# display images\nfor batch_i, data in enumerate(sample_loader):\n\n    # extract data\n    inputs = data['image']\n    labels = data['label'].view(-1, 1)\n    \n    # create plot\n    fig = plt.figure(figsize = (15, 7))\n    for i in range(len(labels)):\n        ax = fig.add_subplot(2, int(len(labels)/2), i + 1, xticks = [], yticks = [])     \n        plt.imshow(inputs[i].numpy().transpose(1, 2, 0))\n        ax.set_title(labels.numpy()[i])\n\n    break","metadata":{"execution":{"iopub.status.busy":"2023-10-30T05:25:41.573755Z","iopub.execute_input":"2023-10-30T05:25:41.574563Z","iopub.status.idle":"2023-10-30T05:25:43.501780Z","shell.execute_reply.started":"2023-10-30T05:25:41.574526Z","shell.execute_reply":"2023-10-30T05:25:43.500610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# IMAGE SIZES\n\n# placeholder\nimage_stats = []\n\n# import loop\nfor index, observation in tqdm(train.iterrows(), total = len(train)):\n    \n    # import image\n    img = cv2.imread('../input/diabetic-retinopathy-resized/resized_train/resized_train/{}.jpeg'.format(observation['image']))\n\n    # compute stats\n    height, width, channels = img.shape\n    ratio = width / height\n    \n    # save\n    image_stats.append(np.array((observation['level'], height, width, channels, ratio)))\n\n# construct DF\nimage_stats = pd.DataFrame(image_stats)\nimage_stats.columns = ['level', 'height', 'width', 'channels', 'ratio']","metadata":{"execution":{"iopub.status.busy":"2023-10-30T05:26:23.694653Z","iopub.execute_input":"2023-10-30T05:26:23.695698Z","iopub.status.idle":"2023-10-30T05:33:47.569567Z","shell.execute_reply.started":"2023-10-30T05:26:23.695657Z","shell.execute_reply":"2023-10-30T05:33:47.568421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# IMAGE SIZE DISTRIBUTION\n\nfig = plt.figure(figsize = (15, 5))\n\n# width\nplt.subplot(1, 3, 1)\nplt.hist(image_stats['width'])\nplt.title('(a) Image Width')\nplt.ylabel('Number of examples')\nplt.xlabel('Width')\n\n# height\nplt.subplot(1, 3, 2)\nplt.hist(image_stats['height'])\nplt.title('(b) Image Height')\nplt.ylabel('Number of examples')\nplt.xlabel('Height')\n\n# ratio\nplt.subplot(1, 3, 3)\nplt.hist(image_stats['ratio'])\nplt.title('(c) Aspect Ratio')\nplt.ylabel('Number of examples')\nplt.xlabel('Ratio')","metadata":{"execution":{"iopub.status.busy":"2023-10-27T13:23:51.445665Z","iopub.execute_input":"2023-10-27T13:23:51.445946Z","iopub.status.idle":"2023-10-27T13:23:52.127682Z","shell.execute_reply.started":"2023-10-27T13:23:51.445920Z","shell.execute_reply":"2023-10-27T13:23:52.126922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#TRANSFORMATIONS\n\n# parameters\nbatch_size = 16\nimage_size = 256\n\n# train transformations\ntrain_trans = transforms.Compose([transforms.ToPILImage(),\n                                  transforms.RandomRotation((-360, 360)),\n                                  transforms.RandomHorizontalFlip(),\n                                  transforms.RandomVerticalFlip(),\n                                  transforms.ToTensor()\n                                 ])\n\n# validation transformations\nvalid_trans = transforms.Compose([transforms.ToPILImage(),\n                                  transforms.ToTensor(),\n                                 ])\n\n# test transformations\ntest_trans = valid_trans","metadata":{"execution":{"iopub.status.busy":"2023-10-27T13:23:52.128776Z","iopub.execute_input":"2023-10-27T13:23:52.129067Z","iopub.status.idle":"2023-10-27T13:23:52.137162Z","shell.execute_reply.started":"2023-10-27T13:23:52.129042Z","shell.execute_reply":"2023-10-27T13:23:52.136388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#EXAMINE FIRST BATCH (TRAIN)\n\n# get dataset\nsample = EyeData(data       = train.iloc[0:10], \n                      directory  = '../input/diabetic-retinopathy-resized/resized_train/resized_train',\n                      transform  = train_trans,\n                      itype ='.jpeg')\n\n# data loader\nsample_loader = torch.utils.data.DataLoader(dataset     = sample, \n                                            batch_size  = batch_size, \n                                            shuffle     = True, \n                                            num_workers = 4)\n\n# display images\nfor batch_i, data in enumerate(sample_loader):\n\n    # extract data\n    inputs = data['image']\n    labels = data['label'].view(-1, 1)\n    \n    # create plot\n    fig = plt.figure(figsize = (20,10))\n    for i in range(len(labels)):\n        ax = fig.add_subplot(2, int(len(labels)/2), i + 1, xticks = [], yticks = [])     \n        plt.imshow(inputs[i].numpy().transpose(1, 2, 0))\n        ax.set_title(labels.numpy()[i])\n\n    break","metadata":{"execution":{"iopub.status.busy":"2023-10-27T13:23:52.138401Z","iopub.execute_input":"2023-10-27T13:23:52.138711Z","iopub.status.idle":"2023-10-27T13:23:53.857253Z","shell.execute_reply.started":"2023-10-27T13:23:52.138683Z","shell.execute_reply":"2023-10-27T13:23:53.856225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#EXAMINE FIRST BATCH (TEST)\ntest = test.reset_index()\n# get dataset\nsample = EyeData(data       = test.iloc[0:10], \n                      directory  = '../input/diabetic-retinopathy-resized/resized_train/resized_train',\n                      transform  = test_trans,\n                      itype ='.jpeg',\n                      do_random_crop = False)\n\n# data loader\nsample_loader = torch.utils.data.DataLoader(dataset     = sample, \n                                            batch_size  = batch_size, \n                                            shuffle     = False, \n                                            num_workers = 4)\n\n# display images\nfor batch_i, data in enumerate(sample_loader):\n\n    # extract data\n    inputs = data['image']\n \n    # create plot\n    fig = plt.figure(figsize = (20,10))\n    for i in range(10):\n        ax = fig.add_subplot(2, int(10/2), i + 1, xticks = [], yticks = [])     \n        plt.imshow(inputs[i].numpy().transpose(1, 2, 0))\n\n    break","metadata":{"execution":{"iopub.status.busy":"2023-10-27T13:23:53.858596Z","iopub.execute_input":"2023-10-27T13:23:53.858898Z","iopub.status.idle":"2023-10-27T13:23:55.399761Z","shell.execute_reply.started":"2023-10-27T13:23:53.858870Z","shell.execute_reply":"2023-10-27T13:23:55.398381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#MODEL ARCHITECTURE\n\n# model name\nmodel_name = 'enet_b7'\n\n# initialization function\ndef init_pre_model(train = True):\n    \n    '''\n    Initialize the model\n    '''\n    \n    ### training mode\n    if train == True:\n        \n        # load pre-trained model\n        model = EfficientNet.from_pretrained('efficientnet-b7', num_classes = 5)\n        \n    ### inference mode\n    if train == False:\n        \n        # load pre-trained model\n        model = EfficientNet.from_name('efficientnet-b7')\n        model._fc = nn.Linear(model._fc.in_features, 5)\n\n        # freeze  layers\n        for param in model.parameters():\n            param.requires_grad = False\n            \n    ### return model\n    return model\n\n\n# check architecture\nmodel = init_pre_model()\nprint(model)","metadata":{"execution":{"iopub.status.busy":"2023-10-27T13:23:55.401168Z","iopub.execute_input":"2023-10-27T13:23:55.401491Z","iopub.status.idle":"2023-10-27T13:23:57.563661Z","shell.execute_reply.started":"2023-10-27T13:23:55.401457Z","shell.execute_reply":"2023-10-27T13:23:57.562396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#VALIDATION SETTINGS\n\n# placeholders\noof_preds = np.zeros((len(test), 5))\n\n# timer\ncv_start = time.time()","metadata":{"execution":{"iopub.status.busy":"2023-10-27T13:23:57.568069Z","iopub.execute_input":"2023-10-27T13:23:57.568643Z","iopub.status.idle":"2023-10-27T13:23:57.575496Z","shell.execute_reply.started":"2023-10-27T13:23:57.568597Z","shell.execute_reply":"2023-10-27T13:23:57.573976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#PARAMETERS\n\n# loss function\ncriterion = nn.CrossEntropyLoss()\n\n# epochs\nmax_epochs = 35###################################################\nearly_stop = 35\n\n# learning rates\neta = 1e-3\n\n# scheduler\nstep  = 5\ngamma = 0.5","metadata":{"execution":{"iopub.status.busy":"2023-10-27T13:23:57.577049Z","iopub.execute_input":"2023-10-27T13:23:57.577400Z","iopub.status.idle":"2023-10-27T13:23:57.606050Z","shell.execute_reply.started":"2023-10-27T13:23:57.577364Z","shell.execute_reply":"2023-10-27T13:23:57.604187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#DATA PREPARATION\n\n# load splits\ndata_train = train\ndata_valid = test\n\n# create datasets\ntrain_dataset = EyeData(data      = data_train, \n                             directory = '../input/diabetic-retinopathy-resized/resized_train/resized_train',\n                             transform = train_trans,\n                             itype ='.jpeg')\nvalid_dataset = EyeData(data       = data_valid, \n                            directory  = '../input/diabetic-retinopathy-resized/resized_train/resized_train',\n                            transform  = valid_trans,\n                            itype ='.jpeg')\n\n# create data loaders\ntrain_loader = torch.utils.data.DataLoader(train_dataset, \n                                           batch_size  = batch_size, \n                                           shuffle     = True, \n                                           num_workers = 4)\nvalid_loader = torch.utils.data.DataLoader(valid_dataset, \n                                           batch_size  = batch_size, \n                                           shuffle     = False, \n                                           num_workers = 4)","metadata":{"execution":{"iopub.status.busy":"2023-10-27T13:23:57.607302Z","iopub.execute_input":"2023-10-27T13:23:57.607601Z","iopub.status.idle":"2023-10-27T13:23:57.617739Z","shell.execute_reply.started":"2023-10-27T13:23:57.607566Z","shell.execute_reply":"2023-10-27T13:23:57.616852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#MODELING EPOCHS\n\n# placeholders\nval_kappas = []\nval_losses = []\ntrn_losses = []\nbad_epochs = 0\n\n# initialize and send to GPU\nmodel = init_pre_model()\nmodel = model.to(device)\n\n# optimizer\noptimizer = optim.Adam(model.parameters(), lr = eta)\nscheduler = lr_scheduler.StepLR(optimizer, step_size = step, gamma = gamma)\n\n# training and validation loop\nfor epoch in range(max_epochs):\n    ### PREPARATION\n\n    # timer\n    epoch_start = time.time()\n\n    # reset losses\n    trn_loss = 0.0\n    val_loss = 0.0\n\n    # placeholders\n    fold_preds = np.zeros((len(data_valid), 5))\n\n\n    #TRAINING\n\n    # switch regime\n    model.train()\n\n    # loop through batches\n    for batch_i, data in enumerate(train_loader):\n\n        # extract inputs and labels\n        inputs = data['image']\n        labels = data['label'].view(-1)\n        inputs = inputs.to(device, dtype = torch.float)\n        labels = labels.to(device, dtype = torch.long)\n        optimizer.zero_grad()\n\n        # forward and backward pass\n        with torch.set_grad_enabled(True):\n            preds = model(inputs)\n            loss  = criterion(preds, labels)\n            loss.backward()\n            optimizer.step()\n\n        # compute loss\n        trn_loss += loss.item() * inputs.size(0)\n        \n        \n    #INFERENCE\n\n    # switch regime\n    model.eval()\n    \n    # loop through batches\n    for batch_i, data in enumerate(valid_loader):\n        \n        # extract inputs and labels\n        inputs = data['image']\n        labels = data['label'].view(-1)\n        inputs = inputs.to(device, dtype = torch.float)\n        labels = labels.to(device, dtype = torch.long)\n\n        # compute predictions\n        with torch.set_grad_enabled(False):\n            preds = model(inputs).detach()\n            fold_preds[batch_i * batch_size:(batch_i + 1) * batch_size, :] = preds.cpu().numpy()\n\n        # compute loss\n        loss      = criterion(preds, labels)\n        val_loss += loss.item() * inputs.size(0)\n        \n    # save predictions\n    oof_preds = fold_preds\n\n    # scheduler step\n    scheduler.step()\n\n\n    #EVALUATION\n\n    # evaluate performance\n    fold_preds_round = fold_preds.argmax(axis = 1)\n    val_kappa = metrics.cohen_kappa_score(data_valid['level'], fold_preds_round.astype('int'), weights = 'quadratic')\n\n    # save perfoirmance values\n    val_kappas.append(val_kappa)\n    val_losses.append(val_loss / len(data_valid))\n    trn_losses.append(trn_loss / len(data_train))\n\n\n    #EARLY STOPPING\n\n    # display info\n    print('- epoch {}/{} | lr = {} | trn_loss = {:.4f} | val_loss = {:.4f} | val_kappa = {:.4f} | {:.2f} min'.format(\n        epoch + 1, max_epochs, scheduler.get_lr()[len(scheduler.get_lr()) - 1],\n        trn_loss / len(data_train), val_loss / len(data_valid), val_kappa,\n        (time.time() - epoch_start) / 60))\n\n    # check if there is any improvement\n    if epoch > 0:       \n        if val_kappas[epoch] < val_kappas[epoch - bad_epochs - 1]:\n            bad_epochs += 1\n        else:\n            bad_epochs = 0\n\n    # save model weights if improvement\n    if bad_epochs == 0:\n        oof_preds_best = oof_preds.copy()\n        torch.save(model.state_dict(), 'models/model_{}.bin'.format(model_name))\n\n    # break if early stop\n    if bad_epochs == early_stop:\n        print('Early stopping. Best results: loss = {:.4f}, kappa = {:.4f} (epoch {})'.format(\n            np.min(val_losses), val_kappas[np.argmin(val_losses)], np.argmin(val_losses) + 1))\n        print('')\n        break\n\n    # break if max epochs\n    if epoch == (max_epochs - 1):\n        print('Did not met early stopping. Best results: loss = {:.4f}, kappa = {:.4f} (epoch {})'.format(\n            np.min(val_losses), val_kappas[np.argmin(val_losses)], np.argmin(val_losses) + 1))\n        print('')\n        break\n\n\n# load best predictions\noof_preds = oof_preds_best\n\n# print performance\nprint('')\nprint('Finished in {:.2f} minutes'.format((time.time() - cv_start) / 60))","metadata":{"execution":{"iopub.status.busy":"2023-10-27T13:23:57.619133Z","iopub.execute_input":"2023-10-27T13:23:57.619471Z","iopub.status.idle":"2023-10-27T13:44:44.729878Z","shell.execute_reply.started":"2023-10-27T13:23:57.619437Z","shell.execute_reply":"2023-10-27T13:44:44.728756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#PLOT LOSS AND KAPPA DYNAMICS\nsns.set()\n# plot size\nfig = plt.figure(figsize = (15, 5))\n\n# plot loss dynamics\nplt.subplot(1, 2, 1)\nplt.plot(trn_losses, 'red',   label = 'Training')\nplt.plot(val_losses, 'green', label = 'Validation')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()\n\n# plot kappa dynamics\nplt.subplot(1, 2, 2)\nplt.plot(val_kappas, 'blue', label = 'Kappa')\nplt.xlabel('Epoch')\nplt.ylabel('Kappa')\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2023-10-27T13:44:44.731568Z","iopub.execute_input":"2023-10-27T13:44:44.731999Z","iopub.status.idle":"2023-10-27T13:44:45.378388Z","shell.execute_reply.started":"2023-10-27T13:44:44.731938Z","shell.execute_reply":"2023-10-27T13:44:45.377442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#RECHECK PERFORMANCE\n\n# rounding\noof_preds_round = oof_preds.argmax(axis = 1)\ncoef = [0.5, 1.5, 2.5, 3.5]\nfor i, pred in enumerate(oof_preds_round):\n    if pred < coef[0]:\n        oof_preds_round[i] = 0\n    elif pred >= coef[0] and pred < coef[1]:\n        oof_preds_round[i] = 1\n    elif pred >= coef[1] and pred < coef[2]:\n        oof_preds_round[i] = 2\n    elif pred >= coef[2] and pred < coef[3]:\n        oof_preds_round[i] = 3\n    else:\n        oof_preds_round[i] = 4\n\n# compute kappa\noof_loss  = criterion(torch.tensor(oof_preds), torch.tensor(test['level']).view(-1).type(torch.long))\noof_kappa = metrics.cohen_kappa_score(test['level'], oof_preds_round.astype('int'), weights = 'quadratic')\nprint('OOF loss  = {:.4f}'.format(oof_loss))\nprint('OOF kappa = {:.4f}'.format(oof_kappa))","metadata":{"execution":{"iopub.status.busy":"2023-10-27T13:44:45.379752Z","iopub.execute_input":"2023-10-27T13:44:45.380116Z","iopub.status.idle":"2023-10-27T13:44:45.463364Z","shell.execute_reply.started":"2023-10-27T13:44:45.380079Z","shell.execute_reply":"2023-10-27T13:44:45.462472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#CONFUSION MATRIX\n\n# construct confusion matrix\ncm = confusion_matrix(test['level'], oof_preds_round)\ncm = cm.astype('float') / cm.sum(axis = 1)[:, np.newaxis]\nannot = np.around(cm, 2)\n\n# plot matrix\nfig, ax = plt.subplots(figsize = (10, 10))\nsns.heatmap(cm, cmap = 'Blues', annot = annot, lw = 0.5)\nax.set_xlabel('Prediction')\nax.set_ylabel('Ground Truth')\nax.set_aspect('equal')","metadata":{"execution":{"iopub.status.busy":"2023-10-27T13:44:45.468358Z","iopub.execute_input":"2023-10-27T13:44:45.468647Z","iopub.status.idle":"2023-10-27T13:44:45.955787Z","shell.execute_reply.started":"2023-10-27T13:44:45.468622Z","shell.execute_reply":"2023-10-27T13:44:45.954887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report\n\n#Classification Report Test\nprint('\\n Classification Report in Test: \\n',classification_report(test['level'], oof_preds_round))","metadata":{"execution":{"iopub.status.busy":"2023-10-27T13:44:45.956869Z","iopub.execute_input":"2023-10-27T13:44:45.957145Z","iopub.status.idle":"2023-10-27T13:44:45.978033Z","shell.execute_reply.started":"2023-10-27T13:44:45.957121Z","shell.execute_reply":"2023-10-27T13:44:45.977090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}