{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport json\nimport csv\nimport random\nimport pickle\nimport cv2\nimport numpy as np\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\nimport torch.optim as optim\nfrom torchvision import models\nimport torchvision.transforms as transforms\n\nfrom PIL import Image\nfrom torch.utils.data import Dataset, DataLoader\nfrom scipy.ndimage.measurements import label\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import roc_auc_score, roc_curve","metadata":{"papermill":{"duration":2.784273,"end_time":"2021-04-26T09:25:03.011883","exception":false,"start_time":"2021-04-26T09:25:00.22761","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset class\n(but first, the augmentation function)","metadata":{"papermill":{"duration":0.014426,"end_time":"2021-04-26T09:25:03.046449","exception":false,"start_time":"2021-04-26T09:25:03.032023","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def twisted_eyes(img, gt=None):\n    \"\"\"    \n    Perform augmentation. If rotation is applied, rotate gts and images equally.\n    \"\"\"\n    color_transform = transforms.Compose([\n        # transforms.Normalize(mean_db, std_db),\n        transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=(-0.08,0.08)),\n        transforms.RandomGrayscale()\n    ])\n    \n    rotation = transforms.RandomRotation(degrees=30)\n    \n    # First allucinate the eye\n    colored_eye = color_transform(img)\n    # Join picture and gt and rotate them the same way\n    if gt is not None:\n        joined_data = torch.cat([colored_eye, gt], dim=0)\n        augmented_data = rotation(joined_data)\n        # Unpack them again\n        final_img = augmented_data[:3]\n        final_gt = augmented_data[3:]\n        return final_img, final_gt\n    else:\n        return rotation(colored_eye)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RefugeDataset(Dataset):\n\n    def __init__(self, root_dir, split='train', output_size=(256,256), normalize=False, augment=True):\n        # Define attributes\n        self.output_size = output_size\n        self.root_dir = root_dir\n        self.split = split\n        self.normalize = normalize\n        self.augment = augment\n        \n        # Std and mean were computed with\n        # torch.std_mean(torch.stack(train_set.images), dim=(0,2,3))\n        self.normalizator = transforms.Normalize([0.2819, 0.1679, 0.0838], [0.1891, 0.1100, 0.0615])\n        \n        # Load data index\n        with open(os.path.join(self.root_dir, self.split, 'index.json')) as f:\n            self.index = json.load(f)\n            \n        self.images = []\n        for k in range(len(self.index)):\n            print('Loading {} image {}/{}...'.format(split, k, len(self.index)), end='\\r')\n            img_name = os.path.join(self.root_dir, self.split, 'images', self.index[str(k)]['ImgName'])\n            # Load images, resize them to 256x256, append them to a list\n            img = np.array(Image.open(img_name).convert('RGB'))\n            img = transforms.functional.to_tensor(img)\n            img = transforms.functional.resize(img, self.output_size, interpolation=Image.BILINEAR)\n            self.images.append(img)\n            \n        # Load ground truth for 'train' and 'val' sets\n        if split != 'test':\n            self.segs = []\n            for k in range(len(self.index)):\n                print('Loading {} segmentation {}/{}...'.format(split, k, len(self.index)), end='\\r')\n                seg_name = os.path.join(self.root_dir, self.split, 'gts', self.index[str(k)]['ImgName'].split('.')[0]+'.bmp')\n                seg = np.array(Image.open(seg_name)).copy()\n                seg = 255. - seg\n                od = (seg>=127.).astype(np.float32) # This will have 1 wherever the optic disk is\n                oc = (seg>=250.).astype(np.float32) # This will have 1 wherever the optic cone is\n                od = torch.from_numpy(od[None,:,:])\n                oc = torch.from_numpy(oc[None,:,:])\n                od = transforms.functional.resize(od, self.output_size, interpolation=Image.NEAREST)\n                oc = transforms.functional.resize(oc, self.output_size, interpolation=Image.NEAREST)\n                seg = torch.cat([od, oc], dim=0)\n                self.segs.append(seg) # Each element of the \"segs\" list is a mask indicating optic disk and cone\n                \n        print('Succesfully loaded {} dataset.'.format(split) + ' '*50)\n            \n            \n    def __len__(self):\n        return len(self.index)\n\n    def __getitem__(self, idx):\n        # Image\n        img = self.images[idx]\n    \n        # Return only images for 'test' set\n        if self.split == 'test':\n            if self.normalize:\n                img = self.normalizator(img)\n            return img\n        \n        # Else, images and ground truth\n        else:\n            # Label\n            lab = torch.tensor(self.index[str(idx)]['Label'], dtype=torch.float32)\n\n            # Segmentation masks\n            seg = self.segs[idx]\n\n            # Fovea localization\n            f_x = self.index[str(idx)]['Fovea_X']\n            f_y = self.index[str(idx)]['Fovea_Y']\n            fov = torch.FloatTensor([f_x, f_y])\n            \n            # Perform augmentation if data is training data\n            if self.split == 'train' and self.augment:\n                img, seg = twisted_eyes(img, seg)\n            \n            if self.normalize:\n                img = self.normalizator(img)\n        \n            return img, lab, seg, fov, self.index[str(idx)]['ImgName']\n        # The returned values are\n        # - image\n        # - label 1/0 (glaucoma or not)\n        # - seg = [od, oc] masks (one hot)\n        # - fov = [fx, fy] forvea position\n        # - image_name, like g0001.jpg. Very useful\n\nclass DummyDataset(Dataset):\n    def __init__(self, root_dir, split='train', output_size=(256,256), normalize=False, augment=True):\n        # Define attributes\n        self.output_size = output_size\n        self.root_dir = root_dir\n        self.split = split\n        self.normalize = normalize\n        self.augment = augment\n        \n        # Std and mean were computed with\n        # torch.std_mean(torch.stack(train_set.images), dim=(0,2,3))\n        self.normalizator = transforms.Normalize([0.2819, 0.1679, 0.0838], [0.1891, 0.1100, 0.0615])\n        \n        # Load data index\n        with open(os.path.join(self.root_dir, self.split, 'index.json')) as f:\n            self.index = json.load(f)\n            \n        self.images = []\n        for k in range(len(self.index)):\n            print('Loading {} image {}/{}...'.format(split, k, len(self.index)), end='\\r')\n            img_name = os.path.join(self.root_dir, self.split, 'images', self.index[str(k)]['ImgName'])\n            # Load images, resize them to 256x256, append them to a list\n            img = np.array(Image.open(img_name).convert('RGB'))\n            img = transforms.functional.to_tensor(img)\n            img = transforms.functional.resize(img, self.output_size, interpolation=Image.BILINEAR)\n            self.images.append(img)\n                \n        print('Succesfully loaded {} dataset.'.format(split) + ' '*50)\n            \n            \n    def __len__(self):\n        return len(self.index)\n\n    def __getitem__(self, idx):\n        # Image\n        img = self.images[idx]\n    \n        # Return only images for 'test' set\n        if self.split == 'test':\n            if self.normalize:\n                img = self.normalizator(img)\n            return img\n        \n        # Else, images and ground truth\n        else:\n            # Label\n            lab = torch.tensor(self.index[str(idx)]['Label'], dtype=torch.float32)\n\n            \n            # Perform augmentation if data is training data\n            if self.split == 'train' and self.augment:\n                img = twisted_eyes(img)\n            \n            if self.normalize:\n                img = self.normalizator(img)\n        \n            return img, lab\n        # The returned val","metadata":{"papermill":{"duration":0.034267,"end_time":"2021-04-26T09:25:03.094999","exception":false,"start_time":"2021-04-26T09:25:03.060732","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Metrics","metadata":{"papermill":{"duration":0.012912,"end_time":"2021-04-26T09:25:03.121542","exception":false,"start_time":"2021-04-26T09:25:03.10863","status":"completed"},"tags":[]}},{"cell_type":"code","source":"EPS = 1e-7\n\ndef compute_dice_coef(input, target):\n    '''\n    Compute dice score metric.\n    '''\n    batch_size = input.shape[0]\n    return sum([dice_coef_sample(input[k,:,:], target[k,:,:]) for k in range(batch_size)])/batch_size\n\ndef dice_coef_sample(input, target):\n    iflat = input.contiguous().view(-1)\n    tflat = target.contiguous().view(-1)\n    intersection = (iflat * tflat).sum()\n    return (2. * intersection) / (iflat.sum() + tflat.sum())\n\n\ndef vertical_diameter(binary_segmentation):\n    '''\n    Get the vertical diameter from a binary segmentation.\n    The vertical diameter is defined as the \"fattest\" area of the binary_segmentation parameter.\n    '''\n\n    # get the sum of the pixels in the vertical axis\n    vertical_axis_diameter = np.sum(binary_segmentation, axis=1)\n\n    # pick the maximum value\n    diameter = np.max(vertical_axis_diameter, axis=1)\n\n    # return it\n    return diameter\n\n\n\ndef vertical_cup_to_disc_ratio(od, oc):\n    '''\n    Compute the vertical cup-to-disc ratio from a given labelling map.\n    '''\n    # compute the cup diameter\n    cup_diameter = vertical_diameter(oc)\n    # compute the disc diameter\n    disc_diameter = vertical_diameter(od)\n\n    return cup_diameter / (disc_diameter + EPS)\n\ndef compute_vCDR_error(pred_od, pred_oc, gt_od, gt_oc):\n    '''\n    Compute vCDR prediction error, along with predicted vCDR and ground truth vCDR.\n    '''\n    pred_vCDR = vertical_cup_to_disc_ratio(pred_od, pred_oc)\n    gt_vCDR = vertical_cup_to_disc_ratio(gt_od, gt_oc)\n    vCDR_err = np.mean(np.abs(gt_vCDR - pred_vCDR))\n    return vCDR_err, pred_vCDR, gt_vCDR\n\n\ndef classif_eval(classif_preds, classif_gts):\n    '''\n    Compute AUC classification score.\n    '''\n    auc = roc_auc_score(classif_gts, classif_preds)\n    return auc\n\n\ndef fov_error(pred_fov, gt_fov):\n    '''\n    Fovea localization error metric (mean root squared error).\n    '''\n    err = np.sqrt(np.sum((gt_fov-pred_fov)**2, axis=1)).mean()\n    return err\n\n\nclass FocalLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n\n    def forward(self, inputs, targets):\n        bce_loss = F.binary_cross_entropy(inputs.squeeze(),  targets.float())\n        loss = self.alpha * (1 - torch.exp(-bce_loss)) ** self.gamma * bce_loss\n        return loss","metadata":{"papermill":{"duration":0.026243,"end_time":"2021-04-26T09:25:03.160687","exception":false,"start_time":"2021-04-26T09:25:03.134444","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Post-processing functions","metadata":{"papermill":{"duration":0.012935,"end_time":"2021-04-26T09:25:03.186948","exception":false,"start_time":"2021-04-26T09:25:03.174013","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def refine_seg(pred):\n    '''\n    Only retain the biggest connected component of a segmentation map.\n    '''\n    np_pred = pred.numpy()\n        \n    largest_ccs = []\n    for i in range(np_pred.shape[0]):\n        labeled, ncomponents = label(np_pred[i,:,:])\n        bincounts = np.bincount(labeled.flat)[1:]\n        if len(bincounts) == 0:\n            largest_cc = labeled == 0\n        else:\n            largest_cc = labeled == np.argmax(bincounts)+1\n        largest_cc = torch.tensor(largest_cc, dtype=torch.float32)\n        largest_ccs.append(largest_cc)\n    largest_ccs = torch.stack(largest_ccs)\n    \n    return largest_ccs","metadata":{"papermill":{"duration":0.023335,"end_time":"2021-04-26T09:25:03.226082","exception":false,"start_time":"2021-04-26T09:25:03.202747","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Network","metadata":{"papermill":{"duration":0.013265,"end_time":"2021-04-26T09:25:03.252404","exception":false,"start_time":"2021-04-26T09:25:03.239139","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class UNet(nn.Module):\n    def __init__(self, n_channels=3, n_classes=2):\n        super(UNet, self).__init__()\n        self.n_channels = n_channels\n        self.n_classes = n_classes\n        self.epoch = 0\n\n        self.inc = DoubleConv(n_channels, 64)\n        self.down1 = Down(64, 128) #[128,128,128]\n        self.down2 = Down(128, 256) #[64,64,256]\n        self.down3 = Down(256, 512) #[32,32,512]\n        factor = 2 \n        self.down4 = Down(512, 1024 // factor) #[16,16,512]\n        self.up1 = Up(1024, 512 // factor) #[32,32,] \n        self.up2 = Up(512, 256 // factor)\n        self.up3 = Up(256, 128 // factor)\n        self.up4 = Up(128, 64)\n        self.output_layer = OutConv(64, n_classes)\n\n    def forward(self, x):\n        # print(f'x: {x.shape}')\n        x1 = self.inc(x)\n        # print(f'x1: {x1.shape}')\n        x2 = self.down1(x1)\n        # print(f'x2: {x2.shape}')\n        x3 = self.down2(x2)\n        # print(f'x3: {x3.shape}')\n        x4 = self.down3(x3)\n        # print(f'x4: {x4.shape}')\n        x5 = self.down4(x4)\n        # print(f'x5: {x5.shape}')\n        out = self.up1(x5, x4)\n        # print(f'up1: {out.shape}')\n        out = self.up2(out, x3)\n        # print(f'up2: {out.shape}')\n        out = self.up3(out, x2)\n        # print(f'up3: {out.shape}')\n        out = self.up4(out, x1)\n        # print(f'up4: {out.shape}')\n        out = self.output_layer(out)\n        # print(f'final output: {out.shape}')\n        out = torch.sigmoid(out)\n        return out\n\n    \nclass DoubleConv(nn.Module):\n    \"\"\"(convolution => [BN] => ReLU) * 2\"\"\"\n\n    def __init__(self, in_channels, out_channels, mid_channels=None):\n        super().__init__()\n        if not mid_channels:\n            mid_channels = out_channels\n        self.double_conv = nn.Sequential(\n            nn.Conv2d(in_channels, mid_channels, kernel_size=3, padding=1),\n            nn.BatchNorm2d(mid_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(mid_channels, out_channels, kernel_size=3, padding=1),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True)\n        )\n\n    def forward(self, x):\n        return self.double_conv(x)\n\n\nclass Down(nn.Module):\n    \"\"\"Downscaling with maxpool then double conv\"\"\"\n\n    def __init__(self, in_channels, out_channels):\n        super().__init__()\n        self.maxpool_conv = nn.Sequential(\n            nn.MaxPool2d(2),\n            DoubleConv(in_channels, out_channels)\n        )\n\n    def forward(self, x):\n        return self.maxpool_conv(x)\n\n\nclass Up(nn.Module):\n    \"\"\"Upscaling then double conv\"\"\"\n\n    def __init__(self, in_channels, out_channels):\n        super().__init__()\n\n        # Use the normal convolutions to reduce the number of channels\n        self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)\n        # UP-CONVOLUTION AS DESCRIBED IN UNET PAPER \n        # (amaarora.github.io/2020/09/13/unet.html)\n        # self.up = nn.ConvTranspose2d(in_channels // 2, in_channels // 2, 2, 2)\n        self.conv = DoubleConv(in_channels, out_channels, in_channels // 2)\n\n\n    def forward(self, x1, x2):\n        # print(f'\\tWithin up: x1 {x1.shape}')\n        x1 = self.up(x1)\n        # input is CHW\n        diffY = x2.size()[2] - x1.size()[2]\n        diffX = x2.size()[3] - x1.size()[3]\n\n        x1 = F.pad(x1, [diffX // 2, diffX - diffX // 2,\n                        diffY // 2, diffY - diffY // 2])\n        # print(f'\\tAfter pad: x1: {x1.shape}')\n        x = torch.cat([x2, x1], dim=1)\n        return self.conv(x)\n\n\nclass OutConv(nn.Module):\n    '''\n    Simple convolution.\n    '''\n    def __init__(self, in_channels, out_channels):\n        super(OutConv, self).__init__()\n        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=1)\n\n    def forward(self, x):\n        return self.conv(x)\n\nclass PanaNet(nn.Module):\n    def __init__(self):\n        super(PanaNet, self).__init__()\n        self.resnet = models.resnet18(pretrained=True)\n        self.resnet.fc = nn.Linear(512, 1)\n        self.act = nn.Sigmoid()\n        self.epoch = 0\n        \n    def forward(self, x):\n        x =  self.resnet(x)\n        return self.act(x)","metadata":{"papermill":{"duration":0.038018,"end_time":"2021-04-26T09:25:03.303603","exception":false,"start_time":"2021-04-26T09:25:03.265585","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import dropbox","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dbx_token = 'secret'\ndbx = dropbox.Dropbox(dbx_token)\nfilename = 'checkpoint_from_batcave.pth'\nmeta = dbx.files_download_to_file(filename, path='/AML/Challenge2/best_AUC_weights_resnet.pth')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Metric Tracker\nKindly provided by Batman","metadata":{}},{"cell_type":"code","source":"import logging\nimport sys\nfrom torch.utils.tensorboard import SummaryWriter\nimport json\nimport os\n\n\nclass MetricTracker():\n    \"\"\"\n    Keeps track of several metrics at the same time like a pro. Logs to tensorboard and also onto a file.\n    Internally keeps a list for several possible parameters.\n    :log_path: path where to put log file\n    :tensorboard_path: path where to put the tensorboard file\n    \"\"\"\n    def __init__(self, log_path, tensorboard_path=None):\n        log_format = '%(asctime)s - %(message)s'\n        datefmt = '%Y-%d-%m %I:%M:%S'\n        # check the logging folder and create it if needed\n        root_folder = os.path.dirname(log_path)\n        if not os.path.isdir(root_folder):\n            os.mkdir(root_folder)\n        handlers = [logging.FileHandler(log_path), logging.StreamHandler(sys.stdout)]\n        logging.basicConfig(level=logging.INFO, format=log_format, datefmt=datefmt, handlers=handlers)\n\n        if tensorboard_path is not None:\n            self.writer = SummaryWriter(tensorboard_path)\n        else:\n            self.writer = SummaryWriter()\n        self.values = {}\n        self.tb_iter = 0\n\n    def genericLog(self, msg):\n        \"\"\"\n        Logs a generic message on the file.\n        :param msg: Message to log\n        \"\"\"\n        logging.info(msg)\n\n    def update(self, name, new_value, tb_iter=None):\n        \"\"\"\n        Appends to the list a certain value of a parameter.\n        :param name: name of the parameter to update\n        :param new_value: value to append to the list of values of that parameter\n        :param tb_iter: tensorboard iteration number. If None, no value is logged to tensorboard\n        \"\"\"\n        # Append to the internally tracked values\n        if name not in self.values:\n            self.values[name] = [new_value]\n        else:\n            self.values[name].append(new_value)\n\n        # Tensorboard update\n        if tb_iter is None:\n            tb_iter = self.tb_iter\n            self.tb_iter += 1\n\n        self.writer.add_scalar(name, new_value, tb_iter)\n\n    def display(self, names=None, timestamp=-1):\n        \"\"\"\n        Display the value of the given metrics at the given timestep.\n        :param names: List of names of metrics. Will print a value form these metrics.\n        :param timestamp: Timestamp of which to print the value. Defaults to -1 (last value added)\n        \"\"\"\n        # By default, log the last of all values\n        if names is None:\n            msg = ' | '.join([f'{name}: {series[timestamp]:6.6}' for name, series in self.values.items()])\n        else:\n            msg = ' | '.join([f'{name}: {self.values[name][timestamp]:6.6}' for name in names])\n        logging.info(msg)\n\n    def save_as_json(self, file_save_path=None):\n        \"\"\"\n        Dump all lists as a JSON file.\n        :param file_save_path:\n        \"\"\"\n        if file_save_path is None:\n            file_save_path = '/content/runs'\n\n        with open(file_save_path, 'w') as fp:\n            json.dump(self.values, fp, indent=2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Settings","metadata":{"papermill":{"duration":0.013662,"end_time":"2021-04-26T09:25:03.330687","exception":false,"start_time":"2021-04-26T09:25:03.317025","status":"completed"},"tags":[]}},{"cell_type":"code","source":"root_dir = '/kaggle/input/eurecom-aml-2021-challenge-2/refuge_data/refuge_data'\nlr = 1e-4\nbatch_size = 8\nnum_workers = 8\ntotal_epoch = 100\nweight_decay = 5e-4","metadata":{"papermill":{"duration":0.019165,"end_time":"2021-04-26T09:25:03.363242","exception":false,"start_time":"2021-04-26T09:25:03.344077","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create datasets and data loaders\nAll image files are loaded in RAM in order to speed up the pipeline. Therefore, each dataset creation should take a few minutes.","metadata":{"papermill":{"duration":0.013181,"end_time":"2021-04-26T09:25:03.389864","exception":false,"start_time":"2021-04-26T09:25:03.376683","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Datasets\ntrain_set = RefugeDataset(root_dir, \n                          split='train', normalize=True)\nval_set = RefugeDataset(root_dir, \n                        split='val', normalize=True)\ntest_set = RefugeDataset(root_dir, \n                         split='test', normalize=True)\n\n# Dataloaders\ntrain_loader = DataLoader(train_set, \n                          batch_size=batch_size, \n                          shuffle=True, \n                          num_workers=num_workers,\n                          pin_memory=True,\n                         )\nval_loader = DataLoader(val_set, \n                        batch_size=batch_size, \n                        shuffle=False, \n                        num_workers=num_workers,\n                        pin_memory=True,\n                        )\ntest_loader = DataLoader(test_set, \n                        batch_size=batch_size, \n                        shuffle=False, \n                        num_workers=num_workers,\n                        pin_memory=True)","metadata":{"papermill":{"duration":299.002965,"end_time":"2021-04-26T09:30:02.405904","exception":false,"start_time":"2021-04-26T09:25:03.402939","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Datasets\ntrain_set = DummyDataset(root_dir, \n                          split='train', normalize=True)\nval_set = DummyDataset(root_dir, \n                        split='val', normalize=True)\ntest_set = DummyDataset(root_dir, \n                         split='test', normalize=True)\n\n# Dataloaders\ntrain_loader = DataLoader(train_set, \n                          batch_size=batch_size, \n                          shuffle=True, \n                          num_workers=num_workers,\n                          pin_memory=True,\n                         )\nval_loader = DataLoader(val_set, \n                        batch_size=batch_size, \n                        shuffle=False, \n                        num_workers=num_workers,\n                        pin_memory=True,\n                        )\ntest_loader = DataLoader(test_set, \n                        batch_size=batch_size, \n                        shuffle=False, \n                        num_workers=num_workers,\n                        pin_memory=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\ndef tensor_show(tensor, title=\"An eyeball, literally.\"):\n    dims = len(tensor.shape)\n    assert dims == 3 or dims == 2, \"Please pass a tensor of shape (C, H, W) or (H, W)\"\n    if dims == 3:\n        # If rgb, permute. else, squeeze.\n        if tensor.shape[0] == 3:\n            to_show = tensor.permute(1, 2, 0)\n            plt.imshow(to_show)\n        else:\n            to_show = tensor.squeeze()\n            plt.imshow(to_show, cmap='gray')\n    else:\n        # If two dimensional, use greyscale\n        to_show = tensor\n        plt.imshow(to_show, cmap='gray')\n    \n    plt.title(title)\n\ntensor_show(val_set[150][0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Device, model, loss and optimizer","metadata":{"papermill":{"duration":0.394978,"end_time":"2021-04-26T09:30:03.082515","exception":false,"start_time":"2021-04-26T09:30:02.687537","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Device\ndevice = torch.device(\"cuda:0\") # change this when running on gpu/cpu\n# Network\nmodel = UNet(n_channels=3, n_classes=2).to(device)\n\n# Uncomment to save from previous checkpoint\n# model.load_state_dict(torch.load('/kaggle/working/best_AUC_weights.pth'))\n\n# Loss\n#seg_loss = torch.nn.BCELoss(reduction='mean')\nseg_loss = FocalLoss(alpha=1, gamma=0.1)\n# Optimizer\noptimizer = optim.Adam(model.parameters(), lr=1e-3)","metadata":{"papermill":{"duration":5.452715,"end_time":"2021-04-26T09:30:09.167869","exception":false,"start_time":"2021-04-26T09:30:03.715154","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Device\ndevice = torch.device(\"cuda:0\") # change this when running on gpu/cpu\n\ndel model\n# Network\nmodel = PanaNet().to(device)\n\n# Uncomment to save from previous checkpoint\nmodel.load_state_dict(torch.load('/kaggle/working/checkpoint_from_batcave.pth'))\n\n# Loss\nloss_fn = torch.nn.BCELoss(reduction='mean')\n# Optimizer\noptimizer = optim.Adam(model.parameters(), lr=1e-4, weight_decay=weight_decay)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Take image with active grad\nI = test_set[27].to(device)\nI.requires_grad = True\n\n# Single forward\nout = model(I.unsqueeze(0))\nout.backward()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tensor_show(I.detach().cpu())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"saliency, _ = torch.max(I.grad.data.abs(), dim=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tensor_show(saliency.cpu())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sac = I.grad.data.abs()\nsac = (sac - sac.min())/(sac.max() - sac.min())\nplt.imshow(sac.cpu().numpy().transpose(1,2,0))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"I.grad.data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_images = 400\nsaliencies = []\n\nfor i in range(N_images):\n    img = val_set[i][0].to(device)\n    label = val_set[i][1].to(device)\n    img.requires_grad = True\n    out = model(img.unsqueeze(0))\n    # out.backward()\n    loss = loss_fn(out.squeeze(), label)\n    loss.backward()\n\n    # Get grads\n    # grads = img.grad.data.abs()\n    grads, _ = torch.max(I.grad.data.abs(), dim=0)\n    saliencies.append(grads)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sals = torch.stack(saliencies)\nsals_mean = sals.sum(0)\nsals_mean = (sals_mean - sals_mean.min())/(sals_mean.max() - sals_mean.min())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sals.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1,2, figsize=(10,5))\nax[0].imshow(sals_mean.detach().cpu().numpy(), cmap='gray')\nax[1].imshow(test_set_unnorm[0].numpy().transpose(1,2,0))\nax[0].axis('off')\nax[0].set_title('Average saliency map', fontsize=18)\nax[1].axis('off')\nax[1].set_title('Sample test image', fontsize=18)\nplt.savefig('saliency.png')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set_unnorm = RefugeDataset(root_dir, \n                         split='test', normalize=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nfig, ax = plt.subplots(1, 2)\nax[0].imshow(image.cpu().detach().numpy().transpose(1, 2, 0))\nax[0].axis('off')\nax[1].imshow(saliency.cpu(), cmap='hot')\nax[1].axis('off')\nplt.tight_layout()\nfig.suptitle('The Image and Its Saliency Map')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train for OC/OD segmentation","metadata":{"papermill":{"duration":0.281506,"end_time":"2021-04-26T09:30:09.730551","exception":false,"start_time":"2021-04-26T09:30:09.449045","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Instantiating tracking\ntracker = MetricTracker('/kaggle/working/empty_log_file2')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define parameters\nnb_train_batches = len(train_loader)\nnb_val_batches = len(val_loader)\nnb_iter = 0\nbest_val_auc = 0.\n\nwhile model.epoch < total_epoch:\n    # Accumulators\n    train_vCDRs, val_vCDRs = [], []\n    train_classif_gts, val_classif_gts = [], []\n    train_loss, val_loss = 0., 0.\n    train_dsc_od, val_dsc_od = 0., 0.\n    train_dsc_oc, val_dsc_oc = 0., 0.\n    train_vCDR_error, val_vCDR_error = 0., 0.\n    \n    ############\n    # TRAINING #\n    ############\n    model.train()\n    train_data = iter(train_loader)\n    for k in range(nb_train_batches):\n        # Loads data\n        imgs, classif_gts, seg_gts, fov_coords, names = train_data.next()\n        imgs, classif_gts, seg_gts = imgs.to(device), classif_gts.to(device), seg_gts.to(device)\n\n        # Forward pass\n        logits = model(imgs)\n        loss = seg_loss(logits, seg_gts)\n \n        # Backward pass\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item() / nb_train_batches\n        \n        with torch.no_grad():\n            # Compute segmentation metric\n            pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            gt_od = seg_gts[:,0,:,:].type(torch.int8)\n            gt_oc = seg_gts[:,1,:,:].type(torch.int8)\n            dsc_od = compute_dice_coef(pred_od, gt_od) # Evaluate goodness of segmentation of optic disk\n            dsc_oc = compute_dice_coef(pred_oc, gt_oc) # Evaluate goodness of segmentation of optic cup\n            train_dsc_od += dsc_od.item()/nb_train_batches\n            train_dsc_oc += dsc_oc.item()/nb_train_batches\n\n            # Compute and store vCDRs\n            vCDR_error, pred_vCDR, gt_vCDR = compute_vCDR_error(pred_od.cpu().numpy(), pred_oc.cpu().numpy(), gt_od.cpu().numpy(), gt_oc.cpu().numpy())\n            train_vCDRs += pred_vCDR.tolist()\n            train_vCDR_error += vCDR_error / nb_train_batches\n            train_classif_gts += classif_gts.cpu().numpy().tolist()\n            \n        # Increase iterations\n        nb_iter += 1\n        \n        # Std out\n        print('Epoch {}, iter {}/{}, loss {:.6f}'.format(model.epoch+1, k+1, nb_train_batches, loss.item()) + ' '*20, \n              end='\\r')\n        \n    # Train a logistic regression on vCDRs\n    train_vCDRs = np.array(train_vCDRs).reshape(-1,1)\n    train_classif_gts = np.array(train_classif_gts)\n    clf = LogisticRegression(random_state=0, solver='lbfgs').fit(train_vCDRs, train_classif_gts)\n    train_classif_preds = clf.predict_proba(train_vCDRs)[:,1]\n    train_auc = classif_eval(train_classif_preds, train_classif_gts)\n    \n    ##############\n    # VALIDATION #\n    ##############\n    model.eval()\n    with torch.no_grad():\n        val_data = iter(val_loader)\n        for k in range(nb_val_batches):\n            # Loads data\n            imgs, classif_gts, seg_gts, fov_coords, names = val_data.next()\n            imgs, classif_gts, seg_gts = imgs.to(device), classif_gts.to(device), seg_gts.to(device)\n\n            # Forward pass\n            logits = model(imgs)\n            val_loss += seg_loss(logits, seg_gts).item() / nb_val_batches\n\n            # Std out\n            print('Validation iter {}/{}'.format(k+1, nb_val_batches) + ' '*50, \n                  end='\\r')\n            \n            # Compute segmentation metric\n            pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            gt_od = seg_gts[:,0,:,:].type(torch.int8)\n            gt_oc = seg_gts[:,1,:,:].type(torch.int8)\n            dsc_od = compute_dice_coef(pred_od, gt_od)\n            dsc_oc = compute_dice_coef(pred_oc, gt_oc)\n            val_dsc_od += dsc_od.item()/nb_val_batches\n            val_dsc_oc += dsc_oc.item()/nb_val_batches\n            \n            # Compute and store vCDRs\n            vCDR_error, pred_vCDR, gt_vCDR = compute_vCDR_error(pred_od.cpu().numpy(), pred_oc.cpu().numpy(), gt_od.cpu().numpy(), gt_oc.cpu().numpy())\n            val_vCDRs += pred_vCDR.tolist()\n            val_vCDR_error += vCDR_error / nb_val_batches\n            val_classif_gts += classif_gts.cpu().numpy().tolist()\n            \n\n    # Glaucoma predictions from vCDRs\n    val_vCDRs = np.array(val_vCDRs).reshape(-1,1)\n    val_classif_gts = np.array(val_classif_gts)\n    val_classif_preds = clf.predict_proba(val_vCDRs)[:,1]\n    val_auc = classif_eval(val_classif_preds, val_classif_gts)\n        \n    # Validation results\n    print('VALIDATION epoch {}'.format(model.epoch+1)+' '*50)\n    print('LOSSES: {:.4f} (train), {:.4f} (val)'.format(train_loss, val_loss))\n    tracker.update('train_loss', train_loss)\n    tracker.update('val_loss', val_loss)\n    print('OD segmentation (Dice Score): {:.4f} (train), {:.4f} (val)'.format(train_dsc_od, val_dsc_od))\n    print('OC segmentation (Dice Score): {:.4f} (train), {:.4f} (val)'.format(train_dsc_oc, val_dsc_oc))\n    tracker.update('od_segmentation_dice_score_train', train_dsc_od)\n    tracker.update('od_segmentation_dice_score_val', val_dsc_od)\n    tracker.update('oc_segmentation_dice_score_train', train_dsc_oc)\n    tracker.update('oc_segmentation_dice_score_val', val_dsc_oc)\n    print('vCDR error: {:.4f} (train), {:.4f} (val)'.format(train_vCDR_error, val_vCDR_error))\n    tracker.update('vCDR_error_train', train_vCDR_error)\n    tracker.update('vCDR_error_val', val_vCDR_error)\n    print('Classification (AUC): {:.4f} (train), {:.4f} (val)'.format(train_auc, val_auc))\n    tracker.update('AUC_train', train_auc)\n    tracker.update('AUC_val', val_auc)\n    \n    # Dump the current track as json\n    tracker.save_as_json(f'metrics_dump_epoch_{model.epoch+1}.json')\n    \n    # Save model if best validation AUC is reached\n    if val_auc > best_val_auc:\n        torch.save(model.state_dict(), '/kaggle/working/best_AUC_weights.pth')\n        with open('/kaggle/working/best_AUC_classifier.pkl', 'wb') as clf_file:\n            pickle.dump(clf, clf_file)\n        best_val_auc = val_auc\n        print('Best validation AUC reached. Saved model weights and classifier.')\n    print('_'*50)\n        \n    # End of epoch\n    model.epoch += 1\n        \n","metadata":{"papermill":{"duration":1811.719552,"end_time":"2021-04-26T10:00:21.732656","exception":false,"start_time":"2021-04-26T09:30:10.013104","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define parameters\nnb_train_batches = len(train_loader)\nnb_val_batches = len(val_loader)\nnb_iter = 0\nbest_val_auc = 0.\n\nwhile model.epoch < total_epoch:\n    # Accumulators\n    train_loss, val_loss = 0., 0.\n    train_classif_gts, train_classif_preds = [], []\n    val_classif_gts, val_classif_preds = [], []\n\n#     ############\n#     # TRAINING #\n#     ############\n#     model.train()\n#     train_data = iter(train_loader)\n#     for k in range(nb_train_batches):\n#         # Loads data\n#         imgs, classif_gts = train_data.next()\n#         imgs, classif_gts = imgs.to(device), classif_gts.to(device)\n\n#         # Forward pass\n#         logits = model(imgs)\n#         loss = loss_fn(torch.squeeze(logits), classif_gts)\n \n#         # Backward pass\n#         optimizer.zero_grad()\n#         loss.backward()\n#         optimizer.step()\n#         train_loss += loss.item() / nb_train_batches\n        \n#         with torch.no_grad():\n#             train_classif_preds += logits.detach().cpu().numpy().tolist()\n#             train_classif_gts += classif_gts.cpu().numpy().tolist()\n            \n#         # Increase iterations\n#         nb_iter += 1\n        \n#         # Std out\n#         print('Epoch {}, iter {}/{}, loss {:.6f}'.format(model.epoch+1, k+1, nb_train_batches, loss.item()) + ' '*20, \n#               end='\\r')\n        \n#     train_classif_preds = np.array(train_classif_preds)\n#     train_classif_gts = np.array(train_classif_gts)\n#     train_auc = classif_eval(train_classif_preds, train_classif_gts)\n    \n    ##############\n    # VALIDATION #\n    ##############\n    model.eval()\n    with torch.no_grad():\n        val_data = iter(val_loader)\n        for k in range(nb_val_batches):\n            # Loads data\n            imgs, classif_gts = val_data.next()\n            imgs, classif_gts = imgs.to(device), classif_gts.to(device)\n\n            # Forward pass\n            logits = model(imgs)\n            val_loss += loss_fn(torch.squeeze(logits), classif_gts).item() / nb_val_batches\n\n            # Std out\n            print('Validation iter {}/{}'.format(k+1, nb_val_batches) + ' '*50, \n                  end='\\r')\n            \n            # Compute segmentation metric\n            val_classif_preds += logits.cpu().numpy().tolist()\n            val_classif_gts += classif_gts.cpu().numpy().tolist()\n            \n\n    # Glaucoma predictions from vCDRs\n    val_classif_gts = np.array(val_classif_gts)\n    val_classif_preds = np.array(val_classif_preds)\n    val_auc = classif_eval(val_classif_preds, val_classif_gts)\n        \n    # Validation results\n    print('VALIDATION epoch {}'.format(model.epoch+1)+' '*50)\n    print('LOSSES: {:.4f} (train), {:.4f} (val)'.format(train_loss, val_loss))\n    tracker.update('train_loss', train_loss)\n    tracker.update('val_loss', val_loss)\n    print('Classification (AUC): {:.4f} (val)'.format(val_auc))\n#     print('Classification (AUC): {:.4f} (train), {:.4f} (val)'.format(train_auc, val_auc))\n#     tracker.update('AUC_train', train_auc)\n    tracker.update('AUC_val', val_auc)\n    \n    # Dump the current track as json\n    tracker.save_as_json('metrics_dump_epoch.json')\n    \n    # Save model if best validation AUC is reached\n    if val_auc > best_val_auc:\n        torch.save(model.state_dict(), '/kaggle/working/best_AUC_weights.pth')\n        #with open('/kaggle/working/best_AUC_classifier.pkl', 'wb') as clf_file:\n        #    pickle.dump(clf, clf_file)\n        best_val_auc = val_auc\n        print('Best validation AUC reached. Saved model weights and classifier.')\n    print('_'*50)\n        \n    # End of epoch\n    model.epoch += 1","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note to self and to the rest of team 6:\n\nThe loss is computed pixel-wise, the feature vector for a pixel in position $(x,y)$ is the two channels spit out by the final network layer. The vector $(c_0, c_1)$ of pixel $(x,y)$ indicates the probablity of the pixel belonging to the disk mask and the cup mask, respectively.  \nThe target for pixel $(x,y)$ is the pixel in the corresponding position in the ground truth mask. This pixel has again two values associated to it: $(y_0, y_1)$.  \nFor each pixel we have two loss components: one computed on $c_0$ and one on $c_1$. Specifically:\n\n$$l_k = y_k\\cdot\\log c_k + (1-y_k)\\cdot\\log(1-c_k)\\;\\;\\;\\text{for}\\;\\;\\;k=0,1$$\n\nThis is done $\\forall (x,y)$, so the final loss is a matrix of the same shape as the output: $(N, C, H, W)$. Then, because $\\texttt{reduction='mean'}$ was set, it is collapsed to a single scalar by taking the mean.\n\n-Batman","metadata":{}},{"cell_type":"markdown","source":"# Load best model + classifier","metadata":{"papermill":{"duration":2.01405,"end_time":"2021-04-26T10:00:25.741981","exception":false,"start_time":"2021-04-26T10:00:23.727931","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Load model and classifier\nmodel = UNet(n_channels=3, n_classes=2).to(device)\nmodel.load_state_dict(torch.load('/kaggle/working/best_AUC_weights.pth'))\nwith open('/kaggle/working/best_AUC_classifier.pkl', 'rb') as clf_file:\n    clf = pickle.load(clf_file)","metadata":{"papermill":{"duration":2.471641,"end_time":"2021-04-26T10:00:30.233875","exception":false,"start_time":"2021-04-26T10:00:27.762234","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = PanaNet().to(device)\nmodel.load_state_dict(torch.load('/kaggle/working/best_AUC_weights.pth'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check performance is maintained on validation","metadata":{"papermill":{"duration":2.042081,"end_time":"2021-04-26T10:00:34.261912","exception":false,"start_time":"2021-04-26T10:00:32.219831","status":"completed"},"tags":[]}},{"cell_type":"code","source":"model.eval()\nval_vCDRs = []\nval_classif_gts = []\nval_loss = 0.\nval_dsc_od = 0.\nval_dsc_oc = 0.\nval_vCDR_error = 0.\nwith torch.no_grad():\n    val_data = iter(val_loader)\n    for k in range(nb_val_batches):\n        # Loads data\n        imgs, classif_gts, seg_gts, fov_coords, names = val_data.next()\n        imgs, classif_gts, seg_gts = imgs.to(device), classif_gts.to(device), seg_gts.to(device)\n\n        # Forward pass\n        logits = model(imgs)\n        val_loss += seg_loss(logits, seg_gts).item() / nb_val_batches\n\n        # Std out\n        print('Validation iter {}/{}'.format(k+1, nb_val_batches) + ' '*50, \n              end='\\r')\n\n        # Compute segmentation metric\n        pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n        pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n        gt_od = seg_gts[:,0,:,:].type(torch.int8)\n        gt_oc = seg_gts[:,1,:,:].type(torch.int8)\n        dsc_od = compute_dice_coef(pred_od, gt_od)\n        dsc_oc = compute_dice_coef(pred_oc, gt_oc)\n        val_dsc_od += dsc_od.item()/nb_val_batches\n        val_dsc_oc += dsc_oc.item()/nb_val_batches\n\n        # Compute and store vCDRs\n        vCDR_error, pred_vCDR, gt_vCDR = compute_vCDR_error(pred_od.cpu().numpy(), pred_oc.cpu().numpy(), gt_od.cpu().numpy(), gt_oc.cpu().numpy())\n        val_vCDRs += pred_vCDR.tolist()\n        val_vCDR_error += vCDR_error / nb_val_batches\n        val_classif_gts += classif_gts.cpu().numpy().tolist()\n\n\n# Glaucoma predictions from vCDRs\nval_vCDRs = np.array(val_vCDRs).reshape(-1,1)\nval_classif_gts = np.array(val_classif_gts)\nval_classif_preds = clf.predict_proba(val_vCDRs)[:,1]\nval_auc = classif_eval(val_classif_preds, val_classif_gts)\n\n# Validation results\nprint('VALIDATION '+' '*50)\nprint('LOSSES: {:.4f} (val)'.format(val_loss))\nprint('OD segmentation (Dice Score): {:.4f} (val)'.format(val_dsc_od))\nprint('OC segmentation (Dice Score): {:.4f} (val)'.format(val_dsc_oc))\nprint('vCDR error: {:.4f} (val)'.format(val_vCDR_error))\nprint('Classification (AUC): {:.4f} (val)'.format(val_auc))","metadata":{"papermill":{"duration":7.689961,"end_time":"2021-04-26T10:00:43.945243","exception":false,"start_time":"2021-04-26T10:00:36.255282","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predictions on test set","metadata":{"papermill":{"duration":1.995085,"end_time":"2021-04-26T10:00:47.967262","exception":false,"start_time":"2021-04-26T10:00:45.972177","status":"completed"},"tags":[]}},{"cell_type":"code","source":"nb_test_batches = len(test_loader)\nmodel.eval()\ntest_vCDRs = []\nwith torch.no_grad():\n    test_data = iter(test_loader)\n    for k in range(nb_test_batches):\n        # Loads data\n        imgs = test_data.next()\n        imgs = imgs.to(device)\n\n        # Forward pass\n        logits = model(imgs)\n\n        # Std out\n        print('Test iter {}/{}'.format(k+1, nb_test_batches) + ' '*50, \n              end='\\r')\n            \n        # Compute segmentation\n        pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n        pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            \n        # Compute and store vCDRs\n        pred_vCDR = vertical_cup_to_disc_ratio(pred_od.cpu().numpy(), pred_oc.cpu().numpy())\n        test_vCDRs += pred_vCDR.tolist()\n            \n\n    # Glaucoma predictions from vCDRs\n    test_vCDRs = np.array(test_vCDRs).reshape(-1,1)\n    test_classif_preds = clf.predict_proba(test_vCDRs)[:,1]\n    \n# Prepare and save .csv file\ndef create_submission_csv(prediction, submission_filename='/kaggle/working/submission.csv'):\n    \"\"\"Create a sumbission file in the appropriate format for evaluation.\n\n    :param\n    prediction: list of predictions (ex: [0.12720, 0.89289, ..., 0.29829])\n    \"\"\"\n    \n    with open(submission_filename, mode='w') as csv_file:\n        fieldnames = ['Id', 'Predicted']\n        writer = csv.DictWriter(csv_file, fieldnames=fieldnames)\n        writer.writeheader()\n\n        for i, p in enumerate(prediction):\n            writer.writerow({'Id': \"T{:04d}\".format(i+1), 'Predicted': '{:f}'.format(p)})\n\ncreate_submission_csv(test_classif_preds, submission_filename='/kaggle/working/submission_augment_normalized_weightdecay.csv')\n\n# The submission.csv file is under /kaggle/working/submission.csv.\n# If you want to submit it, you should download it before closing the current kernel.","metadata":{"papermill":{"duration":7.163045,"end_time":"2021-04-26T10:00:57.140044","exception":false,"start_time":"2021-04-26T10:00:49.976999","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nb_test_batches = len(test_loader)\nmodel.eval()\n\nwith torch.no_grad():\n    test_data = iter(test_loader)\n    test_classif_preds = []\n    for k in range(nb_test_batches):\n        # Loads data\n        imgs = test_data.next()\n        imgs = imgs.to(device)\n\n        # Forward pass\n        logits = torch.squeeze(model(imgs))\n        \n\n        # Std out\n        print('Test iter {}/{}'.format(k+1, nb_test_batches) + ' '*50, \n              end='\\r')\n            \n        test_classif_preds += logits.cpu().numpy().tolist()\n\n    \n# Prepare and save .csv file\ndef create_submission_csv(prediction, submission_filename='/kaggle/working/submission.csv'):\n    \"\"\"Create a sumbission file in the appropriate format for evaluation.\n\n    :param\n    prediction: list of predictions (ex: [0.12720, 0.89289, ..., 0.29829])\n    \"\"\"\n    \n    with open(submission_filename, mode='w') as csv_file:\n        fieldnames = ['Id', 'Predicted']\n        writer = csv.DictWriter(csv_file, fieldnames=fieldnames)\n        writer.writeheader()\n\n        for i, p in enumerate(prediction):\n            writer.writerow({'Id': \"T{:04d}\".format(i+1), 'Predicted': '{:f}'.format(p)})\n\ncreate_submission_csv(test_classif_preds, submission_filename='/kaggle/working/submission_augment_normalized_weightdecay.csv')\n\n# The submission.csv file is under /kaggle/working/submission.csv.\n# If you want to submit it, you should download it before closing the current kernel.","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}