{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport json\nimport csv\nimport random\nimport pickle\nimport cv2\nimport numpy as np\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torchvision.transforms as transforms\n\nfrom PIL import Image\nfrom torch.utils.data import Dataset, DataLoader\nfrom scipy.ndimage.measurements import label\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import roc_auc_score, roc_curve\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.pipeline import make_pipeline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset and Filters","metadata":{}},{"cell_type":"code","source":"class RefugeDataset(Dataset):\n\n    def __init__(self, root_dir, split='train', output_size=(256,256), contour=\"no\"):\n        # Define attributes\n        self.output_size = output_size\n        self.root_dir = root_dir\n        self.split = split\n        print(contour)\n        \n        # Load data index\n        with open(os.path.join(self.root_dir, self.split, 'index.json')) as f:\n            self.index = json.load(f)\n            \n        self.images = []\n        for k in range(len(self.index)):\n            print('Loading {} image {}/{}...'.format(split, k, len(self.index)), end='\\r')\n            img_name = os.path.join(self.root_dir, self.split, 'images', self.index[str(k)]['ImgName'])\n            img = np.array(Image.open(img_name).convert('RGB'))\n            \n            if (contour != \"no\"):\n                img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n            \n                if (contour == \"sobel_x\"):\n                    img = cv2.Sobel(img,cv2.CV_64F,1,0,ksize=5)\n            \n                if (contour == \"sobel_y\"):\n                    img = cv2.Sobel(img,cv2.CV_64F,0,1,ksize=5)\n            \n                #if (contour == \"laplacian\"):\n                    #img= cv2.GaussianBlur(img,(3,3),0)\n                    #img = cv2.Laplacian(img,cv2.CV_64F)\n            \n                #if (contour == \"canny\"):\n                    #img= cv2.GaussianBlur(img,(3,3),0)\n                    #img = cv2.Canny(img,10,70)\n            \n            img = transforms.functional.to_tensor(img)\n            img = img.float()\n            img = transforms.functional.resize(img, self.output_size, interpolation=Image.BILINEAR)\n            self.images.append(img)\n\n        # Load ground truth for 'train' and 'val' sets\n        if split != 'test':\n            self.segs = []\n            for k in range(len(self.index)):\n                print('Loading {} segmentation {}/{}...'.format(split, k, len(self.index)), end='\\r')\n                seg_name = os.path.join(self.root_dir, self.split, 'gts', self.index[str(k)]['ImgName'].split('.')[0]+'.bmp')\n                seg = np.array(Image.open(seg_name)).copy()\n                seg = 255. - seg\n                od = (seg>=127.).astype(np.float32)\n                oc = (seg>=250.).astype(np.float32)\n                od = torch.from_numpy(od[None,:,:])\n                oc = torch.from_numpy(oc[None,:,:])\n                od = transforms.functional.resize(od, self.output_size, interpolation=Image.NEAREST)\n                oc = transforms.functional.resize(oc, self.output_size, interpolation=Image.NEAREST)\n                seg = torch.cat([od, oc], dim=0)\n                self.segs.append(seg)\n                \n        print('Succesfully loaded {} dataset.'.format(split) + ' '*50)\n            \n            \n    def __len__(self):\n        return len(self.index)\n\n    def __getitem__(self, idx):\n        # Image\n        img = self.images[idx]\n    \n        # Return only images for 'test' set\n        if self.split == 'test':\n            return img\n        \n        # Else, images and ground truth\n        else:\n            # Label\n            lab = torch.tensor(self.index[str(idx)]['Label'], dtype=torch.float32)\n\n            # Segmentation masks\n            seg = self.segs[idx]\n\n            # Fovea localization\n            f_x = self.index[str(idx)]['Fovea_X']\n            f_y = self.index[str(idx)]['Fovea_Y']\n            fov = torch.FloatTensor([f_x, f_y])\n        \n            return img, lab, seg, fov, self.index[str(idx)]['ImgName']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Metrics and ROC function","metadata":{}},{"cell_type":"code","source":"EPS = 1e-7\n\ndef compute_dice_coef(input, target):\n    '''\n    Compute dice score metric.\n    '''\n    batch_size = input.shape[0]\n    return sum([dice_coef_sample(input[k,:,:], target[k,:,:]) for k in range(batch_size)])/batch_size\n\ndef dice_coef_sample(input, target):\n    iflat = input.contiguous().view(-1)\n    tflat = target.contiguous().view(-1)\n    intersection = (iflat * tflat).sum()\n    return (2. * intersection) / (iflat.sum() + tflat.sum())\n\n\ndef vertical_diameter(binary_segmentation):\n    '''\n    Get the vertical diameter from a binary segmentation.\n    The vertical diameter is defined as the \"fattest\" area of the binary_segmentation parameter.\n    '''\n\n    # get the sum of the pixels in the vertical axis\n    vertical_axis_diameter = np.sum(binary_segmentation, axis=1)\n\n    # pick the maximum value\n    diameter = np.max(vertical_axis_diameter, axis=1)\n\n    # return it\n    return diameter\n\n\n\ndef vertical_cup_to_disc_ratio(od, oc):\n    '''\n    Compute the vertical cup-to-disc ratio from a given labelling map.\n    '''\n    # compute the cup diameter\n    cup_diameter = vertical_diameter(oc)\n    # compute the disc diameter\n    disc_diameter = vertical_diameter(od)\n\n    return cup_diameter / (disc_diameter + EPS)\n\ndef compute_vCDR_error(pred_od, pred_oc, gt_od, gt_oc):\n    '''\n    Compute vCDR prediction error, along with predicted vCDR and ground truth vCDR.\n    '''\n    pred_vCDR = vertical_cup_to_disc_ratio(pred_od, pred_oc)\n    gt_vCDR = vertical_cup_to_disc_ratio(gt_od, gt_oc)\n    vCDR_err = np.mean(np.abs(gt_vCDR - pred_vCDR))\n    return vCDR_err, pred_vCDR, gt_vCDR\n\n\ndef classif_eval(classif_preds, classif_gts):\n    '''\n    Compute AUC classification score.\n    '''\n    auc = roc_auc_score(classif_gts, classif_preds)\n    return auc\n\n\ndef fov_error(pred_fov, gt_fov):\n    '''\n    Fovea localization error metric (mean root squared error).\n    '''\n    err = np.sqrt(np.sum((gt_fov-pred_fov)**2, axis=1)).mean()\n    return err\n\ndef roc_fct(classif_preds, classif_gts):\n    #Compute ROC curve\n    print(\"a\")\n    false_positive_rate, true_positive_rate, threshold = roc_curve(classif_gts, classif_preds)\n    return false_positive_rate, true_positive_rate","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Post-processing functions","metadata":{}},{"cell_type":"code","source":"def refine_seg(pred):\n    '''\n    Only retain the biggest connected component of a segmentation map.\n    '''\n    np_pred = pred.numpy()\n        \n    largest_ccs = []\n    for i in range(np_pred.shape[0]):\n        labeled, ncomponents = label(np_pred[i,:,:])\n        bincounts = np.bincount(labeled.flat)[1:]\n        if len(bincounts) == 0:\n            largest_cc = labeled == 0\n        else:\n            largest_cc = labeled == np.argmax(bincounts)+1\n        largest_cc = torch.tensor(largest_cc, dtype=torch.float32)\n        largest_ccs.append(largest_cc)\n    largest_ccs = torch.stack(largest_ccs)\n    \n    return largest_ccs","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Network","metadata":{}},{"cell_type":"code","source":"class UNet(nn.Module):\n    def __init__(self, n_channels=3, n_classes=2):\n        super(UNet, self).__init__()\n        self.n_channels = n_channels\n        self.n_classes = n_classes\n        self.epoch = 0\n\n        self.inc = DoubleConv(n_channels, 64)\n        self.down1 = Down(64, 128)\n        self.down2 = Down(128, 256)\n        self.down3 = Down(256, 512)\n        factor = 2 \n        self.down4 = Down(512, 1024 // factor)\n        self.up1 = Up(1024, 512 // factor)\n        self.up2 = Up(512, 256 // factor)\n        self.up3 = Up(256, 128 // factor)\n        self.up4 = Up(128, 64)\n        self.output_layer = OutConv(64, n_classes)\n\n    def forward(self, x):\n        x1 = self.inc(x)\n        x2 = self.down1(x1)\n        x3 = self.down2(x2)\n        x4 = self.down3(x3)\n        x5 = self.down4(x4)\n        out = self.up1(x5, x4)\n        out = self.up2(out, x3)\n        out = self.up3(out, x2)\n        out = self.up4(out, x1)\n        out = self.output_layer(out)\n        out = torch.sigmoid(out)\n        return out\n\n    \nclass DoubleConv(nn.Module):\n    \"\"\"(convolution => [BN] => ReLU) * 2\"\"\"\n\n    def __init__(self, in_channels, out_channels, mid_channels=None):\n        super().__init__()\n        if not mid_channels:\n            mid_channels = out_channels\n        self.double_conv = nn.Sequential(\n            nn.Conv2d(in_channels, mid_channels, kernel_size=3, padding=1),\n            nn.BatchNorm2d(mid_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(mid_channels, out_channels, kernel_size=3, padding=1),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True)\n        )\n\n    def forward(self, x):\n        return self.double_conv(x)\n\n\nclass Down(nn.Module):\n    \"\"\"Downscaling with maxpool then double conv\"\"\"\n\n    def __init__(self, in_channels, out_channels):\n        super().__init__()\n        self.maxpool_conv = nn.Sequential(\n            nn.MaxPool2d(2),\n            DoubleConv(in_channels, out_channels)\n        )\n\n    def forward(self, x):\n        return self.maxpool_conv(x)\n\n\nclass Up(nn.Module):\n    \"\"\"Upscaling then double conv\"\"\"\n\n    def __init__(self, in_channels, out_channels):\n        super().__init__()\n\n        # Use the normal convolutions to reduce the number of channels\n        self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)\n        self.conv = DoubleConv(in_channels, out_channels, in_channels // 2)\n\n\n    def forward(self, x1, x2):\n        x1 = self.up(x1)\n        # input is CHW\n        diffY = x2.size()[2] - x1.size()[2]\n        diffX = x2.size()[3] - x1.size()[3]\n\n        x1 = F.pad(x1, [diffX // 2, diffX - diffX // 2,\n                        diffY // 2, diffY - diffY // 2])\n        x = torch.cat([x2, x1], dim=1)\n        return self.conv(x)\n\n\nclass OutConv(nn.Module):\n    '''\n    Simple convolution.\n    '''\n    def __init__(self, in_channels, out_channels):\n        super(OutConv, self).__init__()\n        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=1)\n\n    def forward(self, x):\n        return self.conv(x)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Settings","metadata":{}},{"cell_type":"code","source":"root_dir = '/kaggle/input/eurecom-aml-2021-challenge-2/refuge_data/refuge_data'\nlr = 1e-4\nbatch_size = 8\nnum_workers = 8\ntotal_epoch = 100\n#choose the contour filters to use\ncontour_types=['no', 'sobel_x', 'sobel_y'] #, 'laplacien', 'canny'] \nnb_models = len(contour_types)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create datasets and data loaders","metadata":{}},{"cell_type":"code","source":"train_set = [0]*nb_models\nval_set = [0]*nb_models\ntest_set = [0]*nb_models\ntrain_loader = [0]*nb_models\nval_loader = [0]*nb_models\ntest_loader = [0]*nb_models\n\nfor i in range (nb_models):\n# Datasets\n    train_set[i] = RefugeDataset(root_dir, \n                              split='train', contour=contour_types[i])\n    val_set[i] = RefugeDataset(root_dir, \n                            split='val', contour=contour_types[i])\n    test_set[i] = RefugeDataset(root_dir, \n                             split='test', contour=contour_types[i])\n\n# Dataloaders\n    train_loader[i] = DataLoader(train_set[i], \n                          batch_size=batch_size, \n                          shuffle=True, \n                          num_workers=num_workers,\n                          pin_memory=True,\n                         )\n    val_loader[i] = DataLoader(val_set[i], \n                        batch_size=batch_size, \n                        shuffle=False, \n                        num_workers=num_workers,\n                        pin_memory=True,\n                        )\n    test_loader[i] = DataLoader(test_set[i], \n                        batch_size=batch_size, \n                        shuffle=False, \n                        num_workers=num_workers,\n                        pin_memory=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Device, model adapted to the filters, loss and optimizer","metadata":{}},{"cell_type":"code","source":"# Device\ndevice = torch.device(\"cuda:0\")\n\nmodel = [0]*nb_models\nseg_loss = [0]*nb_models\noptimizer = [0]*nb_models\nfor i in range (nb_models):\n# Network\n    if (contour_types[i]=='no'):\n        model[i] = UNet(n_channels=3, n_classes=2).to(device) #RGB images\n    else:\n        model[i] = UNet(n_channels=1, n_classes=2).to(device) #with filter GRAY\n\n# Loss\n    seg_loss[i] = torch.nn.BCELoss(reduction='mean')\n\n# Optimizer\n    optimizer[i] = optim.Adam(model[i].parameters(), lr=lr)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train for OC/OD segmentation","metadata":{}},{"cell_type":"markdown","source":"## Method 1 : Using one model","metadata":{}},{"cell_type":"code","source":"# Define parameters\nnb_train_batches = len(train_loader)\nnb_val_batches = len(val_loader)\nnb_iter = 0\nbest_val_auc = 0.\n\nwhile model.epoch < total_epoch:\n    # Accumulators\n    train_vCDRs, val_vCDRs = [], []\n    train_classif_gts, val_classif_gts = [], []\n    train_loss, val_loss = 0., 0.\n    train_dsc_od, val_dsc_od = 0., 0.\n    train_dsc_oc, val_dsc_oc = 0., 0.\n    train_vCDR_error, val_vCDR_error = 0., 0.\n    \n    ############\n    # TRAINING #\n    ############\n    model.train()\n    train_data = iter(train_loader)\n    for k in range(nb_train_batches):\n        # Loads data\n        imgs, classif_gts, seg_gts, fov_coords, names = train_data.next()\n        imgs, classif_gts, seg_gts = imgs.to(device), classif_gts.to(device), seg_gts.to(device)\n\n        # Forward pass\n        logits = model(imgs)\n        loss = seg_loss(logits, seg_gts)\n \n        # Backward pass\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item() / nb_train_batches\n        \n        with torch.no_grad():\n            # Compute segmentation metric\n            pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            gt_od = seg_gts[:,0,:,:].type(torch.int8)\n            gt_oc = seg_gts[:,1,:,:].type(torch.int8)\n            dsc_od = compute_dice_coef(pred_od, gt_od)\n            dsc_oc = compute_dice_coef(pred_oc, gt_oc)\n            train_dsc_od += dsc_od.item()/nb_train_batches\n            train_dsc_oc += dsc_oc.item()/nb_train_batches\n\n\n            # Compute and store vCDRs\n            vCDR_error, pred_vCDR, gt_vCDR = compute_vCDR_error(pred_od.cpu().numpy(), pred_oc.cpu().numpy(), gt_od.cpu().numpy(), gt_oc.cpu().numpy())\n            train_vCDRs += pred_vCDR.tolist()\n            train_vCDR_error += vCDR_error / nb_train_batches\n            train_classif_gts += classif_gts.cpu().numpy().tolist()\n            \n        # Increase iterations\n        nb_iter += 1\n        \n        # Std out\n        print('Epoch {}, iter {}/{}, loss {:.6f}'.format(model.epoch+1, k+1, nb_train_batches, loss.item()) + ' '*20, \n              end='\\r')\n        \n    # Train a logistic regression on vCDRs\n    train_vCDRs = np.array(train_vCDRs).reshape(-1,1)\n    train_classif_gts = np.array(train_classif_gts)\n    clf = LogisticRegression(random_state=0, solver='lbfgs').fit(train_vCDRs, train_classif_gts)\n    train_classif_preds = clf.predict_proba(train_vCDRs)[:,1]\n    train_auc = classif_eval(train_classif_preds, train_classif_gts)\n    \n    ##############\n    # VALIDATION #\n    ##############\n    model.eval()\n    with torch.no_grad():\n        val_data = iter(val_loader)\n        for k in range(nb_val_batches):\n            # Loads data\n            imgs, classif_gts, seg_gts, fov_coords, names = val_data.next()\n            imgs, classif_gts, seg_gts = imgs.to(device), classif_gts.to(device), seg_gts.to(device)\n\n            # Forward pass\n            logits = model(imgs)\n            val_loss += seg_loss(logits, seg_gts).item() / nb_val_batches\n\n            # Std out\n            print('Validation iter {}/{}'.format(k+1, nb_val_batches) + ' '*50, \n                  end='\\r')\n            \n            # Compute segmentation metric\n            pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            gt_od = seg_gts[:,0,:,:].type(torch.int8)\n            gt_oc = seg_gts[:,1,:,:].type(torch.int8)\n            dsc_od = compute_dice_coef(pred_od, gt_od)\n            dsc_oc = compute_dice_coef(pred_oc, gt_oc)\n            val_dsc_od += dsc_od.item()/nb_val_batches\n            val_dsc_oc += dsc_oc.item()/nb_val_batches\n            \n            # Compute and store vCDRs\n            vCDR_error, pred_vCDR, gt_vCDR = compute_vCDR_error(pred_od.cpu().numpy(), pred_oc.cpu().numpy(), gt_od.cpu().numpy(), gt_oc.cpu().numpy())\n            val_vCDRs += pred_vCDR.tolist()\n            val_vCDR_error += vCDR_error / nb_val_batches\n            val_classif_gts += classif_gts.cpu().numpy().tolist()\n            \n\n    # Glaucoma predictions from vCDRs\n    val_vCDRs = np.array(val_vCDRs).reshape(-1,1)\n    val_classif_gts = np.array(val_classif_gts)\n    val_classif_preds = clf.predict_proba(val_vCDRs)[:,1]\n    val_auc = classif_eval(val_classif_preds, val_classif_gts)\n        \n    # Validation results\n    print('VALIDATION epoch {}'.format(model.epoch+1)+' '*50)\n    print('LOSSES: {:.4f} (train), {:.4f} (val)'.format(train_loss, val_loss))\n    print('OD segmentation (Dice Score): {:.4f} (train), {:.4f} (val)'.format(train_dsc_od, val_dsc_od))\n    print('OC segmentation (Dice Score): {:.4f} (train), {:.4f} (val)'.format(train_dsc_oc, val_dsc_oc))\n    print('vCDR error: {:.4f} (train), {:.4f} (val)'.format(train_vCDR_error, val_vCDR_error))\n    print('Classification (AUC): {:.4f} (train), {:.4f} (val)'.format(train_auc, val_auc))\n    \n    # Save model if best validation AUC is reached\n    if val_auc > best_val_auc:\n        torch.save(model.state_dict(), '/kaggle/working/best_AUC_weights.pth')\n        with open('/kaggle/working/best_AUC_classifier.pkl', 'wb') as clf_file:\n            pickle.dump(clf, clf_file)\n        best_val_auc = val_auc\n        print('Best validation AUC reached. Saved model weights and classifier.')\n    print('_'*50)\n        \n    # End of epoch\n    model.epoch += 1","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Method 2 : Regression on the vCDRs","metadata":{}},{"cell_type":"code","source":"# Define parameters\nnb_train_batches = len(train_loader[0])\nnb_val_batches = len(val_loader[0])\nnb_iter = 0\nbest_val_auc = 0.\ntotal_fpr=[]\ntotal_tpr=[]\n\nwhile model[0].epoch < total_epoch:\n    # Accumulators\n    train_vCDRs, val_vCDRs = [[] for i in range(nb_models)], [[] for i in range(nb_models)]\n    train_classif_gts, val_classif_gts = [], [] \n    train_loss, val_loss = [0.]*nb_models, [0.]*nb_models\n    train_dsc_od, val_dsc_od = [0.]*nb_models, [0.]*nb_models\n    train_dsc_oc, val_dsc_oc = [0.]*nb_models, [0.]*nb_models\n    train_vCDR_error, val_vCDR_error = [0.]*nb_models, [0.]*nb_models\n    \n    ############\n    # TRAINING #\n    ############\n    for i in range (nb_models):\n        model[i].train()\n        train_data = iter(train_loader[i])\n        for k in range(nb_train_batches):\n            # Loads data\n            imgs, classif_gts, seg_gts, fov_coords, names = train_data.next()\n            imgs, classif_gts, seg_gts = imgs.to(device), classif_gts.to(device), seg_gts.to(device)\n\n            # Forward pass\n            logits = model[i](imgs)\n            loss = seg_loss[i](logits, seg_gts)\n \n            # Backward pass\n            optimizer[i].zero_grad()\n            loss.backward()\n            optimizer[i].step()\n            train_loss[i] += loss.item() / nb_train_batches\n        \n            with torch.no_grad():\n                # Compute segmentation metric\n                pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n                pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n                gt_od = seg_gts[:,0,:,:].type(torch.int8)\n                gt_oc = seg_gts[:,1,:,:].type(torch.int8)\n                dsc_od = compute_dice_coef(pred_od, gt_od)\n                dsc_oc = compute_dice_coef(pred_oc, gt_oc)\n                train_dsc_od[i] += dsc_od.item()/nb_train_batches\n                train_dsc_oc[i] += dsc_oc.item()/nb_train_batches\n\n\n                # Compute and store vCDRs\n                vCDR_error, pred_vCDR, gt_vCDR = compute_vCDR_error(pred_od.cpu().numpy(), pred_oc.cpu().numpy(), gt_od.cpu().numpy(), gt_oc.cpu().numpy())\n                train_vCDRs[i] += pred_vCDR.tolist()\n                train_vCDR_error[i] += vCDR_error / nb_train_batches\n                if (i==0):\n                    train_classif_gts += classif_gts.cpu().numpy().tolist()\n            \n            # Increase iterations\n            nb_iter += 1\n        \n            # Std out\n            print('Epoch {}, iter {}/{}, loss {:.6f}, model {}'.format(model[i].epoch+1, k+1, nb_train_batches, loss.item(), i) + ' '*20, \n              end='\\r')\n##### end alinea if######\n    #transposing train_vCDRs \n    numpy_array = np.array(train_vCDRs)\n    transpose = numpy_array.T\n    train_vCDRs_T = transpose.tolist()\n    \n    # Train a logistic regression on vCDRs\n    train_classif_gts = np.array(train_classif_gts)\n    clf = LogisticRegression(random_state=0, solver='lbfgs').fit(train_vCDRs_T, train_classif_gts)\n    # clf = DecisionTreeClassifier(random_state=0).fit(train_vCDRs_T, train_classif_gts)\n    # clf = DecisionTreeClassifier(max_depth=5, random_state=0).fit(train_vCDRs_T, train_classif_gts)\n    # clf = make_pipeline(StandardScaler(), SVC(gamma='auto', probability=True)).fit(train_vCDRs_T, train_classif_gts)\n    train_classif_preds = clf.predict_proba(train_vCDRs_T)[:,1]\n    train_auc = classif_eval(train_classif_preds, train_classif_gts)\n    \n    ##############\n    # VALIDATION #\n    ##############\n    for i in range (nb_models):\n        model[i].eval()\n        with torch.no_grad():\n            val_data = iter(val_loader[i])\n            for k in range(nb_val_batches):\n                # Loads data\n                imgs, classif_gts, seg_gts, fov_coords, names = val_data.next()\n                imgs, classif_gts, seg_gts = imgs.to(device), classif_gts.to(device), seg_gts.to(device)\n\n                # Forward pass\n                logits = model[i](imgs)\n                val_loss[i] += seg_loss[i](logits, seg_gts).item() / nb_val_batches\n\n                # Std out\n                print('Validation iter {}/{}, model {}'.format(k+1, nb_val_batches, i) + ' '*50, \n                  end='\\r')\n            \n                # Compute segmentation metric\n                pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n                pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n                gt_od = seg_gts[:,0,:,:].type(torch.int8)\n                gt_oc = seg_gts[:,1,:,:].type(torch.int8)\n                dsc_od = compute_dice_coef(pred_od, gt_od)\n                dsc_oc = compute_dice_coef(pred_oc, gt_oc)\n                val_dsc_od[i] += dsc_od.item()/nb_val_batches\n                val_dsc_oc[i] += dsc_oc.item()/nb_val_batches\n\n                # Compute and store vCDRs\n                vCDR_error, pred_vCDR, gt_vCDR = compute_vCDR_error(pred_od.cpu().numpy(), pred_oc.cpu().numpy(), gt_od.cpu().numpy(), gt_oc.cpu().numpy())\n                val_vCDRs[i] += pred_vCDR.tolist()\n                val_vCDR_error[i] += vCDR_error / nb_val_batches\n                if (i==0):\n                    val_classif_gts += classif_gts.cpu().numpy().tolist()\n            \n#### end alinea if #####\n    #transposing val_vCDRs \n    numpy_array = np.array(val_vCDRs)\n    transpose = numpy_array.T\n    val_vCDRs_T = transpose.tolist()\n    # Glaucoma predictions from vCDRs\n    val_classif_gts = np.array(val_classif_gts)\n    val_classif_preds = clf.predict_proba(val_vCDRs_T)[:,1]\n    val_auc = classif_eval(val_classif_preds, val_classif_gts)\n        \n    # Validation results\n    print('VALIDATION epoch {}'.format(model[0].epoch+1)+' '*50)\n    #print('LOSSES: {:.4f} (train), {:.4f} (val)'.format(train_loss, val_loss))\n    #print('OD segmentation (Dice Score): {:.4f} (train), {:.4f} (val)'.format(train_dsc_od, val_dsc_od))\n    #print('OC segmentation (Dice Score): {:.4f} (train), {:.4f} (val)'.format(train_dsc_oc, val_dsc_oc))\n    #print('vCDR error: {:.4f} (train), {:.4f} (val)'.format(train_vCDR_error, val_vCDR_error))\n    print('Classification (AUC): {:.4f} (train), {:.4f} (val)'.format(train_auc, val_auc))\n    \n    # Save model if best validation AUC is reached\n    if val_auc > best_val_auc:\n        for i in range (nb_models):\n            torch.save(model[i].state_dict(), '/kaggle/working/best_AUC_weights'+str(i)+'.pth')\n        with open('/kaggle/working/best_AUC_classifier.pkl', 'wb') as clf_file:\n            pickle.dump(clf, clf_file)\n        best_val_auc = val_auc\n        false_positive_rate, true_positive_rate = roc_fct(val_classif_preds, val_classif_gts)\n        total_fpr.append(false_positive_rate)\n        total_tpr.append(true_positive_rate)\n        print('Best validation AUC reached. Saved model weights and classifier.')\n    print('_'*50)\n        \n    # End of epoch\n    for i in range (nb_models):\n        model[i].epoch += 1\n        ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Method 3 : Mean of the classifications of the best models","metadata":{}},{"cell_type":"code","source":"# Define parameters\nnb_train_batches = len(train_loader[0])\nnb_val_batches = len(val_loader[0])\nnb_iter = 0\nbest_val_auc = [0]*nb_models\ntrain_auc = [0]*nb_models\nval_auc = [0]*nb_models\n\nwhile model[0].epoch < total_epoch:\n    # Accumulators\n    train_vCDRs, val_vCDRs = [[] for i in range(nb_models)], [[] for i in range(nb_models)]\n    train_classif_gts, val_classif_gts = [], [] \n    train_loss, val_loss = [0.]*nb_models, [0.]*nb_models\n    train_dsc_od, val_dsc_od = [0.]*nb_models, [0.]*nb_models\n    train_dsc_oc, val_dsc_oc = [0.]*nb_models, [0.]*nb_models\n    train_vCDR_error, val_vCDR_error = [0.]*nb_models, [0.]*nb_models\n    train_classif_preds, val_classif_preds = [[] for i in range(nb_models)], [[] for i in range(nb_models)]\n    \n    ############\n    # TRAINING #\n    ############\n    for i in range (nb_models):\n        model[i].train()\n        train_data = iter(train_loader[i])\n        for k in range(nb_train_batches):\n            # Loads data\n            imgs, classif_gts, seg_gts, fov_coords, names = train_data.next()\n            imgs, classif_gts, seg_gts = imgs.to(device), classif_gts.to(device), seg_gts.to(device)\n\n            # Forward pass\n            logits = model[i](imgs)\n            loss = seg_loss[i](logits, seg_gts)\n \n            # Backward pass\n            optimizer[i].zero_grad()\n            loss.backward()\n            optimizer[i].step()\n            train_loss[i] += loss.item() / nb_train_batches\n        \n            with torch.no_grad():\n                # Compute segmentation metric\n                pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n                pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n                gt_od = seg_gts[:,0,:,:].type(torch.int8)\n                gt_oc = seg_gts[:,1,:,:].type(torch.int8)\n                dsc_od = compute_dice_coef(pred_od, gt_od)\n                dsc_oc = compute_dice_coef(pred_oc, gt_oc)\n                train_dsc_od[i] += dsc_od.item()/nb_train_batches\n                train_dsc_oc[i] += dsc_oc.item()/nb_train_batches\n\n\n                # Compute and store vCDRs\n                vCDR_error, pred_vCDR, gt_vCDR = compute_vCDR_error(pred_od.cpu().numpy(), pred_oc.cpu().numpy(), gt_od.cpu().numpy(), gt_oc.cpu().numpy())\n                train_vCDRs[i] += pred_vCDR.tolist()\n                train_vCDR_error[i] += vCDR_error / nb_train_batches\n                if (i==0):\n                    train_classif_gts += classif_gts.cpu().numpy().tolist()\n            \n            # Increase iterations\n            nb_iter += 1\n        \n            # Std out\n            print('Epoch {}, iter {}/{}, loss {:.6f}, model {}'.format(model[i].epoch+1, k+1, nb_train_batches, loss.item(), i) + ' '*20, \n              end='\\r')\n\n        # Train a logistic regression on vCDRs\n        train_vCDRs[i] = np.array(train_vCDRs[i]).reshape(-1,1)\n        train_classif_gts = np.array(train_classif_gts)\n        clf = LogisticRegression(random_state=0, solver='lbfgs').fit(train_vCDRs[i], train_classif_gts)\n        train_classif_preds[i] = clf.predict_proba(train_vCDRs[i])[:,1]\n        train_auc[i] = classif_eval(train_classif_preds[i], train_classif_gts)\n     \n    # averaging the prediction of all models\n    final_preds = []\n    train_classif_preds_a = np.array(train_classif_preds)\n    final_preds = np.mean(train_classif_preds_a, axis = 0)\n    final_train_auc = classif_eval(final_preds, train_classif_gts)\n\n    \n    ##############\n    # VALIDATION #\n    ##############\n    for i in range (nb_models):\n        model[i].eval()\n        with torch.no_grad():\n            val_data = iter(val_loader[i])\n            for k in range(nb_val_batches):\n                # Loads data\n                imgs, classif_gts, seg_gts, fov_coords, names = val_data.next()\n                imgs, classif_gts, seg_gts = imgs.to(device), classif_gts.to(device), seg_gts.to(device)\n\n                # Forward pass\n                logits = model[i](imgs)\n                val_loss[i] += seg_loss[i](logits, seg_gts).item() / nb_val_batches\n\n                # Std out\n                print('Validation iter {}/{}, model {}'.format(k+1, nb_val_batches, i) + ' '*50, \n                  end='\\r')\n            \n                # Compute segmentation metric\n                pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n                pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n                gt_od = seg_gts[:,0,:,:].type(torch.int8)\n                gt_oc = seg_gts[:,1,:,:].type(torch.int8)\n                dsc_od = compute_dice_coef(pred_od, gt_od)\n                dsc_oc = compute_dice_coef(pred_oc, gt_oc)\n                val_dsc_od[i] += dsc_od.item()/nb_val_batches\n                val_dsc_oc[i] += dsc_oc.item()/nb_val_batches\n            \n                # Compute and store vCDRs\n                vCDR_error, pred_vCDR, gt_vCDR = compute_vCDR_error(pred_od.cpu().numpy(), pred_oc.cpu().numpy(), gt_od.cpu().numpy(), gt_oc.cpu().numpy())\n                val_vCDRs[i] += pred_vCDR.tolist()\n                val_vCDR_error[i] += vCDR_error / nb_val_batches\n                if (i==0):\n                    val_classif_gts += classif_gts.cpu().numpy().tolist()\n            \n\n        # Glaucoma predictions from vCDRs\n        val_vCDRs[i]=np.array(val_vCDRs[i]).reshape(-1,1)\n        val_classif_gts = np.array(val_classif_gts)\n        val_classif_preds[i] = clf.predict_proba(val_vCDRs[i])[:,1]\n        val_auc[i] = classif_eval(val_classif_preds[i], val_classif_gts)\n        \n    # averaging the prediction of all models\n    final_preds = []\n    val_classif_preds_a = np.array(val_classif_preds)\n    final_preds = np.mean(val_classif_preds_a, axis = 0)\n    final_val_auc = classif_eval(final_preds, val_classif_gts)\n\n        \n    # Validation results\n    print('VALIDATION epoch {}'.format(model[0].epoch+1)+' '*50)\n    \n    # Save model if best validation AUC is reached\n    for i in range (nb_models):\n        if val_auc[i] > best_val_auc[i]:\n            torch.save(model[i].state_dict(), '/kaggle/working/best_AUC_weights'+str(i)+'.pth')\n            with open('/kaggle/working/best_AUC_classifier.pkl', 'wb') as clf_file:\n                pickle.dump(clf, clf_file)\n            best_val_auc[i] = val_auc[i]\n            print('Best validation AUC reached for model' + str(i) + 'Best AUC = ' + str(best_val_auc[i])+'. Saved model weights and classifier.')\n    print('_'*50)\n        \n    # End of epoch\n    for i in range (nb_models):\n        model[i].epoch += 1\n        ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Method 4 : Linear Regression on the classifications of the best models","metadata":{}},{"cell_type":"code","source":"# Define parameters\nnb_train_batches = len(train_loader[0])\nnb_val_batches = len(val_loader[0])\nnb_iter = 0\nbest_val_auc = [0]*nb_models\ntrain_auc = [0]*nb_models\nval_auc = [0]*nb_models\n\nwhile model[0].epoch < total_epoch:\n    # Accumulators\n    train_vCDRs, val_vCDRs = [[] for i in range(nb_models)], [[] for i in range(nb_models)]\n    train_classif_gts, val_classif_gts = [], [] \n    train_loss, val_loss = [0.]*nb_models, [0.]*nb_models\n    train_dsc_od, val_dsc_od = [0.]*nb_models, [0.]*nb_models\n    train_dsc_oc, val_dsc_oc = [0.]*nb_models, [0.]*nb_models\n    train_vCDR_error, val_vCDR_error = [0.]*nb_models, [0.]*nb_models\n    train_classif_preds, val_classif_preds = [[] for i in range(nb_models)], [[] for i in range(nb_models)]\n    \n    ############\n    # TRAINING #\n    ############\n    for i in range (nb_models):\n        model[i].train()\n        train_data = iter(train_loader[i])\n        for k in range(nb_train_batches):\n            # Loads data\n            imgs, classif_gts, seg_gts, fov_coords, names = train_data.next()\n            imgs, classif_gts, seg_gts = imgs.to(device), classif_gts.to(device), seg_gts.to(device)\n\n            # Forward pass\n            logits = model[i](imgs)\n            loss = seg_loss[i](logits, seg_gts)\n \n            # Backward pass\n            optimizer[i].zero_grad()\n            loss.backward()\n            optimizer[i].step()\n            train_loss[i] += loss.item() / nb_train_batches\n        \n            with torch.no_grad():\n                # Compute segmentation metric\n                pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n                pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n                gt_od = seg_gts[:,0,:,:].type(torch.int8)\n                gt_oc = seg_gts[:,1,:,:].type(torch.int8)\n                dsc_od = compute_dice_coef(pred_od, gt_od)\n                dsc_oc = compute_dice_coef(pred_oc, gt_oc)\n                train_dsc_od[i] += dsc_od.item()/nb_train_batches\n                train_dsc_oc[i] += dsc_oc.item()/nb_train_batches\n\n\n                # Compute and store vCDRs\n                vCDR_error, pred_vCDR, gt_vCDR = compute_vCDR_error(pred_od.cpu().numpy(), pred_oc.cpu().numpy(), gt_od.cpu().numpy(), gt_oc.cpu().numpy())\n                train_vCDRs[i] += pred_vCDR.tolist()\n                train_vCDR_error[i] += vCDR_error / nb_train_batches\n                if (i==0):\n                    train_classif_gts += classif_gts.cpu().numpy().tolist()\n            \n            # Increase iterations\n            nb_iter += 1\n        \n            # Std out\n            print('Epoch {}, iter {}/{}, loss {:.6f}, model {}'.format(model[i].epoch+1, k+1, nb_train_batches, loss.item(), i) + ' '*20, \n              end='\\r')\n\n        # Train a logistic regression on vCDRs\n        train_vCDRs[i] = np.array(train_vCDRs[i]).reshape(-1,1)\n        train_classif_gts = np.array(train_classif_gts)\n        clf = LogisticRegression(random_state=0, solver='lbfgs').fit(train_vCDRs[i], train_classif_gts)\n        train_classif_preds[i] = clf.predict_proba(train_vCDRs[i])[:,1]\n        train_auc[i] = classif_eval(train_classif_preds[i], train_classif_gts)\n     \n    \n    ##############\n    # VALIDATION #\n    ##############\n    for i in range (nb_models):\n        model[i].eval()\n        with torch.no_grad():\n            val_data = iter(val_loader[i])\n            for k in range(nb_val_batches):\n                # Loads data\n                imgs, classif_gts, seg_gts, fov_coords, names = val_data.next()\n                imgs, classif_gts, seg_gts = imgs.to(device), classif_gts.to(device), seg_gts.to(device)\n\n                # Forward pass\n                logits = model[i](imgs)\n                val_loss[i] += seg_loss[i](logits, seg_gts).item() / nb_val_batches\n\n                # Std out\n                print('Validation iter {}/{}, model {}'.format(k+1, nb_val_batches, i) + ' '*50, \n                  end='\\r')\n            \n                # Compute segmentation metric\n                pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n                pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n                gt_od = seg_gts[:,0,:,:].type(torch.int8)\n                gt_oc = seg_gts[:,1,:,:].type(torch.int8)\n                dsc_od = compute_dice_coef(pred_od, gt_od)\n                dsc_oc = compute_dice_coef(pred_oc, gt_oc)\n                val_dsc_od[i] += dsc_od.item()/nb_val_batches\n                val_dsc_oc[i] += dsc_oc.item()/nb_val_batches\n            \n                # Compute and store vCDRs\n                vCDR_error, pred_vCDR, gt_vCDR = compute_vCDR_error(pred_od.cpu().numpy(), pred_oc.cpu().numpy(), gt_od.cpu().numpy(), gt_oc.cpu().numpy())\n                val_vCDRs[i] += pred_vCDR.tolist()\n                val_vCDR_error[i] += vCDR_error / nb_val_batches\n                if (i==0):\n                    val_classif_gts += classif_gts.cpu().numpy().tolist()\n            \n\n        # Glaucoma predictions from vCDRs\n        val_vCDRs[i]=np.array(val_vCDRs[i]).reshape(-1,1)\n        val_classif_gts = np.array(val_classif_gts)\n        val_classif_preds[i] = clf.predict_proba(val_vCDRs[i])[:,1]\n        val_auc[i] = classif_eval(val_classif_preds[i], val_classif_gts)\n        \n        \n    # Validation results\n    print('VALIDATION epoch {}'.format(model[0].epoch+1)+' '*50)\n    \n    # Save model if best validation AUC is reached\n    for i in range (nb_models):\n        if val_auc[i] > best_val_auc[i]:\n            torch.save(model[i].state_dict(), '/kaggle/working/best_AUC_weights'+str(i)+'.pth')\n            with open('/kaggle/working/best_AUC_classifier.pkl', 'wb') as clf_file:\n                pickle.dump(clf, clf_file)\n            best_val_auc[i] = val_auc[i]\n            print('Best validation AUC reached for model' + str(i) + 'Best AUC = ' + str(best_val_auc[i])+'. Saved model weights and classifier.')\n    print('_'*50)\n        \n    # End of epoch\n    for i in range (nb_models):\n        model[i].epoch += 1","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load best model + classifier","metadata":{}},{"cell_type":"code","source":"# Load model and classifier\nfor i in range (nb_models):\n    if contour_types[i]==\"no\":\n        model[i] = UNet(n_channels=3, n_classes=2).to(device)\n    else:\n        model[i] = UNet(n_channels=1, n_classes=2).to(device)\n    model[i].load_state_dict(torch.load('/kaggle/working/best_AUC_weights'+str(i)+'.pth'))\nwith open('/kaggle/working/best_AUC_classifier.pkl', 'rb') as clf_file:\n    clf = pickle.load(clf_file)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Linear Regression for model 4","metadata":{}},{"cell_type":"code","source":"model[i].eval()\ntrain_vCDRs = [[] for i in range(nb_models)]\ntrain_classif_gts = []\ntrain_loss = [0.]*nb_models\ntrain_dsc_od = [0.]*nb_models\ntrain_dsc_oc = [0.]*nb_models\ntrain_vCDR_error = [0.]*nb_models\ntrain_classif_preds = [[] for i in range(nb_models)]\n\n\nfor i in range (nb_models):\n    with torch.no_grad():\n        train_data = iter(train_loader[i])\n        for k in range(nb_train_batches):\n            # Loads data\n            imgs, classif_gts, seg_gts, fov_coords, names = train_data.next()\n            imgs, classif_gts, seg_gts = imgs.to(device), classif_gts.to(device), seg_gts.to(device)\n\n            # Forward pass\n            logits = model[i](imgs)\n            val_loss[i] += seg_loss[i](logits, seg_gts).item() / nb_train_batches\n\n            # Std out\n            print('Train iter {}/{}, model {}'.format(k+1, nb_train_batches, i) + ' '*50, \n              end='\\r')\n\n            # Compute segmentation metric\n            pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            gt_od = seg_gts[:,0,:,:].type(torch.int8)\n            gt_oc = seg_gts[:,1,:,:].type(torch.int8)\n            dsc_od = compute_dice_coef(pred_od, gt_od)\n            dsc_oc = compute_dice_coef(pred_oc, gt_oc)\n            train_dsc_od[i] += dsc_od.item()/nb_train_batches\n            train_dsc_oc[i] += dsc_oc.item()/nb_train_batches\n\n            # Compute and store vCDRs\n            vCDR_error, pred_vCDR, gt_vCDR = compute_vCDR_error(pred_od.cpu().numpy(), pred_oc.cpu().numpy(), gt_od.cpu().numpy(), gt_oc.cpu().numpy())\n            train_vCDRs[i] += pred_vCDR.tolist()\n            train_vCDR_error[i] += vCDR_error / nb_train_batches\n            if (i==0):\n                train_classif_gts += classif_gts.cpu().numpy().tolist()\n                \n    train_vCDRs[i]=np.array(train_vCDRs[i]).reshape(-1,1)\n    train_classif_gts = np.array(train_classif_gts)\n    train_classif_preds[i] = clf.predict_proba(train_vCDRs[i])[:,1]\ntrain_classif_preds = np.array(train_classif_preds).T\n    \nclf_final = LogisticRegression(random_state=0, solver='lbfgs').fit(train_classif_preds, train_classif_gts)\ntrain_final_preds = clf_final.predict_proba(train_classif_preds)[:,1]\ntrain_auc = classif_eval(train_final_preds, train_classif_gts)\n\n# Validation results\nprint('Train'+' '*50)\n#print('LOSSES: {:.4f} (val)'.format(val_loss))\n#print('OD segmentation (Dice Score): {:.4f} (val)'.format(val_dsc_od))\n#print('OC segmentation (Dice Score): {:.4f} (val)'.format(val_dsc_oc))\n#print('vCDR error: {:.4f} (val)'.format(val_vCDR_error))\nprint('Classification (AUC): {:.4f} (val)'.format(val_auc))","metadata":{"jupyter":{"source_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check performance is maintained on validation","metadata":{}},{"cell_type":"markdown","source":"## AUC","metadata":{}},{"cell_type":"markdown","source":"### Model 1","metadata":{}},{"cell_type":"code","source":"model.eval()\nval_vCDRs = []\nval_classif_gts = []\nval_loss = 0.\nval_dsc_od = 0.\nval_dsc_oc = 0.\nval_vCDR_error = 0.\nwith torch.no_grad():\n    val_data = iter(val_loader)\n    for k in range(nb_val_batches):\n        # Loads data\n        imgs, classif_gts, seg_gts, fov_coords, names = val_data.next()\n        imgs, classif_gts, seg_gts = imgs.to(device), classif_gts.to(device), seg_gts.to(device)\n\n        # Forward pass\n        logits = model(imgs)\n        val_loss += seg_loss(logits, seg_gts).item() / nb_val_batches\n\n        # Std out\n        print('Validation iter {}/{}'.format(k+1, nb_val_batches) + ' '*50, \n              end='\\r')\n\n        # Compute segmentation metric\n        pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n        pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n        gt_od = seg_gts[:,0,:,:].type(torch.int8)\n        gt_oc = seg_gts[:,1,:,:].type(torch.int8)\n        dsc_od = compute_dice_coef(pred_od, gt_od)\n        dsc_oc = compute_dice_coef(pred_oc, gt_oc)\n        val_dsc_od += dsc_od.item()/nb_val_batches\n        val_dsc_oc += dsc_oc.item()/nb_val_batches\n\n        # Compute and store vCDRs\n        vCDR_error, pred_vCDR, gt_vCDR = compute_vCDR_error(pred_od.cpu().numpy(), pred_oc.cpu().numpy(), gt_od.cpu().numpy(), gt_oc.cpu().numpy())\n        val_vCDRs += pred_vCDR.tolist()\n        val_vCDR_error += vCDR_error / nb_val_batches\n        val_classif_gts += classif_gts.cpu().numpy().tolist()\n\n\n# Glaucoma predictions from vCDRs\nval_vCDRs = np.array(val_vCDRs).reshape(-1,1)\nval_classif_gts = np.array(val_classif_gts)\nval_classif_preds = clf.predict_proba(val_vCDRs)[:,1]\nval_auc = classif_eval(val_classif_preds, val_classif_gts)\n\n# Validation results\nprint('VALIDATION '+' '*50)\nprint('LOSSES: {:.4f} (val)'.format(val_loss))\nprint('OD segmentation (Dice Score): {:.4f} (val)'.format(val_dsc_od))\nprint('OC segmentation (Dice Score): {:.4f} (val)'.format(val_dsc_oc))\nprint('vCDR error: {:.4f} (val)'.format(val_vCDR_error))\nprint('Classification (AUC): {:.4f} (val)'.format(val_auc))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model 2","metadata":{}},{"cell_type":"code","source":"val_vCDRs = [[] for i in range(nb_models)]\nval_classif_gts = []\nval_loss = [0.]*nb_models\nval_dsc_od = [0.]*nb_models\nval_dsc_oc = [0.]*nb_models\nval_vCDR_error = [0.]*nb_models\n\nfor i in range (nb_models):\n    model[i].eval()\n    with torch.no_grad():\n        val_data = iter(val_loader[i])\n        for k in range(nb_val_batches):\n            # Loads data\n            imgs, classif_gts, seg_gts, fov_coords, names = val_data.next()\n            imgs, classif_gts, seg_gts = imgs.to(device), classif_gts.to(device), seg_gts.to(device)\n\n            # Forward pass\n            logits = model[i](imgs)\n            val_loss[i] += seg_loss[i](logits, seg_gts).item() / nb_val_batches\n\n            # Std out\n            print('Validation iter {}/{}, model {}'.format(k+1, nb_val_batches, i+1) + ' '*50, \n              end='\\r')\n\n            # Compute segmentation metric\n            pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            gt_od = seg_gts[:,0,:,:].type(torch.int8)\n            gt_oc = seg_gts[:,1,:,:].type(torch.int8)\n            dsc_od = compute_dice_coef(pred_od, gt_od)\n            dsc_oc = compute_dice_coef(pred_oc, gt_oc)\n            val_dsc_od[i] += dsc_od.item()/nb_val_batches\n            val_dsc_oc[i] += dsc_oc.item()/nb_val_batches\n\n            # Compute and store vCDRs\n            vCDR_error, pred_vCDR, gt_vCDR = compute_vCDR_error(pred_od.cpu().numpy(), pred_oc.cpu().numpy(), gt_od.cpu().numpy(), gt_oc.cpu().numpy())\n            val_vCDRs[i] += pred_vCDR.tolist()\n            val_vCDR_error[i] += vCDR_error / nb_val_batches\n            if (i==0):\n                val_classif_gts += classif_gts.cpu().numpy().tolist()\n\n#### end alinea if ####\n#transposing val_vCDRs \nnumpy_array = np.array(val_vCDRs)\ntranspose = numpy_array.T\nval_vCDRs_T = transpose.tolist()\n# Glaucoma predictions from vCDRs\nval_classif_gts = np.array(val_classif_gts)\nval_classif_preds = clf.predict_proba(val_vCDRs_T)[:,1]\nval_auc = classif_eval(val_classif_preds, val_classif_gts)\n\n# Validation results\nprint('VALIDATION '+' '*50)\n#print('LOSSES: {:.4f} (val)'.format(val_loss))\n#print('OD segmentation (Dice Score): {:.4f} (val)'.format(val_dsc_od))\n#print('OC segmentation (Dice Score): {:.4f} (val)'.format(val_dsc_oc))\n#print('vCDR error: {:.4f} (val)'.format(val_vCDR_error))\nprint('Classification (AUC): {:.4f} (val)'.format(val_auc))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model 3","metadata":{}},{"cell_type":"code","source":"model[i].eval()\nval_vCDRs = [[] for i in range(nb_models)]\nval_classif_gts = []\nval_loss = [0.]*nb_models\nval_dsc_od = [0.]*nb_models\nval_dsc_oc = [0.]*nb_models\nval_vCDR_error = [0.]*nb_models\ntotal_fpr=[]\ntotal_tpr=[]\nval_classif_preds = [[] for i in range(nb_models)]\n\nfor i in range (nb_models):\n    with torch.no_grad():\n        val_data = iter(val_loader[i])\n        for k in range(nb_val_batches):\n            # Loads data\n            imgs, classif_gts, seg_gts, fov_coords, names = val_data.next()\n            imgs, classif_gts, seg_gts = imgs.to(device), classif_gts.to(device), seg_gts.to(device)\n\n            # Forward pass\n            logits = model[i](imgs)\n            val_loss[i] += seg_loss[i](logits, seg_gts).item() / nb_val_batches\n\n            # Std out\n            print('Validation iter {}/{}, model {}'.format(k+1, nb_val_batches, i) + ' '*50, \n              end='\\r')\n\n            # Compute segmentation metric\n            pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            gt_od = seg_gts[:,0,:,:].type(torch.int8)\n            gt_oc = seg_gts[:,1,:,:].type(torch.int8)\n            dsc_od = compute_dice_coef(pred_od, gt_od)\n            dsc_oc = compute_dice_coef(pred_oc, gt_oc)\n            val_dsc_od[i] += dsc_od.item()/nb_val_batches\n            val_dsc_oc[i] += dsc_oc.item()/nb_val_batches\n\n            # Compute and store vCDRs\n            vCDR_error, pred_vCDR, gt_vCDR = compute_vCDR_error(pred_od.cpu().numpy(), pred_oc.cpu().numpy(), gt_od.cpu().numpy(), gt_oc.cpu().numpy())\n            val_vCDRs[i] += pred_vCDR.tolist()\n            val_vCDR_error[i] += vCDR_error / nb_val_batches\n            if (i==0):\n                val_classif_gts += classif_gts.cpu().numpy().tolist()\n                \n    val_vCDRs[i]=np.array(val_vCDRs[i]).reshape(-1,1)\n    val_classif_gts = np.array(val_classif_gts)\n    val_classif_preds[i] = clf.predict_proba(val_vCDRs[i])[:,1]\n    \nval_classif_preds_a = np.array(val_classif_preds)\nfinal_preds = np.mean(val_classif_preds_a, axis = 0)\nval_auc = classif_eval(final_preds, val_classif_gts)\n\nfalse_positive_rate, true_positive_rate = roc_fct(final_preds, val_classif_gts)\ntotal_fpr.append(false_positive_rate)\ntotal_tpr.append(true_positive_rate)\n    \n# Validation results\nprint('VALIDATION '+' '*50)\nprint('Classification (AUC): {:.4f} (val)'.format(val_auc))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model 4","metadata":{}},{"cell_type":"code","source":"model[i].eval()\nval_vCDRs = [[] for i in range(nb_models)]\nval_classif_gts = []\nval_loss = [0.]*nb_models\nval_dsc_od = [0.]*nb_models\nval_dsc_oc = [0.]*nb_models\nval_vCDR_error = [0.]*nb_models\ntotal_fpr=[]\ntotal_tpr=[]\nval_classif_preds = [[] for i in range(nb_models)]\n\nfor i in range (nb_models):\n    with torch.no_grad():\n        val_data = iter(val_loader[i])\n        for k in range(nb_val_batches):\n            # Loads data\n            imgs, classif_gts, seg_gts, fov_coords, names = val_data.next()\n            imgs, classif_gts, seg_gts = imgs.to(device), classif_gts.to(device), seg_gts.to(device)\n\n            # Forward pass\n            logits = model[i](imgs)\n            val_loss[i] += seg_loss[i](logits, seg_gts).item() / nb_val_batches\n\n            # Std out\n            print('Validation iter {}/{}, model {}'.format(k+1, nb_val_batches, i) + ' '*50, \n              end='\\r')\n\n            # Compute segmentation metric\n            pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            gt_od = seg_gts[:,0,:,:].type(torch.int8)\n            gt_oc = seg_gts[:,1,:,:].type(torch.int8)\n            dsc_od = compute_dice_coef(pred_od, gt_od)\n            dsc_oc = compute_dice_coef(pred_oc, gt_oc)\n            val_dsc_od[i] += dsc_od.item()/nb_val_batches\n            val_dsc_oc[i] += dsc_oc.item()/nb_val_batches\n\n            # Compute and store vCDRs\n            vCDR_error, pred_vCDR, gt_vCDR = compute_vCDR_error(pred_od.cpu().numpy(), pred_oc.cpu().numpy(), gt_od.cpu().numpy(), gt_oc.cpu().numpy())\n            val_vCDRs[i] += pred_vCDR.tolist()\n            val_vCDR_error[i] += vCDR_error / nb_val_batches\n            if (i==0):\n                val_classif_gts += classif_gts.cpu().numpy().tolist()\n                \n    val_vCDRs[i]=np.array(val_vCDRs[i]).reshape(-1,1)\n    val_classif_gts = np.array(val_classif_gts)\n    val_classif_preds[i] = clf.predict_proba(val_vCDRs[i])[:,1]\n    \nval_classif_preds = np.array(val_classif_preds).T\nval_final_preds = clf_final.predict_proba(val_classif_preds)[:,1]\nval_auc = classif_eval(train_final_preds, train_classif_gts)\n\nfalse_positive_rate, true_positive_rate = roc_fct(final_preds, val_classif_gts)\ntotal_fpr.append(false_positive_rate)\ntotal_tpr.append(true_positive_rate)\n    \n# Validation results\nprint('VALIDATION '+' '*50)\nprint('Classification (AUC): {:.4f} (val)'.format(val_auc))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ROC","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n#plotting ROC curves\nplt.subplots(1, figsize=(10,10))\nplt.title('Receiver Operating Characteristic')\nfor i in range (len(total_fpr)):\n    plt.plot(total_fpr[i], total_tpr[i])\nplt.plot([0, 1], ls=\"--\")\nplt.plot([0, 0], [1, 0] , c=\".7\"), plt.plot([1, 1] , c=\".7\")\nplt.ylabel('True Positive Rate')\nplt.xlabel('False Positive Rate')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predictions on test set","metadata":{}},{"cell_type":"markdown","source":"## Model 1","metadata":{}},{"cell_type":"code","source":"nb_test_batches = len(test_loader)\nmodel.eval()\ntest_vCDRs = []\nwith torch.no_grad():\n    test_data = iter(test_loader)\n    for k in range(nb_test_batches):\n        # Loads data\n        imgs = test_data.next()\n        imgs = imgs.to(device)\n\n        # Forward pass\n        logits = model(imgs)\n\n        # Std out\n        print('Test iter {}/{}'.format(k+1, nb_test_batches) + ' '*50, \n              end='\\r')\n            \n        # Compute segmentation\n        pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n        pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            \n        # Compute and store vCDRs\n        pred_vCDR = vertical_cup_to_disc_ratio(pred_od.cpu().numpy(), pred_oc.cpu().numpy())\n        test_vCDRs += pred_vCDR.tolist()\n            \n\n    # Glaucoma predictions from vCDRs\n    test_vCDRs = np.array(test_vCDRs).reshape(-1,1)\n    test_classif_preds = clf.predict_proba(test_vCDRs)[:,1]\n    \n# Prepare and save .csv file\ndef create_submission_csv(prediction, submission_filename='/kaggle/working/submission.csv'):\n    \"\"\"Create a sumbission file in the appropriate format for evaluation.\n\n    :param\n    prediction: list of predictions (ex: [0.12720, 0.89289, ..., 0.29829])\n    \"\"\"\n    \n    with open(submission_filename, mode='w') as csv_file:\n        fieldnames = ['Id', 'Predicted']\n        writer = csv.DictWriter(csv_file, fieldnames=fieldnames)\n        writer.writeheader()\n\n        for i, p in enumerate(prediction):\n            writer.writerow({'Id': \"T{:04d}\".format(i+1), 'Predicted': '{:f}'.format(p)})\n\ncreate_submission_csv(test_classif_preds)\n\n# The submission.csv file is under /kaggle/working/submission.csv.\n# If you want to submit it, you should download it before closing the current kernel.","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model 2","metadata":{}},{"cell_type":"code","source":"test_vCDRs = [[] for i in range(nb_models)]\nfor i in range (nb_models):\n    model[i].eval()\n    nb_test_batches = len(test_loader[i])\n    with torch.no_grad():\n        test_data = iter(test_loader[i])\n        for k in range(nb_test_batches):\n            # Loads data\n            imgs = test_data.next()\n            imgs = imgs.to(device)\n\n            # Forward pass\n            logits = model[i](imgs)\n\n            # Std out\n            print('Test iter {}/{}, model {}/{}'.format(k+1, nb_test_batches, i+1, nb_models) + ' '*50, \n              end='\\r')\n            \n            # Compute segmentation\n            pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            \n            # Compute and store vCDRs\n            pred_vCDR = vertical_cup_to_disc_ratio(pred_od.cpu().numpy(), pred_oc.cpu().numpy())\n            test_vCDRs[i] += pred_vCDR.tolist()\n\n#transposing test_vCDRs \nnumpy_array = np.array(test_vCDRs)\ntranspose = numpy_array.T\ntest_vCDRs_T = transpose.tolist()\n# Glaucoma predictions from vCDRs\ntest_classif_preds = clf.predict_proba(test_vCDRs_T)[:,1]\n    \n# Prepare and save .csv file\ndef create_submission_csv(prediction, submission_filename='/kaggle/working/submission.csv'):\n    \"\"\"Create a sumbission file in the appropriate format for evaluation.\n\n    :param\n    prediction: list of predictions (ex: [0.12720, 0.89289, ..., 0.29829])\n    \"\"\"\n    \n    with open(submission_filename, mode='w') as csv_file:\n        fieldnames = ['Id', 'Predicted']\n        writer = csv.DictWriter(csv_file, fieldnames=fieldnames)\n        writer.writeheader()\n\n        for i, p in enumerate(prediction):\n            writer.writerow({'Id': \"T{:04d}\".format(i+1), 'Predicted': '{:f}'.format(p)})\n\ncreate_submission_csv(test_classif_preds)\n\n# The submission.csv file is under /kaggle/working/submission.csv.\n# If you want to submit it, you should download it before closing the current kernel.","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model 3","metadata":{}},{"cell_type":"code","source":"test_classif_preds = [[] for i in range(nb_models)]\nfor i in range (nb_models):\n    nb_test_batches = len(test_loader[i])\n    model[i].eval()\n    test_vCDRs = [[] for i in range(nb_models)]\n    with torch.no_grad():\n        test_data = iter(test_loader[i])\n        for k in range(nb_test_batches):\n            # Loads data\n            imgs = test_data.next()\n            imgs = imgs.to(device)\n\n            # Forward pass\n            logits = model[i](imgs)\n\n            # Std out\n            print('Test iter {}/{}, model {}'.format(k+1, nb_test_batches, i) + ' '*50, \n              end='\\r')\n            \n            # Compute segmentation\n            pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            \n            # Compute and store vCDRs\n            pred_vCDR = vertical_cup_to_disc_ratio(pred_od.cpu().numpy(), pred_oc.cpu().numpy())\n            test_vCDRs[i] += pred_vCDR.tolist()\n                   \n    test_vCDRs[i]=np.array(test_vCDRs[i]).reshape(-1,1)\n    test_classif_preds[i] = clf.predict_proba(test_vCDRs[i])[:,1]\n\ntest_classif_preds_a = np.array(test_classif_preds)\nfinal_preds = np.mean(test_classif_preds_a, axis = 0)\n            \n    \n# Prepare and save .csv file\ndef create_submission_csv(prediction, submission_filename='/kaggle/working/submission.csv'):\n    \"\"\"Create a sumbission file in the appropriate format for evaluation.\n\n    :param\n    prediction: list of predictions (ex: [0.12720, 0.89289, ..., 0.29829])\n    \"\"\"\n    \n    with open(submission_filename, mode='w') as csv_file:\n        fieldnames = ['Id', 'Predicted']\n        writer = csv.DictWriter(csv_file, fieldnames=fieldnames)\n        writer.writeheader()\n\n        for i, p in enumerate(prediction):\n            writer.writerow({'Id': \"T{:04d}\".format(i+1), 'Predicted': '{:f}'.format(p)})\n\ncreate_submission_csv(final_preds)\n\n# The submission.csv file is under /kaggle/working/submission.csv.\n# If you want to submit it, you should download it before closing the current kernel.","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model 4","metadata":{}},{"cell_type":"code","source":"test_classif_preds = [[] for i in range(nb_models)]\nfor i in range (nb_models):\n    nb_test_batches = len(test_loader[i])\n    model[i].eval()\n    test_vCDRs = [[] for i in range(nb_models)]\n    with torch.no_grad():\n        test_data = iter(test_loader[i])\n        for k in range(nb_test_batches):\n            # Loads data\n            imgs = test_data.next()\n            imgs = imgs.to(device)\n\n            # Forward pass\n            logits = model[i](imgs)\n\n            # Std out\n            print('Test iter {}/{}, model {}'.format(k+1, nb_test_batches, i) + ' '*50, \n              end='\\r')\n            \n            # Compute segmentation\n            pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            \n            # Compute and store vCDRs\n            pred_vCDR = vertical_cup_to_disc_ratio(pred_od.cpu().numpy(), pred_oc.cpu().numpy())\n            test_vCDRs[i] += pred_vCDR.tolist()\n                   \n    test_vCDRs[i]=np.array(test_vCDRs[i]).reshape(-1,1)\n    test_classif_preds[i] = clf.predict_proba(test_vCDRs[i])[:,1]\n\ntest_classif_preds = np.array(test_classif_preds).T\nfinal_preds = clf_final.predict_proba(test_classif_preds)[:,1]\n            \n    \n# Prepare and save .csv file\ndef create_submission_csv(prediction, submission_filename='/kaggle/working/submission.csv'):\n    \"\"\"Create a sumbission file in the appropriate format for evaluation.\n\n    :param\n    prediction: list of predictions (ex: [0.12720, 0.89289, ..., 0.29829])\n    \"\"\"\n    \n    with open(submission_filename, mode='w') as csv_file:\n        fieldnames = ['Id', 'Predicted']\n        writer = csv.DictWriter(csv_file, fieldnames=fieldnames)\n        writer.writeheader()\n\n        for i, p in enumerate(prediction):\n            writer.writerow({'Id': \"T{:04d}\".format(i+1), 'Predicted': '{:f}'.format(p)})\n\ncreate_submission_csv(final_preds)\n\n# The submission.csv file is under /kaggle/working/submission.csv.\n# If you want to submit it, you should download it before closing the current kernel.","metadata":{},"execution_count":null,"outputs":[]}]}