{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Notebook for team 33","metadata":{}},{"cell_type":"markdown","source":"**Necessary imports :**","metadata":{}},{"cell_type":"code","source":"import os\nimport json\nimport csv\nimport random\nimport pickle\nimport cv2\nimport numpy as np\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torchvision.transforms as transforms\n\nfrom keras.preprocessing.image import load_img\nfrom keras.preprocessing.image import img_to_array\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.preprocessing.image import save_img\n\nfrom PIL import Image\nfrom torch.utils.data import Dataset, DataLoader\nfrom scipy.ndimage.measurements import label as label_function\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.svm import SVC\nfrom sklearn.svm import LinearSVC\nfrom sklearn.metrics import roc_auc_score, roc_curve\nfrom torchvision import datasets, transforms, models\n\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2021-05-22T20:15:46.32131Z","iopub.execute_input":"2021-05-22T20:15:46.321727Z","iopub.status.idle":"2021-05-22T20:15:53.814037Z","shell.execute_reply.started":"2021-05-22T20:15:46.321648Z","shell.execute_reply":"2021-05-22T20:15:53.813222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data augmentation :\nRealized only once to avoid too much computation on the kernel (no need to run the following cell, we import the augmented dataset from github).","metadata":{}},{"cell_type":"markdown","source":"```Python\nimport os\nimport cv2\nimport numpy as np\n\nfrom keras.preprocessing.image import load_img\nfrom keras.preprocessing.image import img_to_array\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.preprocessing.image import save_img\n\noriginal_dir = './original_dataset/'\naugmented_dir = './augmented_dataset/'\n\nfor filename in os.listdir(original_dir):\n    image = cv2.imread(original_dir+filename)\n    \n    if filename[0] == 'n':\n        cv2.imwrite(augmented_dir+filename,image)\n\n    # There are 360 n-images in the original dataset and only 40 g-images. We need to balance the classes :\n    if filename[0] == 'g':\n        # We will first add vertical flips of g-images to the dataset (we now have 80 images available):\n        flipped_image= cv2.flip(image, 0)\n\n        # We duplicate the original and flipped images to the augmented dataset, in order to reach balance in our training set :\n        cv2.imwrite(augmented_dir+filename.split(\".\")[0]+'_0.jpg',image)\n        cv2.imwrite(augmented_dir+filename.split(\".\")[0]+'_1.jpg',image)\n        cv2.imwrite(augmented_dir+filename.split(\".\")[0]+'_2.jpg',image)\n        cv2.imwrite(augmented_dir+filename.split(\".\")[0]+'_3.jpg',image)\n        cv2.imwrite(augmented_dir+filename.split(\".\")[0]+'_4.jpg',image)\n\n        cv2.imwrite(augmented_dir+filename.split(\".\")[0]+'_f0.jpg',flipped_image)\n        cv2.imwrite(augmented_dir+filename.split(\".\")[0]+'_f1.jpg',flipped_image)\n        cv2.imwrite(augmented_dir+filename.split(\".\")[0]+'_f2.jpg',flipped_image)\n        cv2.imwrite(augmented_dir+filename.split(\".\")[0]+'_f3.jpg',flipped_image)\n        # For each g-image in the original set, we now have 9 g-images in the augmented set, that's 360 images, the classes are well balanced.\n\n\nfor filename in os.listdir(augmented_dir):\n    img = load_img(augmented_dir+filename)\n    data = img_to_array(img)\n    samples = np.expand_dims(data, 0)\n    \n    # Image generator with appropriate transformations and ranges\n    datagen = ImageDataGenerator(brightness_range=[0.2,1.8], rotation_range=180)\n    iterator = datagen.flow(samples, batch_size=1)\n    # We go from 720 images to 3600 images in the augmented dataset (we create 4 new images with augmentation for each of the 720 firsts)\n    for i in range(4):\n        batch = iterator.next()\n        image = batch[0].astype('uint8')\n        # Save images to the augmented dataset.\n        save_img(\"./augmented_dataset/\"+filename.split(\".\")[0]+\"_t\"+str(i)+\".jpg\",image)\n ```","metadata":{}},{"cell_type":"markdown","source":"# Part 1 : training a classification model","metadata":{}},{"cell_type":"code","source":"class RefugeDataset(Dataset):\n\n    def __init__(self, root_dir, split='train', output_size=(299,299)): #The default size used by the inception model is 299*299 \n        # Define attributes\n        self.output_size = output_size\n        self.root_dir = root_dir\n        self.split = split\n        \n        if self.split == 'train':\n            self.labels = []\n            self.images = []\n            self.image_names = []\n            k=1\n            for filename in os.listdir(root_dir):\n                print('Loading {} image {}/{}...'.format('train', k, 3600), end='\\r')\n                img_name = os.path.join(root_dir, filename)\n                img = np.array(Image.open(img_name).convert('RGB'))\n                img = transforms.functional.to_tensor(img)\n                img = transforms.functional.resize(img, self.output_size, interpolation=Image.BILINEAR)\n                self.images.append(img)\n                \n                if filename[0] == 'g':\n                    self.labels.append(1)\n                if filename[0] == 'n':\n                    self.labels.append(0)\n                \n                self.image_names.append(filename)\n                k+=1\n                \n        \n        else :\n            with open(os.path.join(self.root_dir, self.split,'index.json')) as f:\n                self.index = json.load(f)\n\n            self.labels = []\n            self.images = []\n            for k in range(len(self.index)):\n                print('Loading {} image {}/{}...'.format(split, k, len(self.index)), end='\\r')\n                img_name = os.path.join(self.root_dir, self.split, 'images', self.index[str(k)]['ImgName'])\n                img = np.array(Image.open(img_name).convert('RGB'))\n                img = transforms.functional.to_tensor(img)\n                img = transforms.functional.resize(img, self.output_size, interpolation=Image.BILINEAR)\n                self.images.append(img)\n\n                if self.split != 'test':\n                    self.labels.append(torch.tensor(self.index[str(k)]['Label'], dtype=torch.long))\n\n\n            print('Succesfully loaded {} dataset.'.format(split) + ' '*50)\n            \n            \n    def __len__(self):\n        return len(self.images)\n\n    def __getitem__(self, idx):\n        # Image\n        img = self.images[idx]\n    \n        # Return only images for 'test' set\n        if self.split == 'test':\n            return img\n        \n        # Else, images and ground truth\n        if self.split == 'val':\n            # Label\n            lab = self.labels[idx]\n        \n            return img, lab, self.index[str(idx)]['ImgName']\n        \n        if self.split == 'train':\n            # Label\n            lab = self.labels[idx]\n        \n            return img, lab, self.image_names[idx]","metadata":{"execution":{"iopub.status.busy":"2021-05-20T23:45:22.305879Z","iopub.execute_input":"2021-05-20T23:45:22.306148Z","iopub.status.idle":"2021-05-20T23:45:22.322993Z","shell.execute_reply.started":"2021-05-20T23:45:22.30612Z","shell.execute_reply":"2021-05-20T23:45:22.322189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading the data :","metadata":{}},{"cell_type":"code","source":"root_dir_val = '/kaggle/input/eurecom-aml-2021-challenge-2/refuge_data/refuge_data/'\nroot_dir_train = '/kaggle/input/aml-challenge2-augmented-trainset/augmented_dataset/'\n\n# Datasets\ntrain_set = RefugeDataset(root_dir_train,\n                          split='train')\nval_set = RefugeDataset(root_dir_val, \n                        split='val')\ntest_set = RefugeDataset(root_dir_val, \n                         split='test')","metadata":{"execution":{"iopub.status.busy":"2021-05-21T21:10:18.219065Z","iopub.execute_input":"2021-05-21T21:10:18.219474Z","iopub.status.idle":"2021-05-21T21:10:18.312881Z","shell.execute_reply.started":"2021-05-21T21:10:18.219439Z","shell.execute_reply":"2021-05-21T21:10:18.310668Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 8\nnum_workers = 8\n\n# Dataloaders\ntrain_loader = DataLoader(train_set, \n                          batch_size=batch_size, \n                          shuffle=True, \n                          num_workers=num_workers,\n                          pin_memory=True,\n                          )\nval_loader = DataLoader(val_set, \n                        batch_size=batch_size, \n                        shuffle=False, \n                        num_workers=num_workers,\n                        pin_memory=True,\n                        )\ntest_loader = DataLoader(test_set, \n                        batch_size=batch_size, \n                        shuffle=False, \n                        num_workers=num_workers,\n                        pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-21T21:10:20.767605Z","iopub.execute_input":"2021-05-21T21:10:20.768022Z","iopub.status.idle":"2021-05-21T21:10:20.810133Z","shell.execute_reply.started":"2021-05-21T21:10:20.767985Z","shell.execute_reply":"2021-05-21T21:10:20.807658Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading a model :","metadata":{}},{"cell_type":"markdown","source":"### Keras inceptionV3","metadata":{}},{"cell_type":"code","source":"import keras\nimport tensorflow as tf\nfrom keras.applications.inception_v3 import InceptionV3\nfrom keras.preprocessing import image\nfrom keras.models import Model\nfrom keras.layers import Dense, Dropout, Flatten, GlobalAveragePooling2D\nfrom keras import backend as K\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.callbacks import TensorBoard","metadata":{"execution":{"iopub.status.busy":"2021-05-22T15:36:03.961684Z","iopub.execute_input":"2021-05-22T15:36:03.962066Z","iopub.status.idle":"2021-05-22T15:36:03.970718Z","shell.execute_reply.started":"2021-05-22T15:36:03.962034Z","shell.execute_reply":"2021-05-22T15:36:03.968696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() \n                                  else \"cpu\")\nmodel = InceptionV3(input_shape=(256,256,3),include_top=False)\nfor layer in model.layers[249:]:\n    layer.trainable = False\nx = model.output\nx = GlobalAveragePooling2D()(x)\n# let's add a fully-connected layer\nx = Dense(512,activation=\"relu\")(x)\nx = Dropout(0.2)(x)\npredictions = Dense(1, activation='sigmoid')(x)\nkeras_model = Model(inputs=model.input, outputs=predictions)\n#print(model)","metadata":{"execution":{"iopub.status.busy":"2021-05-21T21:10:25.199656Z","iopub.execute_input":"2021-05-21T21:10:25.200064Z","iopub.status.idle":"2021-05-21T21:10:29.087649Z","shell.execute_reply.started":"2021-05-21T21:10:25.200031Z","shell.execute_reply":"2021-05-21T21:10:29.08644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pwd","metadata":{"execution":{"iopub.status.busy":"2021-05-21T21:08:24.610589Z","iopub.execute_input":"2021-05-21T21:08:24.611038Z","iopub.status.idle":"2021-05-21T21:08:25.338558Z","shell.execute_reply.started":"2021-05-21T21:08:24.610997Z","shell.execute_reply":"2021-05-21T21:08:25.337104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T23:56:19.075358Z","iopub.execute_input":"2021-05-20T23:56:19.075667Z","iopub.status.idle":"2021-05-20T23:56:19.208556Z","shell.execute_reply.started":"2021-05-20T23:56:19.07564Z","shell.execute_reply":"2021-05-20T23:56:19.207749Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keras_model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:49:10.941046Z","iopub.execute_input":"2021-05-21T12:49:10.941368Z","iopub.status.idle":"2021-05-21T12:49:11.075647Z","shell.execute_reply.started":"2021-05-21T12:49:10.941336Z","shell.execute_reply":"2021-05-21T12:49:11.074791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Loading data for keras","metadata":{}},{"cell_type":"code","source":"def InitDataset(root_dir,mode):\n    \n    labels = []\n    filenames = []\n    ids = []\n    if mode == \"train\":\n        for filename in os.listdir(root_dir):\n            filenames += [root_dir + filename]\n            ids += [filename[:-4]]\n            if mode == \"train\":\n                if filename[0] == 'n':\n                    labels += [0] \n                else:\n                    labels += [1]\n    if mode == \"val\":\n        with open(os.path.join(root_dir,mode, 'index.json')) as f:\n                    index = json.load(f)       \n        for k in range(len(index)):\n                print('Loading {} image {}/{}...'.format(mode, k, len(index)), end='\\r')\n                filenames += [os.path.join(root_dir,mode,'images', index[str(k)]['ImgName'])]\n                ids += [str((index[str(k)]['ImgName'])[:-4])]\n                labels += [index[str(k)]['Label']]\n    if mode == \"test\":\n        with open(os.path.join(root_dir,mode, 'index.json')) as f:\n                    index = json.load(f)       \n        for k in range(len(index)):\n                print('Loading {} image {}/{}...'.format(mode, k, len(index)), end='\\r')\n                filenames += [os.path.join(root_dir,mode,'images', index[str(k)]['ImgName'])]\n                ids += [str((index[str(k)]['ImgName'])[:-4])]\n                labels += [1] #for the test data we put always 1 just to be able to use the built in methods for building datasets\n    print(ids[:3])\n    return tf.data.Dataset.from_tensor_slices((filenames,labels)),tf.data.Dataset.from_tensor_slices((ids,labels))\n\n","metadata":{"execution":{"iopub.status.busy":"2021-05-22T17:28:53.586223Z","iopub.execute_input":"2021-05-22T17:28:53.586561Z","iopub.status.idle":"2021-05-22T17:28:53.599347Z","shell.execute_reply.started":"2021-05-22T17:28:53.586532Z","shell.execute_reply":"2021-05-22T17:28:53.597846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def map_fn(path, label):\n    # path/label represent values for a single example\n    image = tf.image.decode_jpeg(tf.io.read_file(path),channels=3)\n    # some mapping to constant size - be careful with distorting aspec ratios\n    image = tf.image.resize(image, [256,256])\n    return image, label","metadata":{"execution":{"iopub.status.busy":"2021-05-22T17:28:54.542903Z","iopub.execute_input":"2021-05-22T17:28:54.543249Z","iopub.status.idle":"2021-05-22T17:28:54.55005Z","shell.execute_reply.started":"2021-05-22T17:28:54.54322Z","shell.execute_reply":"2021-05-22T17:28:54.548731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def configure_for_performance(ds):\n    ds = ds.cache()\n    #ds = ds.shuffle(buffer_size=1000)\n    ds = ds.batch(batch_size)\n    ds = ds.prefetch(buffer_size=tf.data.AUTOTUNE)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2021-05-22T17:28:54.952955Z","iopub.execute_input":"2021-05-22T17:28:54.953296Z","iopub.status.idle":"2021-05-22T17:28:54.958532Z","shell.execute_reply.started":"2021-05-22T17:28:54.953268Z","shell.execute_reply":"2021-05-22T17:28:54.957344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def CreateDataset(root_dir,mode):\n    dataset, dataset_ids = InitDataset(root_dir,mode)\n    dataset = dataset.map(map_fn,num_parallel_calls=tf.data.AUTOTUNE)\n    dataset= configure_for_performance(dataset)\n    dataset_ids = configure_for_performance(dataset_ids)\n    return dataset,dataset_ids","metadata":{"execution":{"iopub.status.busy":"2021-05-22T17:28:55.538457Z","iopub.execute_input":"2021-05-22T17:28:55.538865Z","iopub.status.idle":"2021-05-22T17:28:55.544767Z","shell.execute_reply.started":"2021-05-22T17:28:55.538795Z","shell.execute_reply":"2021-05-22T17:28:55.543308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds,_ = CreateDataset(root_dir_train,\"train\")\nval_ds,_ = CreateDataset(root_dir_val,\"val\")","metadata":{"execution":{"iopub.status.busy":"2021-05-22T15:44:51.062937Z","iopub.execute_input":"2021-05-22T15:44:51.06335Z","iopub.status.idle":"2021-05-22T15:44:51.092233Z","shell.execute_reply.started":"2021-05-22T15:44:51.063318Z","shell.execute_reply":"2021-05-22T15:44:51.090233Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for image, label in val_ds.take(1):\n#    print(\"Image shape: \", image.numpy().shape)\n#    print(\"Label: \", label.numpy())","metadata":{"execution":{"iopub.status.busy":"2021-05-21T00:02:55.416507Z","iopub.execute_input":"2021-05-21T00:02:55.416862Z","iopub.status.idle":"2021-05-21T00:02:55.422305Z","shell.execute_reply.started":"2021-05-21T00:02:55.416828Z","shell.execute_reply":"2021-05-21T00:02:55.421396Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keras_model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001), loss='binary_crossentropy', metrics=['accuracy',tf.keras.metrics.AUC()])","metadata":{"execution":{"iopub.status.busy":"2021-05-21T21:10:57.249546Z","iopub.execute_input":"2021-05-21T21:10:57.249986Z","iopub.status.idle":"2021-05-21T21:10:57.286574Z","shell.execute_reply.started":"2021-05-21T21:10:57.249943Z","shell.execute_reply":"2021-05-21T21:10:57.285507Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"callbacks = [\n    keras.callbacks.ModelCheckpoint(\n        # Path where to save the model\n        # The two parameters below mean that we will overwrite\n        # the current checkpoint if and only if\n        # the `val_loss` score has improved.\n        # The saved model name will include the current epoch.\n        filepath=\"mymodel_{epoch}\",\n        save_best_only=True,  # Only save a model if `val_loss` has improved.\n        monitor='val_auc',\n        mode='max',\n        verbose=1,\n    )\n]","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:25:00.605405Z","iopub.execute_input":"2021-05-21T12:25:00.605843Z","iopub.status.idle":"2021-05-21T12:25:00.61366Z","shell.execute_reply.started":"2021-05-21T12:25:00.605813Z","shell.execute_reply":"2021-05-21T12:25:00.609945Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 0\nepochs = 100\nval_auc = 0.","metadata":{"execution":{"iopub.status.busy":"2021-05-21T21:11:11.469231Z","iopub.execute_input":"2021-05-21T21:11:11.46966Z","iopub.status.idle":"2021-05-21T21:11:11.475119Z","shell.execute_reply.started":"2021-05-21T21:11:11.469606Z","shell.execute_reply":"2021-05-21T21:11:11.473785Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"while i < epochs:\n    i += 1\n    keras_model.fit(train_ds,epochs=1,validation_data=val_ds)\n    curr_auc = keras_model.evaluate(val_ds)[2]\n    if  curr_auc > val_auc:\n        val_auc = curr_auc\n        keras_model.save(\"mymodel\")\n        print(\"model saved ; val auc = \", val_auc)","metadata":{"execution":{"iopub.status.busy":"2021-05-21T21:11:13.687584Z","iopub.execute_input":"2021-05-21T21:11:13.688014Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"model save ; val auc = \", val_auc)\nkeras_model.evaluate(val_ds)","metadata":{"execution":{"iopub.status.busy":"2021-05-21T14:34:31.498012Z","iopub.execute_input":"2021-05-21T14:34:31.498453Z","iopub.status.idle":"2021-05-21T14:34:32.760975Z","shell.execute_reply.started":"2021-05-21T14:34:31.498419Z","shell.execute_reply":"2021-05-21T14:34:32.760222Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loaded_model = keras.models.load_model(\"/kaggle/input/inceptionmodelkeras/mymodel\")","metadata":{"execution":{"iopub.status.busy":"2021-05-22T15:03:09.003757Z","iopub.execute_input":"2021-05-22T15:03:09.00416Z","iopub.status.idle":"2021-05-22T15:03:39.123248Z","shell.execute_reply.started":"2021-05-22T15:03:09.004129Z","shell.execute_reply":"2021-05-22T15:03:39.122197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img, lbl = next(iter(val_ds))\nprint(loaded_model(img),lbl)","metadata":{"execution":{"iopub.status.busy":"2021-05-21T14:39:49.184888Z","iopub.execute_input":"2021-05-21T14:39:49.185231Z","iopub.status.idle":"2021-05-21T14:39:49.28556Z","shell.execute_reply.started":"2021-05-21T14:39:49.185198Z","shell.execute_reply":"2021-05-21T14:39:49.28458Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Pytorch resnet 50","metadata":{}},{"cell_type":"code","source":"\nmodel =models.inception_v3(pretrained=True)\n\n#for param in model.parameters():\n#    param.requires_grad = False\n    \nmodel.fc = nn.Sequential(nn.Linear(2048, 512),\n                                 nn.ReLU(),\n                                 nn.Dropout(0.2),\n                                 nn.Linear(512, 2),\n                                 nn.Softmax(dim=1))\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.fc.parameters(), lr=1e-3,weight_decay=0.15)\nmodel.to(device)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T23:40:39.889411Z","iopub.execute_input":"2021-05-20T23:40:39.889816Z","iopub.status.idle":"2021-05-20T23:40:41.858602Z","shell.execute_reply.started":"2021-05-20T23:40:39.889782Z","shell.execute_reply":"2021-05-20T23:40:41.857596Z"},"jupyter":{"source_hidden":true,"outputs_hidden":true},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training the model :","metadata":{}},{"cell_type":"code","source":"def classif_eval(classif_preds, classif_gts):\n    '''\n    Compute AUC classification score.\n    '''\n    auc = roc_auc_score(classif_gts, classif_preds)\n    return auc","metadata":{"execution":{"iopub.status.busy":"2021-05-22T20:30:03.905218Z","iopub.execute_input":"2021-05-22T20:30:03.905549Z","iopub.status.idle":"2021-05-22T20:30:03.910042Z","shell.execute_reply.started":"2021-05-22T20:30:03.905515Z","shell.execute_reply":"2021-05-22T20:30:03.908972Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_epoch = 100\n\n# Define parameters\nnb_train_batches = len(train_loader)\nnb_val_batches = len(val_loader)\nnb_iter = 0\nbest_val_auc = 0.\nepoch = 0\n\nwhile epoch < total_epoch:\n    # Accumulators\n    train_loss, val_loss = 0., 0.\n    \n    ############\n    # TRAINING #\n    ############\n    model.train()\n    train_data = iter(train_loader)\n    train_preds = []\n    train_gts = []\n    for k in range(nb_train_batches):\n        # Loads data\n        imgs, labels, names = train_data.next()\n        for label in labels :\n            if label == 1:\n                train_gts.append([0,1])\n            else :\n                train_gts.append([1,0])\n\n        imgs = imgs.to(device)\n        labels = labels.to(device)\n\n        # Forward pass\n        preds = model.forward(imgs)\n        train_preds += preds.cpu().detach().numpy().tolist()\n        loss = criterion(preds, labels)\n        \n        # Backward pass\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item() / nb_train_batches\n            \n        # Increase iterations\n        nb_iter += 1\n        \n        # Std out\n        print('Epoch {}, iter {}/{}, loss {:.6f}'.format(epoch+1, k+1, nb_train_batches, loss.item()) + ' '*20, end='\\r')\n    \n    train_preds = np.array(train_preds)\n    train_gts = np.array(train_gts)\n    train_auc = classif_eval(train_preds, train_gts)\n    \n    ##############\n    # VALIDATION #\n    ##############\n    model.eval()\n    with torch.no_grad():\n        val_data = iter(val_loader)\n        val_preds = []\n        val_gts = []\n        for k in range(nb_val_batches):\n            # Loads data\n            imgs, labels, names = val_data.next()\n            for label in labels :\n                if label == 1:\n                    val_gts.append([0,1])\n                else :\n                    val_gts.append([1,0])\n            imgs, labels = imgs.to(device), labels.to(device)\n\n            # Forward pass\n            preds = model.forward(imgs)\n            val_preds += preds.cpu().detach().numpy().tolist()\n            val_loss += criterion(preds, labels) / nb_val_batches\n\n            # Std out\n            print('Validation iter {}/{}'.format(k+1, nb_val_batches) + ' '*50, end='\\r')\n    \n    val_preds = np.array(val_preds)\n    val_gts = np.array(val_gts)\n    val_auc = classif_eval(val_preds, val_gts)\n        \n    # Validation results\n    print('VALIDATION epoch {}'.format(epoch+1)+' '*50)\n    print('LOSSES: {:.4f} (train), {:.4f} (val)'.format(train_loss, val_loss))\n    print('Classification (AUC): {:.4f} (train), {:.4f} (val)'.format(train_auc, val_auc))\n    \n    # Save model if best validation AUC is reached\n    if val_auc > best_val_auc:\n        torch.save(model, 'eyemodel.pth')\n        best_val_auc = val_auc\n        print('Best validation AUC reached. Saved model weights and classifier.')\n    print('_'*50)\n        \n    # End of epoch\n    epoch += 1","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-05-20T18:55:01.875926Z","iopub.execute_input":"2021-05-20T18:55:01.876276Z","iopub.status.idle":"2021-05-20T19:07:13.376404Z","shell.execute_reply.started":"2021-05-20T18:55:01.876244Z","shell.execute_reply":"2021-05-20T19:07:13.368716Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n = np.random.randint(0,len(train_gts) - 5)\nprint(n)\nprint(train_gts[n:n+5])\nprint(train_preds[n:n+5])","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Result model :","metadata":{}},{"cell_type":"code","source":"device = torch.device(\"cuda\")","metadata":{"execution":{"iopub.status.busy":"2021-05-20T15:51:43.204518Z","iopub.execute_input":"2021-05-20T15:51:43.204864Z","iopub.status.idle":"2021-05-20T15:51:43.211447Z","shell.execute_reply.started":"2021-05-20T15:51:43.204834Z","shell.execute_reply":"2021-05-20T15:51:43.210599Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model2 = torch.load(\"../input/modelinception/eyemodel(1).pth\")\nmodel2.to(device)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T15:52:11.225576Z","iopub.execute_input":"2021-05-20T15:52:11.225918Z","iopub.status.idle":"2021-05-20T15:52:19.981548Z","shell.execute_reply.started":"2021-05-20T15:52:11.225889Z","shell.execute_reply":"2021-05-20T15:52:19.980748Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check performance is maintained on validation :","metadata":{}},{"cell_type":"code","source":"model2.eval()\ncriterion = nn.CrossEntropyLoss()\nnb_val_batches = len(val_loader_TA)\nwith torch.no_grad():\n    val_loss = 0.\n    val_data = iter(val_loader_TA)\n    val_preds = []\n    val_gts = []\n    for k in range(nb_val_batches):\n        # Loads data\n        imgs, labels, seg, fov, names = val_data.next()\n        imgs, labels = imgs.to(device), labels.to(device)\n\n        # Forward pass\n        preds = model2(imgs)\n        val_preds += preds.cpu().numpy()[:,1].tolist()\n        labels = labels.long()\n        val_gts += labels.cpu().numpy().tolist()\n        val_loss += criterion(preds, labels) / nb_val_batches\n\n        # Std out\n        print('Validation iter {}/{}'.format(k+1, nb_val_batches) + ' '*50, end='\\r')\n\nval_preds = np.array(val_preds)\nval_gts = np.array(val_gts)\nval_auc = classif_eval(val_preds, val_gts)\n\n# Validation results\nprint('VALIDATION')\nprint('LOSS: {:.4f}'.format(val_loss))\nprint('Classification (AUC): {:.4f}'.format(val_auc))","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Optimizing the TA's model","metadata":{}},{"cell_type":"markdown","source":"### Dataset","metadata":{}},{"cell_type":"code","source":"class RefugeDataset_TA(Dataset):\n\n    def __init__(self, root_dir, split='train', output_size=(256,256)):\n        # Define attributes\n        self.output_size = output_size\n        self.root_dir = root_dir\n        self.split = split\n        \n        # Load data index\n        with open(os.path.join(self.root_dir, self.split, 'index.json')) as f:\n            self.index = json.load(f)\n            \n        self.images = []\n        for k in range(len(self.index)):\n            print('Loading {} image {}/{}...'.format(split, k, len(self.index)), end='\\r')\n            img_name = os.path.join(self.root_dir, self.split, 'images', self.index[str(k)]['ImgName'])\n            img = np.array(Image.open(img_name).convert('RGB'))\n            img = transforms.functional.to_tensor(img)\n            img = transforms.functional.resize(img, self.output_size, interpolation=Image.BILINEAR)\n            self.images.append(img)\n            \n        # Load ground truth for 'train' and 'val' sets\n        if split != 'test':\n            self.segs = []\n            for k in range(len(self.index)):\n                print('Loading {} segmentation {}/{}...'.format(split, k, len(self.index)), end='\\r')\n                seg_name = os.path.join(self.root_dir, self.split, 'gts', self.index[str(k)]['ImgName'].split('.')[0]+'.bmp')\n                seg = np.array(Image.open(seg_name)).copy()\n                seg = 255. - seg\n                od = (seg>=127.).astype(np.float32)\n                oc = (seg>=250.).astype(np.float32)\n                od = torch.from_numpy(od[None,:,:])\n                oc = torch.from_numpy(oc[None,:,:])\n                od = transforms.functional.resize(od, self.output_size, interpolation=Image.NEAREST)\n                oc = transforms.functional.resize(oc, self.output_size, interpolation=Image.NEAREST)\n                seg = torch.cat([od, oc], dim=0)\n                self.segs.append(seg)\n                \n        print('Succesfully loaded {} dataset.'.format(split) + ' '*50)\n            \n            \n    def __len__(self):\n        return len(self.index)\n\n    def __getitem__(self, idx):\n        # Image\n        img = self.images[idx]\n    \n        # Return only images for 'test' set\n        if self.split == 'test':\n            return img\n        \n        # Else, images and ground truth\n        else:\n            # Label\n            lab = torch.tensor(self.index[str(idx)]['Label'], dtype=torch.float32)\n\n            # Segmentation masks\n            seg = self.segs[idx]\n\n            # Fovea localization\n            f_x = self.index[str(idx)]['Fovea_X']\n            f_y = self.index[str(idx)]['Fovea_Y']\n            fov = torch.FloatTensor([f_x, f_y])\n        \n            return img, lab, seg, fov, self.index[str(idx)]['ImgName']","metadata":{"execution":{"iopub.status.busy":"2021-05-22T18:19:40.173691Z","iopub.execute_input":"2021-05-22T18:19:40.174064Z","iopub.status.idle":"2021-05-22T18:19:40.192136Z","shell.execute_reply.started":"2021-05-22T18:19:40.174034Z","shell.execute_reply":"2021-05-22T18:19:40.191167Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root_dir_TA = '../input/eurecom-aml-2021-challenge-2/refuge_data/refuge_data'\nbatch_size = 8\nnum_workers = 8\n\n# Datasets\ntrain_set_TA = RefugeDataset_TA(root_dir_TA, \n                          split='train')\nval_set_TA = RefugeDataset_TA(root_dir_TA, \n                        split='val')\ntest_set_TA = RefugeDataset_TA(root_dir_TA, \n                         split='test')\n\n# Dataloaders\ntrain_loader_TA = DataLoader(train_set_TA, \n                          batch_size=batch_size, \n                          shuffle=True, \n                          num_workers=num_workers,\n                          pin_memory=True,\n                         )\nval_loader_TA = DataLoader(val_set_TA, \n                        batch_size=batch_size, \n                        shuffle=False, \n                        num_workers=num_workers,\n                        pin_memory=True,\n                        )\ntest_loader_TA = DataLoader(test_set_TA, \n                        batch_size=batch_size, \n                        shuffle=False, \n                        num_workers=num_workers,\n                        pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T15:27:19.302783Z","iopub.execute_input":"2021-05-22T15:27:19.303172Z","iopub.status.idle":"2021-05-22T15:30:21.059191Z","shell.execute_reply.started":"2021-05-22T15:27:19.303142Z","shell.execute_reply":"2021-05-22T15:30:21.057906Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Metrics","metadata":{}},{"cell_type":"code","source":"EPS = 1e-7\n\ndef compute_dice_coef(input, target):\n    '''\n    Compute dice score metric.\n    '''\n    batch_size = input.shape[0]\n    return sum([dice_coef_sample(input[k,:,:], target[k,:,:]) for k in range(batch_size)])/batch_size\n\ndef dice_coef_sample(input, target):\n    iflat = input.contiguous().view(-1)\n    tflat = target.contiguous().view(-1)\n    intersection = (iflat * tflat).sum()\n    return (2. * intersection) / (iflat.sum() + tflat.sum())\n\n\ndef vertical_diameter(binary_segmentation):\n    '''\n    Get the vertical diameter from a binary segmentation.\n    The vertical diameter is defined as the \"fattest\" area of the binary_segmentation parameter.\n    '''\n\n    # get the sum of the pixels in the vertical axis\n    vertical_axis_diameter = np.sum(binary_segmentation, axis=1)\n\n    # pick the maximum value\n    diameter = np.max(vertical_axis_diameter, axis=1)\n\n    # return it\n    return diameter\n\n\n\ndef vertical_cup_to_disc_ratio(od, oc):\n    '''\n    Compute the vertical cup-to-disc ratio from a given labelling map.\n    '''\n    # compute the cup diameter\n    cup_diameter = vertical_diameter(oc)\n    # compute the disc diameter\n    disc_diameter = vertical_diameter(od)\n\n    return cup_diameter / (disc_diameter + EPS)\n\ndef compute_vCDR_error(pred_od, pred_oc, gt_od, gt_oc):\n    '''\n    Compute vCDR prediction error, along with predicted vCDR and ground truth vCDR.\n    '''\n    pred_vCDR = vertical_cup_to_disc_ratio(pred_od, pred_oc)\n    gt_vCDR = vertical_cup_to_disc_ratio(gt_od, gt_oc)\n    vCDR_err = np.mean(np.abs(gt_vCDR - pred_vCDR))\n    return vCDR_err, pred_vCDR, gt_vCDR\n\n\ndef fov_error(pred_fov, gt_fov):\n    '''\n    Fovea localization error metric (mean root squared error).\n    '''\n    err = np.sqrt(np.sum((gt_fov-pred_fov)**2, axis=1)).mean()\n    return err","metadata":{"execution":{"iopub.status.busy":"2021-05-20T15:59:24.125199Z","iopub.execute_input":"2021-05-20T15:59:24.125711Z","iopub.status.idle":"2021-05-20T15:59:24.136156Z","shell.execute_reply.started":"2021-05-20T15:59:24.125672Z","shell.execute_reply":"2021-05-20T15:59:24.135293Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Post processing function","metadata":{}},{"cell_type":"code","source":"def refine_seg(pred):\n    '''\n    Only retain the biggest connected component of a segmentation map.\n    '''\n    np_pred = pred.numpy()\n        \n    largest_ccs = []\n    for i in range(np_pred.shape[0]):\n        labeled, ncomponents = label_function(np_pred[i,:,:])\n        bincounts = np.bincount(labeled.flat)[1:]\n        if len(bincounts) == 0:\n            largest_cc = labeled == 0\n        else:\n            largest_cc = labeled == np.argmax(bincounts)+1\n        largest_cc = torch.tensor(largest_cc, dtype=torch.float32)\n        largest_ccs.append(largest_cc)\n    largest_ccs = torch.stack(largest_ccs)\n    \n    return largest_ccs","metadata":{"execution":{"iopub.status.busy":"2021-05-20T15:59:24.137794Z","iopub.execute_input":"2021-05-20T15:59:24.138156Z","iopub.status.idle":"2021-05-20T15:59:24.151622Z","shell.execute_reply.started":"2021-05-20T15:59:24.138109Z","shell.execute_reply":"2021-05-20T15:59:24.150712Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Network","metadata":{}},{"cell_type":"code","source":"class UNet(nn.Module):\n    def __init__(self, n_channels=3, n_classes=2):\n        super(UNet, self).__init__()\n        self.n_channels = n_channels\n        self.n_classes = n_classes\n        self.epoch = 0\n\n        self.inc = DoubleConv(n_channels, 64)\n        self.down1 = Down(64, 128)\n        self.down2 = Down(128, 256)\n        self.down3 = Down(256, 512)\n        self.down4 = Down(512, 1024)\n        factor = 2 \n        self.down5 = Down(1024, 2048 // factor)\n        \n        self.up1 = Up(2048, 1024 // factor)\n        self.up2 = Up(1024, 512 // factor)\n        self.up3 = Up(512, 256 // factor)\n        self.up4 = Up(256, 128 // factor)\n        self.up5 = Up(128, 64)\n\n        self.output_layer = OutConv(64, n_classes)\n\n    def forward(self, x):\n        x1 = self.inc(x)\n        x2 = self.down1(x1)\n        x3 = self.down2(x2)\n        x4 = self.down3(x3)\n        x5 = self.down4(x4)\n        x6 = self.down5(x5)\n        \n        out = self.up1(x6, x5)\n        out = self.up2(out, x4)\n        out = self.up3(out, x3)\n        out = self.up4(out, x2)\n        out = self.up5(out, x1)\n        out = self.output_layer(out)\n        out = torch.sigmoid(out)\n        return out\n\n    \nclass DoubleConv(nn.Module):\n    \"\"\"(convolution => [BN] => ReLU) * 2\"\"\"\n\n    def __init__(self, in_channels, out_channels, mid_channels=None):\n        super().__init__()\n        if not mid_channels:\n            mid_channels = out_channels\n        self.double_conv = nn.Sequential(\n            nn.Conv2d(in_channels, mid_channels, kernel_size=3, padding=1),\n            nn.BatchNorm2d(mid_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(mid_channels, out_channels, kernel_size=3, padding=1),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True)\n        )\n\n    def forward(self, x):\n        return self.double_conv(x)\n\n\nclass Down(nn.Module):\n    \"\"\"Downscaling with maxpool then double conv\"\"\"\n\n    def __init__(self, in_channels, out_channels):\n        super().__init__()\n        self.maxpool_conv = nn.Sequential(\n            nn.MaxPool2d(2),\n            DoubleConv(in_channels, out_channels)\n        )\n\n    def forward(self, x):\n        return self.maxpool_conv(x)\n\n\nclass Up(nn.Module):\n    \"\"\"Upscaling then double conv\"\"\"\n\n    def __init__(self, in_channels, out_channels):\n        super().__init__()\n\n        # Use the normal convolutions to reduce the number of channels\n        self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)\n        self.conv = DoubleConv(in_channels, out_channels, in_channels // 2)\n\n\n    def forward(self, x1, x2):\n        x1 = self.up(x1)\n        # input is CHW\n        diffY = x2.size()[2] - x1.size()[2]\n        diffX = x2.size()[3] - x1.size()[3]\n\n        x1 = F.pad(x1, [diffX // 2, diffX - diffX // 2,\n                        diffY // 2, diffY - diffY // 2])\n        x = torch.cat([x2, x1], dim=1)\n        return self.conv(x)\n\n\nclass OutConv(nn.Module):\n    '''\n    Simple convolution.\n    '''\n    def __init__(self, in_channels, out_channels):\n        super(OutConv, self).__init__()\n        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=1)\n\n    def forward(self, x):\n        return self.conv(x)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T15:59:24.153117Z","iopub.execute_input":"2021-05-20T15:59:24.153616Z","iopub.status.idle":"2021-05-20T15:59:24.175979Z","shell.execute_reply.started":"2021-05-20T15:59:24.153582Z","shell.execute_reply":"2021-05-20T15:59:24.175059Z"},"jupyter":{"source_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Settings","metadata":{}},{"cell_type":"code","source":"lr = 1e-4\nweight_decay = 0.15\ntotal_epoch = 70","metadata":{"execution":{"iopub.status.busy":"2021-05-20T15:59:24.17724Z","iopub.execute_input":"2021-05-20T15:59:24.177645Z","iopub.status.idle":"2021-05-20T15:59:24.187277Z","shell.execute_reply.started":"2021-05-20T15:59:24.177602Z","shell.execute_reply":"2021-05-20T15:59:24.186409Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Device, model, loss and optimizer","metadata":{}},{"cell_type":"code","source":"# Device\n#device = torch.device(\"cuda:0\")\n\n# Network\nmodel_TA = UNet(n_channels=3, n_classes=2).to(device)\n\n# Loss\nseg_loss = torch.nn.BCELoss(reduction='mean')\n\n# Optimizer\noptimizer_TA = optim.Adam(model_TA.parameters(), lr=lr, weight_decay=weight_decay)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:01:23.640237Z","iopub.execute_input":"2021-05-20T16:01:23.64058Z","iopub.status.idle":"2021-05-20T16:01:24.24051Z","shell.execute_reply.started":"2021-05-20T16:01:23.640529Z","shell.execute_reply":"2021-05-20T16:01:24.239672Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train for OC/OD segmentation","metadata":{}},{"cell_type":"code","source":"# Define parameters\nnb_train_batches = len(train_loader_TA)\nnb_val_batches = len(val_loader_TA)\nnb_iter = 0\n#best_val_auc = 0.\n\nwhile model_TA.epoch < total_epoch:\n    # Accumulators\n    train_vCDRs, val_vCDRs, train_model2_preds, val_model2_preds = [], [], [], []\n    train_classif_gts, val_classif_gts = [], []\n    train_loss, val_loss = 0., 0.\n    train_dsc_od, val_dsc_od = 0., 0.\n    train_dsc_oc, val_dsc_oc = 0., 0.\n    train_vCDR_error, val_vCDR_error = 0., 0.\n    \n    ############\n    # TRAINING #\n    ############\n    model_TA.train()\n    train_data_TA = iter(train_loader_TA)\n    for k in range(nb_train_batches):\n        # Loads data\n        imgs, classif_gts, seg_gts, fov_coords, names = train_data_TA.next()\n        imgs, classif_gts, seg_gts = imgs.to(device), classif_gts.to(device), seg_gts.to(device)\n\n        # Forward pass\n        logits = model_TA(imgs)\n        loss = seg_loss(logits, seg_gts)\n \n        # Backward pass\n        optimizer_TA.zero_grad()\n        loss.backward()\n        optimizer_TA.step()\n        train_loss += loss.item() / nb_train_batches\n        \n        with torch.no_grad():\n            # Compute segmentation metric\n            pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            gt_od = seg_gts[:,0,:,:].type(torch.int8)\n            gt_oc = seg_gts[:,1,:,:].type(torch.int8)\n            dsc_od = compute_dice_coef(pred_od, gt_od)\n            dsc_oc = compute_dice_coef(pred_oc, gt_oc)\n            train_dsc_od += dsc_od.item()/nb_train_batches\n            train_dsc_oc += dsc_oc.item()/nb_train_batches\n\n\n            # Compute and store vCDRs\n            vCDR_error, pred_vCDR, gt_vCDR = compute_vCDR_error(pred_od.cpu().numpy(), pred_oc.cpu().numpy(), gt_od.cpu().numpy(), gt_oc.cpu().numpy())\n            train_vCDRs += pred_vCDR.tolist()\n            train_vCDR_error += vCDR_error / nb_train_batches\n            train_classif_gts += classif_gts.cpu().numpy().tolist()\n            \n            train_model2_preds += model2(imgs).cpu().numpy()[:,1].tolist()\n\n            \n        # Increase iterations\n        nb_iter += 1\n        \n        # Std out\n        print('Epoch {}, iter {}/{}, loss {:.6f}'.format(model_TA.epoch+1, k+1, nb_train_batches, loss.item()) + ' '*20, \n              end='\\r')\n  \n    # Train a logistic regression on vCDRs\n    train_vCDRs = np.array(train_vCDRs).reshape(-1,1)\n    train_model2_preds = np.array(train_model2_preds).reshape(-1,1)\n\n    train_inputs = np.concatenate((train_vCDRs, train_model2_preds), axis=1)\n    #print(train_inputs)\n    \n    train_classif_gts = np.array(train_classif_gts)\n    \n    clf = SVC(random_state=0, probability=True).fit(train_inputs, train_classif_gts)\n    train_classif_preds = clf.predict_proba(train_inputs)[:,1]\n    train_auc = classif_eval(train_classif_preds, train_classif_gts)\n    \n    ##############\n    # VALIDATION #\n    ##############\n    model_TA.eval()\n    with torch.no_grad():\n        val_data = iter(val_loader_TA)\n        for k in range(nb_val_batches):\n            # Loads data\n            imgs, classif_gts, seg_gts, fov_coords, names = val_data.next()\n            imgs, classif_gts, seg_gts = imgs.to(device), classif_gts.to(device), seg_gts.to(device)\n\n            # Forward pass\n            logits = model_TA(imgs)\n            val_loss += seg_loss(logits, seg_gts).item() / nb_val_batches\n\n            # Std out\n            print('Validation iter {}/{}'.format(k+1, nb_val_batches) + ' '*50, \n                  end='\\r')\n            \n            # Compute segmen numpy.concatenate((a1, a2, ...), axis=0, out=None, dtype=None, casting=\"same_kind\")¶tation metric\n            pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n            gt_od = seg_gts[:,0,:,:].type(torch.int8)\n            gt_oc = seg_gts[:,1,:,:].type(torch.int8)\n            dsc_od = compute_dice_coef(pred_od, gt_od)\n            dsc_oc = compute_dice_coef(pred_oc, gt_oc)\n            val_dsc_od += dsc_od.item()/nb_val_batches\n            val_dsc_oc += dsc_oc.item()/nb_val_batches\n            \n            # Compute and store vCDRs\n            vCDR_error, pred_vCDR, gt_vCDR = compute_vCDR_error(pred_od.cpu().numpy(), pred_oc.cpu().numpy(), gt_od.cpu().numpy(), gt_oc.cpu().numpy())\n            val_vCDRs += pred_vCDR.tolist()\n            val_vCDR_error += vCDR_error / nb_val_batches\n            val_classif_gts += classif_gts.cpu().numpy().tolist()\n            \n            val_model2_preds += model2(imgs).cpu().numpy()[:,1].tolist()\n\n\n            \n\n    # Glaucoma predictions from vCDRs\n    val_vCDRs = np.array(val_vCDRs).reshape(-1,1)\n    val_model2_preds = np.array(val_model2_preds).reshape(-1,1)\n\n    val_inputs = np.concatenate((val_vCDRs, val_model2_preds), axis=1)    \n    \n    val_classif_gts = np.array(val_classif_gts)\n    val_classif_preds = clf.predict_proba(val_inputs)[:,1]\n    val_auc = classif_eval(val_classif_preds, val_classif_gts)\n    \n    # Validation results\n    print('VALIDATION epoch {}'.format(model_TA.epoch+1)+' '*50)\n    print('LOSSES: {:.4f} (train), {:.4f} (val)'.format(train_loss, val_loss))\n    print('OD segmentation (Dice Score): {:.4f} (train), {:.4f} (val)'.format(train_dsc_od, val_dsc_od))\n    print('OC segmentation (Dice Score): {:.4f} (train), {:.4f} (val)'.format(train_dsc_oc, val_dsc_oc))\n    print('vCDR error: {:.4f} (train), {:.4f} (val)'.format(train_vCDR_error, val_vCDR_error))\n    print('Classification (AUC): {:.4f} (train), {:.4f} (val)'.format(train_auc, val_auc))\n    \n    # Save model_TA if best validation AUC is reached\n    if val_auc > best_val_auc:\n        torch.save(model_TA.state_dict(), '/kaggle/working/best_AUC_weights.pth')\n        with open('/kaggle/working/best_AUC_classifier.pkl', 'wb') as clf_file:\n            pickle.dump(clf, clf_file)\n        best_val_auc = val_auc\n        print('Best validation AUC reached. Saved model_TA weights and classifier.')\n    print('_'*50)\n        \n    # End of epoch\n    model_TA.epoch += 1\n        \n\n","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:01:24.791352Z","iopub.execute_input":"2021-05-20T16:01:24.791678Z","iopub.status.idle":"2021-05-20T16:17:58.534263Z","shell.execute_reply.started":"2021-05-20T16:01:24.791646Z","shell.execute_reply":"2021-05-20T16:17:58.532311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs.shape","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:24:05.096133Z","iopub.execute_input":"2021-05-20T16:24:05.096491Z","iopub.status.idle":"2021-05-20T16:24:05.102938Z","shell.execute_reply.started":"2021-05-20T16:24:05.096455Z","shell.execute_reply":"2021-05-20T16:24:05.101907Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load best segmentation model + classifier","metadata":{}},{"cell_type":"code","source":"# Load model and classifier\nmodel_TA_loaded = UNet(n_channels=3, n_classes=2).to(device)\nmodel_TA_loaded.load_state_dict(torch.load('/kaggle/working/best_AUC_weights.pth'))\nwith open('/kaggle/working/best_AUC_classifier.pkl', 'rb') as clf_file:\n    clf_TA_loaded = pickle.load(clf_file)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"exp_data = iter(val_loader_TA)\n","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs_TA, classif_gts, seg_gts, fov_coords, names_TA = exp_data_TA.next()\nprint(imgs_TA.shape,imgs.shape, imgs.dtype,imgs_TA.dtype)","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs,labels, _, _, names = exp_data.next()\nimgs, labels = imgs.to(device), labels.to(device)\nlogits = model_TA_loaded(imgs)\n# Compute segmentation\npred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\npred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n\n# Compute and store vCDRs\npred_vCDR = vertical_cup_to_disc_ratio(pred_od.cpu().numpy(), pred_oc.cpu().numpy())\ntest_vCDRs = pred_vCDR.tolist()\n\ntest_vCDRs = np.array(test_vCDRs).reshape(-1,1)\ntest_classif_preds = clf_TA_loaded.predict_proba(test_vCDRs)\npreds = model2(imgs)\nprint(preds, test_classif_preds)\nprint(labels)","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = model2(imgs)\nval_gts = []\nfor label in labels.cpu().detach().numpy().tolist() :\n    if label == 1:\n        val_gts.append([0,1])\n    else :\n        val_gts.append([1,0])\n","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels.cpu().detach()","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(s(100*preds),s(preds))","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"roc_auc_score(np.array(val_gts),np.array(s(preds).cpu().detach().numpy().tolist()))","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_TA_loaded.eval()\nval_vCDRs = []\nval_classif_gts = []\nval_loss = 0.\nval_dsc_od = 0.\nval_dsc_oc = 0.\nval_vCDR_error = 0.\nwith torch.no_grad():\n    val_data_TA = iter(val_loader_TA)\n    for k in range(nb_val_batches):\n        # Loads data\n        imgs, classif_gts, seg_gts, fov_coords, names = val_data_TA.next()\n        imgs, classif_gts, seg_gts = imgs.to(device), classif_gts.to(device), seg_gts.to(device)\n\n        # Forward pass\n        logits = model_TA_loaded(imgs)\n        val_loss += seg_loss(logits, seg_gts).item() / nb_val_batches\n\n        # Std out\n        print('Validation iter {}/{}'.format(k+1, nb_val_batches) + ' '*50, \n              end='\\r')\n\n        # Compute segmentation metric\n        pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n        pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n        gt_od = seg_gts[:,0,:,:].type(torch.int8)\n        gt_oc = seg_gts[:,1,:,:].type(torch.int8)\n        dsc_od = compute_dice_coef(pred_od, gt_od)\n        dsc_oc = compute_dice_coef(pred_oc, gt_oc)\n        val_dsc_od += dsc_od.item()/nb_val_batches\n        val_dsc_oc += dsc_oc.item()/nb_val_batches\n\n        # Compute and store vCDRs\n        vCDR_error, pred_vCDR, gt_vCDR = compute_vCDR_error(pred_od.cpu().numpy(), pred_oc.cpu().numpy(), gt_od.cpu().numpy(), gt_oc.cpu().numpy())\n        val_vCDRs += pred_vCDR.tolist()\n        val_vCDR_error += vCDR_error / nb_val_batches\n        val_classif_gts += classif_gts.cpu().numpy().tolist()\n\n\n# Glaucoma predictions from vCDRs\nval_vCDRs = np.array(val_vCDRs).reshape(-1,1)\nval_classif_gts = np.array(val_classif_gts)\nval_classif_preds = clf_TA_loaded.predict_proba(val_vCDRs)[:,1]\nval_auc = classif_eval(val_classif_preds, val_classif_gts)\n\n# Validation results\nprint('VALIDATION '+' '*50)\nprint('LOSSES: {:.4f} (val)'.format(val_loss))\nprint('OD segmentation (Dice Score): {:.4f} (val)'.format(val_dsc_od))\nprint('OC segmentation (Dice Score): {:.4f} (val)'.format(val_dsc_oc))\nprint('vCDR error: {:.4f} (val)'.format(val_vCDR_error))\nprint('Classification (AUC): {:.4f} (val)'.format(val_auc))","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Making an voting classifier with our models","metadata":{}},{"cell_type":"code","source":"def VotingClassifier():\n    def __init__(end_to_end_model,segmentation_model,classifier,w1 = 0.5, w2 = 0.5): # w1 + w2 should be equal to 1 or be int values\n        self.ete_model = end_to_end_model\n        self.seg = segmentation_model\n        self.clf = classifier\n        self.w1 = w1/(w1 + w2)\n        self.w2 = w2/(w1 + w2)\n    \n    def predict(imgs):\n        preds_ete = self.ete_model(imgs)\n        \n        logits = self.seg(imgs)\n        # Compute segmentation\n        pred_od = refine_seg((logits[:,0,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n        pred_oc = refine_seg((logits[:,1,:,:]>=0.5).type(torch.int8).cpu()).to(device)\n\n        # Compute and store vCDRs\n        pred_vCDR = vertical_cup_to_disc_ratio(pred_od.cpu().numpy(), pred_oc.cpu().numpy())\n        test_vCDRs = pred_vCDR.tolist()\n\n        test_vCDRs = np.array(test_vCDRs).reshape(-1,1)\n        test_classif_preds = self.clf.predict_proba(test_vCDRs)\n        return self.w1*preds_ete + self.w2*test_clasif_preds","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vote = VotingClassifier(model2,model_TA_loaded,clf_TA_loaded)","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create inception predictions csv","metadata":{}},{"cell_type":"code","source":"#exp_iter = iter(train_loader_TA)\ntrain_ds_imgs, train_ds_ids = CreateDataset(\"/kaggle/input/eurecom-aml-2021-challenge-2/refuge_data/refuge_data/train/images/\",mode= \"train\")\nval_ds_imgs, val_ds_ids = CreateDataset(\"/kaggle/input/eurecom-aml-2021-challenge-2/refuge_data/refuge_data/\",mode= \"val\")\ntest_ds_imgs, test_ds_ids = CreateDataset(\"/kaggle/input/eurecom-aml-2021-challenge-2/refuge_data/refuge_data/\", mode = \"test\")","metadata":{"execution":{"iopub.status.busy":"2021-05-22T17:32:33.010912Z","iopub.execute_input":"2021-05-22T17:32:33.011284Z","iopub.status.idle":"2021-05-22T17:32:33.256419Z","shell.execute_reply.started":"2021-05-22T17:32:33.011254Z","shell.execute_reply":"2021-05-22T17:32:33.255358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iterator1 = iter(val_ds_imgs)\niterator2 = iter(val_ds_ids)\n\nnb_batches = 400//8\npreds_list= []\nids_list= []\nlbls_list= []\nfor k in range(nb_batches):\n    imgs, lbls = next(iterator1)\n    ids, _ = next(iterator2)\n    lbls_list += lbls.numpy().tolist()\n    preds_list += loaded_model(imgs).numpy().reshape(-1).tolist()\n    ids_list += [str(p)[2:-1] for p in ids.numpy().tolist()]","metadata":{"execution":{"iopub.status.busy":"2021-05-22T17:35:45.740014Z","iopub.execute_input":"2021-05-22T17:35:45.740434Z","iopub.status.idle":"2021-05-22T17:35:55.767529Z","shell.execute_reply.started":"2021-05-22T17:35:45.740386Z","shell.execute_reply":"2021-05-22T17:35:55.766481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_keras_prediction_csv(predictions, ids, lbls=None, submission_filename='./submission_test.csv'):\n    \"\"\"Create a sumbission file in the appropriate format for evaluation.\n\n    :param\n    prediction: list of predictions (ex: [0.12720, 0.89289, ..., 0.29829])\n    \"\"\"\n    if lbls != None:\n        with open(submission_filename, mode='w') as csv_file:\n            fieldnames = ['Id', 'predictions', 'labels']\n            writer = csv.DictWriter(csv_file, fieldnames=fieldnames)\n            writer.writeheader()\n\n            for i,p in enumerate(predictions):\n                writer.writerow({'Id': ids[i], 'predictions': '{:f}'.format(p), 'labels' : '{:.1f}'.format(lbls[i])})\n    else:\n        with open(submission_filename, mode='w') as csv_file:\n            fieldnames = ['Id', 'predictions']\n            writer = csv.DictWriter(csv_file, fieldnames=fieldnames)\n            writer.writeheader()\n\n            for i,p in enumerate(predictions):\n                writer.writerow({'Id': ids[i], 'predictions': '{:f}'.format(p)})","metadata":{"execution":{"iopub.status.busy":"2021-05-22T17:33:24.515366Z","iopub.status.idle":"2021-05-22T17:33:24.516035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_keras_prediction_csv(preds_list,ids_list,lbls=lbls_list,submission_filename=\"./predictions_val.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-05-22T17:37:17.025172Z","iopub.execute_input":"2021-05-22T17:37:17.025532Z","iopub.status.idle":"2021-05-22T17:37:17.03404Z","shell.execute_reply.started":"2021-05-22T17:37:17.025502Z","shell.execute_reply":"2021-05-22T17:37:17.033008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load CSV and train classifiers","metadata":{}},{"cell_type":"code","source":"import pandas as pd","metadata":{"execution":{"iopub.status.busy":"2021-05-22T20:16:03.875296Z","iopub.execute_input":"2021-05-22T20:16:03.875646Z","iopub.status.idle":"2021-05-22T20:16:03.879874Z","shell.execute_reply.started":"2021-05-22T20:16:03.875615Z","shell.execute_reply":"2021-05-22T20:16:03.878785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_preds_df = pd.read_csv(\"/kaggle/input/predictionscsv/predictions_train.csv\")\nval_preds_df = pd.read_csv(\"/kaggle/input/predictionscsv/predictions_val.csv\")\ntest_preds_df = pd.read_csv(\"/kaggle/input/predictionscsv/predictions_test.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-05-22T20:28:17.474135Z","iopub.execute_input":"2021-05-22T20:28:17.474484Z","iopub.status.idle":"2021-05-22T20:28:17.499601Z","shell.execute_reply.started":"2021-05-22T20:28:17.474452Z","shell.execute_reply":"2021-05-22T20:28:17.498791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_preds_df['Id']= train_preds_df['Id'].astype(\"string\")\nval_preds_df['Id']= val_preds_df['Id'].astype(\"string\")\ntest_preds_df['Id']= test_preds_df['Id'].astype(\"string\")","metadata":{"execution":{"iopub.status.busy":"2021-05-22T20:29:38.610931Z","iopub.execute_input":"2021-05-22T20:29:38.611282Z","iopub.status.idle":"2021-05-22T20:29:38.62127Z","shell.execute_reply.started":"2021-05-22T20:29:38.611246Z","shell.execute_reply":"2021-05-22T20:29:38.620399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_preds_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-22T20:45:14.989125Z","iopub.execute_input":"2021-05-22T20:45:14.989473Z","iopub.status.idle":"2021-05-22T20:45:14.999583Z","shell.execute_reply.started":"2021-05-22T20:45:14.989433Z","shell.execute_reply":"2021-05-22T20:45:14.998814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classif_eval(val_preds_df.predictions,val_preds_df.labels) # Verif that the auc is what we expected","metadata":{"execution":{"iopub.status.busy":"2021-05-22T20:33:34.704976Z","iopub.execute_input":"2021-05-22T20:33:34.705286Z","iopub.status.idle":"2021-05-22T20:33:34.713046Z","shell.execute_reply.started":"2021-05-22T20:33:34.705257Z","shell.execute_reply":"2021-05-22T20:33:34.712226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_vCDR_df = pd.read_csv(\"/kaggle/input/vcdr-csv-dataset/vCDRs_train.csv\")\nval_vCDR_df = pd.read_csv(\"/kaggle/input/vcdr-csv-dataset/vCDRs_val.csv\")\ntest_vCDR_df = pd.read_csv(\"/kaggle/input/vcdr-csv-dataset/vCDRs_test.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-05-22T22:02:18.848043Z","iopub.execute_input":"2021-05-22T22:02:18.848377Z","iopub.status.idle":"2021-05-22T22:02:18.874524Z","shell.execute_reply.started":"2021-05-22T22:02:18.848347Z","shell.execute_reply":"2021-05-22T22:02:18.873748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_vCDR_df['Id']= train_vCDR_df['Id'].astype(\"string\")\nval_vCDR_df['Id']= val_vCDR_df['Id'].astype(\"string\")\ntest_vCDR_df['Id']= test_vCDR_df['Id'].astype(\"string\")","metadata":{"execution":{"iopub.status.busy":"2021-05-22T22:02:20.69186Z","iopub.execute_input":"2021-05-22T22:02:20.692203Z","iopub.status.idle":"2021-05-22T22:02:20.699033Z","shell.execute_reply.started":"2021-05-22T22:02:20.692172Z","shell.execute_reply":"2021-05-22T22:02:20.698221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_vCDR_df_nolbl =train_vCDR_df.drop(columns=['labels'])\nval_vCDR_df_nolbl = val_vCDR_df.drop(columns=['labels'])","metadata":{"execution":{"iopub.status.busy":"2021-05-22T22:02:23.727662Z","iopub.execute_input":"2021-05-22T22:02:23.728008Z","iopub.status.idle":"2021-05-22T22:02:23.734099Z","shell.execute_reply.started":"2021-05-22T22:02:23.727977Z","shell.execute_reply":"2021-05-22T22:02:23.732953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_preds_df.join(train_vCDR_df_nolbl.set_index('Id'),on=\"Id\")\nval_df = val_preds_df.join(val_vCDR_df_nolbl.set_index('Id'),on = \"Id\")\ntest_df = test_preds_df.join(test_vCDR_df.set_index('Id'),on='Id')","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:46:16.770921Z","iopub.execute_input":"2021-05-22T23:46:16.771241Z","iopub.status.idle":"2021-05-22T23:46:16.790371Z","shell.execute_reply.started":"2021-05-22T23:46:16.77121Z","shell.execute_reply":"2021-05-22T23:46:16.78965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_inputs = np.array(train_df[[\"vCDRs\",\"predictions\"]])\nclassif_gts = np.array(train_df[\"labels\"])","metadata":{"execution":{"iopub.status.busy":"2021-05-22T22:17:07.744789Z","iopub.execute_input":"2021-05-22T22:17:07.745103Z","iopub.status.idle":"2021-05-22T22:17:07.75264Z","shell.execute_reply.started":"2021-05-22T22:17:07.745076Z","shell.execute_reply":"2021-05-22T22:17:07.751802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classif_eval(val_df[\"predictions\"].tolist(),val_df[\"labels\"].tolist())","metadata":{"execution":{"iopub.status.busy":"2021-05-22T22:25:14.592351Z","iopub.execute_input":"2021-05-22T22:25:14.592702Z","iopub.status.idle":"2021-05-22T22:25:14.600701Z","shell.execute_reply.started":"2021-05-22T22:25:14.592662Z","shell.execute_reply":"2021-05-22T22:25:14.599878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import gaussian_process\nfrom sklearn import neural_network\nfrom sklearn import ensemble\nclf = LogisticRegression(random_state=0)\nclf.fit(np.array(train_df[\"vCDRs\"]).reshape(-1,1),train_df[\"labels\"].tolist())\nval_proba_seg = clf.predict_proba(np.array(val_df[\"vCDRs\"]).reshape(-1,1))[:,1]\nprint(classif_eval(val_proba_seg,val_df[\"labels\"].tolist()))","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:31:41.809862Z","iopub.execute_input":"2021-05-22T23:31:41.810177Z","iopub.status.idle":"2021-05-22T23:31:41.827029Z","shell.execute_reply.started":"2021-05-22T23:31:41.810149Z","shell.execute_reply":"2021-05-22T23:31:41.825987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_proba_preds = np.array(val_df[\"predictions\"])","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:06:14.216578Z","iopub.execute_input":"2021-05-22T23:06:14.216925Z","iopub.status.idle":"2021-05-22T23:06:14.229274Z","shell.execute_reply.started":"2021-05-22T23:06:14.216894Z","shell.execute_reply":"2021-05-22T23:06:14.228111Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_val_proba = (val_proba_preds + val_proba)/2\nclassif_eval(mean_val_proba,val_df[\"labels\"].tolist())","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:25:39.367674Z","iopub.execute_input":"2021-05-22T23:25:39.36799Z","iopub.status.idle":"2021-05-22T23:25:39.376796Z","shell.execute_reply.started":"2021-05-22T23:25:39.36796Z","shell.execute_reply":"2021-05-22T23:25:39.375749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_submission_csv(prediction, submission_filename='/kaggle/working/submission.csv'):\n    \"\"\"Create a sumbission file in the appropriate format for evaluation.\n\n    :param\n    prediction: list of predictions (ex: [0.12720, 0.89289, ..., 0.29829])\n    \"\"\"\n    \n    with open(submission_filename, mode='w') as csv_file:\n        fieldnames = ['Id', 'Predicted']\n        writer = csv.DictWriter(csv_file, fieldnames=fieldnames)\n        writer.writeheader()\n\n        for i, p in enumerate(prediction):\n            writer.writerow({'Id': \"T{:04d}\".format(i+1), 'Predicted': '{:f}'.format(p)})","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:25:46.490524Z","iopub.execute_input":"2021-05-22T23:25:46.490848Z","iopub.status.idle":"2021-05-22T23:25:46.496488Z","shell.execute_reply.started":"2021-05-22T23:25:46.490817Z","shell.execute_reply":"2021-05-22T23:25:46.495527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_probas_seg = clf.predict_proba(np.array(test_vCDR_df[\"vCDR\"]).reshape(-1,1))[:,1]","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:42:21.48023Z","iopub.execute_input":"2021-05-22T23:42:21.480756Z","iopub.status.idle":"2021-05-22T23:42:21.485677Z","shell.execute_reply.started":"2021-05-22T23:42:21.480711Z","shell.execute_reply":"2021-05-22T23:42:21.484868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_probas_preds = test_preds_df[\"predictions\"].tolist()","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:40:23.768222Z","iopub.execute_input":"2021-05-22T23:40:23.768549Z","iopub.status.idle":"2021-05-22T23:40:23.772679Z","shell.execute_reply.started":"2021-05-22T23:40:23.768516Z","shell.execute_reply":"2021-05-22T23:40:23.771611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_probas = 0.5*(test_probas_preds + test_probas_seg)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:42:44.607275Z","iopub.execute_input":"2021-05-22T23:42:44.607637Z","iopub.status.idle":"2021-05-22T23:42:44.611476Z","shell.execute_reply.started":"2021-05-22T23:42:44.607606Z","shell.execute_reply":"2021-05-22T23:42:44.610535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf = gaussian_process.GaussianProcessClassifier(random_state=0)\nclf.fit(train_inputs,classif_gts.tolist())\n#val_proba_seg = clf.predict_proba(val_i).reshape(-1,1))[:,1]\n#print(classif_eval(val_proba_seg,val_df[\"labels\"].tolist()))","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:52:31.777113Z","iopub.execute_input":"2021-05-22T23:52:31.777463Z","iopub.status.idle":"2021-05-22T23:52:31.844297Z","shell.execute_reply.started":"2021-05-22T23:52:31.777414Z","shell.execute_reply":"2021-05-22T23:52:31.843484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_probas_4= clf.predict_proba(np.array(test_df[[\"vCDR\",\"predictions\"]]))[:,1]","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:52:33.046937Z","iopub.execute_input":"2021-05-22T23:52:33.047268Z","iopub.status.idle":"2021-05-22T23:52:33.069032Z","shell.execute_reply.started":"2021-05-22T23:52:33.047238Z","shell.execute_reply":"2021-05-22T23:52:33.067978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_submission_csv(test_probas_4)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:52:38.649496Z","iopub.execute_input":"2021-05-22T23:52:38.649815Z","iopub.status.idle":"2021-05-22T23:52:38.656885Z","shell.execute_reply.started":"2021-05-22T23:52:38.649787Z","shell.execute_reply":"2021-05-22T23:52:38.656128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_probas[10]","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:31:12.891101Z","iopub.execute_input":"2021-05-22T23:31:12.891483Z","iopub.status.idle":"2021-05-22T23:31:12.897834Z","shell.execute_reply.started":"2021-05-22T23:31:12.891448Z","shell.execute_reply":"2021-05-22T23:31:12.896945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_proba=clf.predict_proba(np.array(val_vCDR_df[\"vCDRs\"]).reshape(-1,1))[:,1]\nplt.plot(np.sort(np.array(val_vCDR_df[\"vCDRs\"]).tolist()),val_proba.tolist())\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:30:09.934092Z","iopub.execute_input":"2021-05-22T23:30:09.934418Z","iopub.status.idle":"2021-05-22T23:30:10.061327Z","shell.execute_reply.started":"2021-05-22T23:30:09.934387Z","shell.execute_reply":"2021-05-22T23:30:10.060392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(val_proba[20:40],'\\n',np.array(val_vCDR_df[\"vCDRs\"]).tolist()[20:40],np.array(val_vCDR_df[\"labels\"]).tolist()[20:40])","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:36:15.688751Z","iopub.execute_input":"2021-05-22T23:36:15.689063Z","iopub.status.idle":"2021-05-22T23:36:15.695308Z","shell.execute_reply.started":"2021-05-22T23:36:15.689033Z","shell.execute_reply":"2021-05-22T23:36:15.694352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}