{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#Credits: https://www.kaggle.com/pestipeti/competition-metric-map-0-4","metadata":{"execution":{"iopub.status.busy":"2021-06-18T11:02:48.844318Z","iopub.execute_input":"2021-06-18T11:02:48.844721Z","iopub.status.idle":"2021-06-18T11:02:48.848536Z","shell.execute_reply.started":"2021-06-18T11:02:48.844682Z","shell.execute_reply":"2021-06-18T11:02:48.847353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Competiton metric calculator\n\n> The challenge uses the standard [PASCAL VOC 2010 mean Average Precision (mAP)](http://host.robots.ox.ac.uk/pascal/VOC/voc2010/devkit_doc_08-May-2010.pdf) at IoU > 0.5.","metadata":{}},{"cell_type":"code","source":"!pip install pycocotools -q","metadata":{"execution":{"iopub.status.busy":"2021-07-08T16:16:12.558131Z","iopub.execute_input":"2021-07-08T16:16:12.558498Z","iopub.status.idle":"2021-07-08T16:16:27.510935Z","shell.execute_reply.started":"2021-07-08T16:16:12.558468Z","shell.execute_reply":"2021-07-08T16:16:27.510074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport plotly.express as px\nimport plotly.graph_objects as go\n\nfrom pycocotools.coco import COCO\nfrom pycocotools.cocoeval import COCOeval\nimport shutil, os\nfrom tqdm.auto import tqdm\nfrom glob import glob\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2021-07-08T16:16:27.515002Z","iopub.execute_input":"2021-07-08T16:16:27.515289Z","iopub.status.idle":"2021-07-08T16:16:30.716229Z","shell.execute_reply.started":"2021-07-08T16:16:27.51526Z","shell.execute_reply":"2021-07-08T16:16:30.715454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_level_df = pd.read_csv(\"/kaggle/input/siim-covid19-detection/train_image_level.csv\")\ndim_df = pd.read_csv(\"/kaggle/input/siim-covid19-detection-512/train.csv\")\ndim_df[\"id\"] = dim_df[\"id_image\"] + '_image'\nimage_level_df = pd.merge(image_level_df, dim_df[['id', 'width', 'height']] , on = 'id', how = 'left')\nimage_level_df[\"none\"]=image_level_df.label.apply(lambda x: 0 if x=='none 1 0 0 1 1' else 1)\n\nimage_level_df = image_level_df[0:50]","metadata":{"execution":{"iopub.status.busy":"2021-07-08T16:16:30.717413Z","iopub.execute_input":"2021-07-08T16:16:30.717745Z","iopub.status.idle":"2021-07-08T16:16:30.840354Z","shell.execute_reply.started":"2021-07-08T16:16:30.717711Z","shell.execute_reply":"2021-07-08T16:16:30.839569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(image_level_df.shape[0]):\n    if image_level_df.loc[i,'label'] == \"none 1 0 0 1 1\":\n        image_level_df.loc[i,'label']='0 1 0 0 1 1'\n        continue\n    sub_df_split = image_level_df.loc[i,'label'].split()\n    sub_df_list = []\n    for j in range(int(len(sub_df_split) / 6)):\n        sub_df_list.append('1')\n        sub_df_list.append(sub_df_split[6 * j + 1])\n        sub_df_list.append(sub_df_split[6 * j + 2])\n        sub_df_list.append(sub_df_split[6 * j + 3])\n        sub_df_list.append(sub_df_split[6 * j + 4])\n        sub_df_list.append(sub_df_split[6 * j + 5])\n    image_level_df.loc[i,'label'] = ' '.join(sub_df_list)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T16:16:30.843767Z","iopub.execute_input":"2021-07-08T16:16:30.844046Z","iopub.status.idle":"2021-07-08T16:16:33.476252Z","shell.execute_reply.started":"2021-07-08T16:16:30.844017Z","shell.execute_reply":"2021-07-08T16:16:33.475465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import GroupKFold\n\ngkf  = GroupKFold(n_splits = 5)\nimage_level_df['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(image_level_df, \n                groups = image_level_df.StudyInstanceUID.tolist())):\n    image_level_df.loc[val_idx, 'fold'] = fold","metadata":{"execution":{"iopub.status.busy":"2021-07-08T16:16:33.48014Z","iopub.execute_input":"2021-07-08T16:16:33.480471Z","iopub.status.idle":"2021-07-08T16:16:33.527272Z","shell.execute_reply.started":"2021-07-08T16:16:33.480424Z","shell.execute_reply":"2021-07-08T16:16:33.526623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_level_df['image_path'] = f'/kaggle/input/siimcovid19-512-img-png-600-study-png/image/'+ image_level_df.id + '.png'\nimage_level_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T16:16:33.528688Z","iopub.execute_input":"2021-07-08T16:16:33.529029Z","iopub.status.idle":"2021-07-08T16:16:33.553592Z","shell.execute_reply.started":"2021-07-08T16:16:33.528994Z","shell.execute_reply":"2021-07-08T16:16:33.552887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold in tqdm(range(5)):\n    test_dir = f'/kaggle/tmp/image_fold_{fold}'\n    os.makedirs(test_dir, exist_ok=True)\n    for path in image_level_df[image_level_df['fold'] == fold]['image_path'].values:\n        shutil.copyfile(path, os.path.join(test_dir, path.split(\"/\")[-1]))","metadata":{"execution":{"iopub.status.busy":"2021-07-08T16:16:33.55646Z","iopub.execute_input":"2021-07-08T16:16:33.556706Z","iopub.status.idle":"2021-07-08T16:17:11.138748Z","shell.execute_reply.started":"2021-07-08T16:16:33.556682Z","shell.execute_reply":"2021-07-08T16:17:11.137874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **2 Class**","metadata":{}},{"cell_type":"code","source":"PUBLIC=True","metadata":{"execution":{"iopub.status.busy":"2021-07-08T16:17:11.140092Z","iopub.execute_input":"2021-07-08T16:17:11.14063Z","iopub.status.idle":"2021-07-08T16:17:11.14462Z","shell.execute_reply.started":"2021-07-08T16:17:11.14059Z","shell.execute_reply":"2021-07-08T16:17:11.143794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if PUBLIC:\n    !pip install /kaggle/input/kerasapplications/keras-team-keras-applications-3b180cb -q\n    !pip install /kaggle/input/efficientnet-keras-source-code/ -q --no-deps\n\n    import os\n\n    import efficientnet.tfkeras as efn\n    import numpy as np\n    import pandas as pd\n    import tensorflow as tf\n\n    def auto_select_accelerator():\n        try:\n            tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print(\"Running on TPU:\", tpu.master())\n        except ValueError:\n            strategy = tf.distribute.get_strategy()\n        print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n\n        return strategy\n\n\n    def build_decoder(with_labels=True, target_size=(300, 300), ext='jpg'):\n        def decode(path):\n            file_bytes = tf.io.read_file(path)\n            if ext == 'png':\n                img = tf.image.decode_png(file_bytes, channels=3)\n            elif ext in ['jpg', 'jpeg']:\n                img = tf.image.decode_jpeg(file_bytes, channels=3)\n            else:\n                raise ValueError(\"Image extension not supported\")\n\n            img = tf.cast(img, tf.float32) / 255.0\n            img = tf.image.resize(img, target_size)\n\n            return img\n\n        def decode_with_labels(path, label):\n            return decode(path), label\n\n        return decode_with_labels if with_labels else decode\n\n\n    def build_augmenter(with_labels=True):\n        def augment(img):\n            img = tf.image.random_flip_left_right(img)\n            img = tf.image.random_flip_up_down(img)\n            return img\n\n        def augment_with_labels(img, label):\n            return augment(img), label\n\n        return augment_with_labels if with_labels else augment\n\n\n    def build_dataset(paths, labels=None, bsize=32, cache=True,\n                      decode_fn=None, augment_fn=None,\n                      augment=True, repeat=True, shuffle=1024, \n                      cache_dir=\"\"):\n        if cache_dir != \"\" and cache is True:\n            os.makedirs(cache_dir, exist_ok=True)\n\n        if decode_fn is None:\n            decode_fn = build_decoder(labels is not None)\n\n        if augment_fn is None:\n            augment_fn = build_augmenter(labels is not None)\n\n        AUTO = tf.data.experimental.AUTOTUNE\n        slices = paths if labels is None else (paths, labels)\n\n        dset = tf.data.Dataset.from_tensor_slices(slices)\n        dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n        dset = dset.cache(cache_dir) if cache else dset\n        dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset\n        dset = dset.repeat() if repeat else dset\n        dset = dset.shuffle(shuffle) if shuffle else dset\n        dset = dset.batch(bsize).prefetch(AUTO)\n\n        return dset\n    \n    strategy = auto_select_accelerator()\n    BATCH_SIZE = strategy.num_replicas_in_sync * 16\n    \n    sub_df_2 = image_level_df.copy()\n    sub_df_2['none'] = 0\n    test_paths = [sub_df_2[sub_df_2[\"fold\"]==fold][\"image_path\"].values for fold in range(5)]\n    \n    IMSIZE = (224, 240, 260, 300, 380, 456, 528, 600, 512)\n\n    test_decoder = build_decoder(with_labels=False, target_size=(IMSIZE[8], IMSIZE[8]), ext='png')\n    \n    \n    with strategy.scope():\n\n        models = []\n\n        models0 = tf.keras.models.load_model(\n            '/kaggle/input/siim-covid19-efnb7-train-fold0-5-2class/model0.h5'\n        )\n        models1 = tf.keras.models.load_model(\n            '/kaggle/input/siim-covid19-efnb7-train-fold0-5-2class/model1.h5'\n        )\n        models2 = tf.keras.models.load_model(\n            '/kaggle/input/siim-covid19-efnb7-train-fold0-5-2class/model2.h5'\n        )\n        models3 = tf.keras.models.load_model(\n            '/kaggle/input/siim-covid19-efnb7-train-fold0-5-2class/model3.h5'\n        )\n        models4 = tf.keras.models.load_model(\n            '/kaggle/input/siim-covid19-efnb7-train-fold0-5-2class/model4.h5'\n        )\n\n        models.append(models0)\n        models.append(models1)\n        models.append(models2)\n        models.append(models3)\n        models.append(models4)\n\n    two_class_df=[]\n\n    for fold in range(5):\n        dtest = build_dataset(\n        test_paths[fold], bsize=BATCH_SIZE, repeat=False, \n        shuffle=False, augment=False, cache=False,\n        decode_fn=test_decoder\n        )\n        df = sub_df_2[sub_df_2[\"fold\"]==fold]\n        df['none'] = models[fold].predict(dtest, verbose=1)\n        two_class_df.append(df)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T16:17:11.145854Z","iopub.execute_input":"2021-07-08T16:17:11.146382Z","iopub.status.idle":"2021-07-08T16:17:11.224744Z","shell.execute_reply.started":"2021-07-08T16:17:11.146345Z","shell.execute_reply":"2021-07-08T16:17:11.223751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if PUBLIC:\n    del models\n    del models0, models1, models2, models3, models4\n    from numba import cuda\n    cuda.select_device(0)\n    cuda.close()\n    cuda.select_device(0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Yolo-V5**","metadata":{}},{"cell_type":"code","source":"weights_dir = ['/kaggle/input/siim-cov19-yolov5-train/yolov5/runs/train/exp/weights/best.pt',\n               '/kaggle/input/siim-cov19-yolov5-train/yolov5/runs/train/exp/weights/best.pt',\n               '/kaggle/input/siim-cov19-yolov5-train/yolov5/runs/train/exp/weights/best.pt',\n               '/kaggle/input/siim-cov19-yolov5-train/yolov5/runs/train/exp/weights/best.pt',\n               '/kaggle/input/siim-cov19-yolov5-train/yolov5/runs/train/exp/weights/best.pt']\n\nshutil.copytree('/kaggle/input/yolov5-official-v31-dataset/yolov5', '/kaggle/working/yolov5')\nos.chdir('/kaggle/working/yolov5') # install dependencies\n#!pip install -r requirements.txt -q\n\nimport torch\n#from IPython.display import Image, clear_output  # to display images\n\n#clear_output()\n#print('Setup complete. Using torch %s %s' % (torch.__version__, torch.cuda.get_device_properties(0) if torch.cuda.is_available() else 'CPU'))\n\n#fold 0\nwts = weights_dir[0]\n!python detect.py --weights $wts\\\n--img 512\\\n--conf 0.001\\\n--iou 0.5\\\n--source /kaggle/tmp/image_fold_0/ \\\n--project /kaggle/working/yolov5_fold0/ --name test_iou_0.5_0.001\\\n--save-txt --save-conf --exist-ok\n\n#fold 1\nwts = weights_dir[1]\n!python detect.py --weights $wts\\\n--img 512\\\n--conf 0.001\\\n--iou 0.5\\\n--source /kaggle/tmp/image_fold_1/ \\\n--project /kaggle/working/yolov5_fold1/ --name test_iou_0.5_0.001\\\n--save-txt --save-conf --exist-ok\n\n#fold 2\nwts = weights_dir[2]\n!python detect.py --weights $wts\\\n--img 512\\\n--conf 0.001\\\n--iou 0.5\\\n--source /kaggle/tmp/image_fold_2/ \\\n--project /kaggle/working/yolov5_fold2/ --name test_iou_0.5_0.001\\\n--save-txt --save-conf --exist-ok\n\n#fold 3\nwts = weights_dir[3]\n!python detect.py --weights $wts\\\n--img 512\\\n--conf 0.001\\\n--iou 0.5\\\n--source /kaggle/tmp/image_fold_3/ \\\n--project /kaggle/working/yolov5_fold3/ --name test_iou_0.5_0.001\\\n--save-txt --save-conf --exist-ok\n\n#fold 4\nwts = weights_dir[4]\n!python detect.py --weights $wts\\\n--img 512\\\n--conf 0.001\\\n--iou 0.5\\\n--source /kaggle/tmp/image_fold_4/ \\\n--project /kaggle/working/yolov5_fold4/ --name test_iou_0.5_0.001\\\n--save-txt --save-conf --exist-ok\n\ndef yolo2voc(image_height, image_width, bboxes):\n    \"\"\"\n    yolo => [xmid, ymid, w, h] (normalized)\n    voc  => [x1, y1, x2, y1]\n\n    \"\"\" \n    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n\n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]]* image_width\n    bboxes[..., [1, 3]] = bboxes[..., [1, 3]]* image_height\n\n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]]/2\n    bboxes[..., [2, 3]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]\n\n    return bboxes\n\npreds_df_all = []\n\nfor fold in range(5):\n\n    image_ids = []\n    PredictionStrings = []\n\n    for file_path in tqdm(glob('/kaggle/working/yolov5_fold{}/test_iou_0.5_0.001/labels/*.txt'.format(fold))):\n        image_id = file_path.split('/')[-1].split('.')[0]\n        w, h = image_level_df.loc[image_level_df.id==image_id,['width', 'height']].values[0]\n        f = open(file_path, 'r')\n        data = np.array(f.read().replace('\\n', ' ').strip().split(' ')).astype(np.float32).reshape(-1, 6)\n        data = data[:, [0, 5, 1, 2, 3, 4]]\n        bboxes = list(np.round(np.concatenate((data[:, :2], np.round(yolo2voc(h, w, data[:, 2:]))), axis =1).reshape(-1), 12).astype(str))\n        for idx in range(len(bboxes)):\n            bboxes[idx] = str(int(float(bboxes[idx]))) if idx%6!=1 else bboxes[idx]\n        image_ids.append(image_id)\n        PredictionStrings.append(' '.join(bboxes))\n\n\n    full_df = image_level_df[image_level_df['fold']==fold].copy()[\"id\"]\n    preds_df = pd.DataFrame({'id':image_ids,'PredictionString':PredictionStrings})\n    preds_df_full = pd.merge(full_df, preds_df, on = 'id', how = 'left').fillna(\"none 1 0 0 1 1\")\n    preds_df_all.append(preds_df_full)\n","metadata":{"execution":{"iopub.status.busy":"2021-07-08T16:20:37.106511Z","iopub.execute_input":"2021-07-08T16:20:37.10685Z","iopub.status.idle":"2021-07-08T16:28:10.970745Z","shell.execute_reply.started":"2021-07-08T16:20:37.106813Z","shell.execute_reply":"2021-07-08T16:28:10.969681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold in range(5):\n    \n    preds_df_all[fold] = pd.merge(preds_df_all[fold], two_class_df[fold][['id', 'none']] , on = 'id', how = 'left')\n    \n    for i in tqdm(range(preds_df_all[fold].shape[0])):\n        if preds_df_all[fold].loc[i,'PredictionString'] == \"none 1 0 0 1 1\":\n            preds_df_all[fold].loc[i,'PredictionString']='0 1 0 0 1 1'\n            continue\n        sub_df_split = preds_df_all[fold].loc[i,'PredictionString'].split()\n        sub_df_list = []\n        for j in range(int(len(sub_df_split) / 6)):\n            sub_df_list.append('1')\n            sub_df_list.append(sub_df_split[6 * j + 1])\n            sub_df_list.append(sub_df_split[6 * j + 2])\n            sub_df_list.append(sub_df_split[6 * j + 3])\n            sub_df_list.append(sub_df_split[6 * j + 4])\n            sub_df_list.append(sub_df_split[6 * j + 5])\n        preds_df_all[fold].loc[i,'PredictionString'] = ' '.join(sub_df_list)\n        preds_df_all[fold].loc[i,'PredictionString'] = preds_df_all[fold].loc[i,'PredictionString'] + ' 0 ' + \\\n        str(preds_df_all[fold].loc[i,'none']) + ' 0 0 1 1'","metadata":{"execution":{"iopub.status.busy":"2021-07-08T16:42:41.877044Z","iopub.execute_input":"2021-07-08T16:42:41.877389Z","iopub.status.idle":"2021-07-08T16:42:50.303812Z","shell.execute_reply.started":"2021-07-08T16:42:41.877356Z","shell.execute_reply":"2021-07-08T16:42:50.303005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CovidDataEval:\n    \"\"\"Helper class for calculating the competition metric.\n    \n    You should remove the duplicated annoatations from the `true_df` dataframe\n    before using this script. Otherwise it may give incorrect results.\n\n        >>> covideval = CovidDataEval(valid_df)\n        >>> cocoEvalResults = covideval.evaluate(pred_df)\n\n    Arguments:\n        true_df: pd.DataFrame Clean (no duplication) Training/Validating dataframe.\n\n    Authors:\n        Peter (https://kaggle.com/pestipeti)\n\n    See:\n        https://www.kaggle.com/pestipeti/competition-metric-map-0-4\n\n    Returns: None\n    \n    \"\"\"\n    def __init__(self, true_df, study=False):\n        \n        self.true_df = true_df\n        self.study = study\n\n        self.image_ids = true_df[\"id\"].unique()\n        self.annotations = {\n            \"type\": \"instances\",\n            \"images\": self.__gen_images(self.image_ids),\n            \"categories\": self.__gen_categories(self.true_df),\n            \"annotations\": self.__gen_annotations(self.true_df, self.image_ids)\n        }\n        \n        self.predictions = {\n            \"images\": self.annotations[\"images\"].copy(),\n            \"categories\": self.annotations[\"categories\"].copy(),\n            \"annotations\": None\n        }\n\n        \n    def __gen_images(self, image_ids):\n        print(\"Generating image data...\")\n        results = []\n\n        for idx, image_id in enumerate(image_ids):\n\n            # Add image identification.\n            results.append({\n                \"id\": idx,\n            })\n            \n        return results\n    \n    \n    def __gen_categories(self, df):\n        print(\"Generating category data...\")\n        \n        if self.study:\n        \n            if \"class_name\" not in df.columns:\n                df[\"class_name\"] = df[\"class_id\"]\n\n            cats = df[[\"class_name\", \"class_id\"]]\n            cats = cats.drop_duplicates().sort_values(by='class_id').values\n\n            results = []\n\n            for cat in cats:\n                results.append({\n                    \"id\": cat[1],\n                    \"name\": cat[0],\n                    \"supercategory\": \"none\",\n                })\n\n            return results\n        \n        else:\n            results = []\n            \n            cats = df[[\"label\",\"none\"]]\n            for cat in cats:\n                results.append({\n                    \"id\": cat[1],\n                    \"name\": cat[0].split(\" \")[0],\n                    \"supercategory\": \" \",\n                })\n            return results\n        \n    def __decode_prediction_string(self, pred_str):\n        data = np.array(list(pred_str.split(\" \")))\n        return data.reshape(-1, 6)    \n    \n    def __gen_annotations(self, df, image_ids):\n        print(\"Generating annotation data...\")\n        k = 0\n        results = []\n        \n        for i, row in df.iterrows():\n            \n            image_id = row[\"id\"]\n            preds = self.__decode_prediction_string(row[\"label\"])\n\n            for j, pred in enumerate(preds):\n\n                results.append({\n                    \"id\": k,\n                    \"image_id\": int(np.where(image_ids == image_id)[0]),\n                    \"category_id\": int(pred[0]),\n                    \"bbox\": np.array([\n                        float(pred[2]), float(pred[3]), (float(pred[4])-float(pred[2])), (float(pred[5])-float(pred[3]))\n                    ]),\n                    \"segmentation\": [],\n                    \"ignore\": 0,\n                    \"area\": (float(pred[4]) - float(pred[2])) * (float(pred[5]) - float(pred[3])),\n                    \"iscrowd\": 0,\n                    \"score\": float(pred[1])\n                })\n\n                k += 1\n                \n        return results\n                \n    \n    def __gen_predictions(self, df, image_ids):\n        print(\"Generating prediction data...\")\n        k = 0\n        results = []\n        \n        for i, row in df.iterrows():\n            \n            image_id = row[\"id\"]\n            preds = self.__decode_prediction_string(row[\"PredictionString\"])\n\n            for j, pred in enumerate(preds):\n\n                results.append({\n                    \"id\": k,\n                    \"image_id\": int(np.where(image_ids == image_id)[0]),\n                    \"category_id\": int(pred[0]),\n                    \"bbox\": np.array([\n                         float(pred[2]), float(pred[3]), (float(pred[4])-float(pred[2])), (float(pred[5])-float(pred[3]))\n                    ]),\n                    \"segmentation\": [],\n                    \"ignore\": 0,\n                    \"area\": (float(pred[4]) - float(pred[2])) * (float(pred[5]) - float(pred[3])),\n                    \"iscrowd\": 0,\n                    \"score\": float(pred[1])\n                })\n\n                k += 1\n                \n        return results\n                \n    def evaluate(self, pred_df, n_imgs = -1):\n        \"\"\"Evaluating your results\n        \n        Arguments:\n            pred_df: pd.DataFrame your predicted results in the\n                     competition output format.\n\n            n_imgs:  int Number of images use for calculating the\n                     result.All of the images if `n_imgs` <= 0\n                     \n        Returns:\n            COCOEval object\n        \"\"\"\n        \n        if pred_df is not None:\n            self.predictions[\"annotations\"] = self.__gen_predictions(pred_df, self.image_ids)\n\n        coco_ds = COCO()\n        coco_ds.dataset = self.annotations\n        coco_ds.createIndex()\n        \n        coco_dt = COCO()\n        coco_dt.dataset = self.predictions\n        coco_dt.createIndex()\n        \n        imgIds=sorted(coco_ds.getImgIds())\n        \n        if n_imgs > 0:\n            imgIds = np.random.choice(imgIds, n_imgs)\n\n        cocoEval = COCOeval(coco_ds, coco_dt, 'bbox')\n        cocoEval.params.imgIds  = imgIds\n        cocoEval.params.useCats = True\n        cocoEval.params.iouType = \"bbox\"\n        cocoEval.params.iouThrs = np.array([0.5])\n\n        cocoEval.evaluate()\n        cocoEval.accumulate()\n        cocoEval.summarize()\n        \n        return cocoEval","metadata":{"execution":{"iopub.status.busy":"2021-07-08T16:42:52.909554Z","iopub.execute_input":"2021-07-08T16:42:52.909979Z","iopub.status.idle":"2021-07-08T16:42:52.947731Z","shell.execute_reply.started":"2021-07-08T16:42:52.909926Z","shell.execute_reply":"2021-07-08T16:42:52.945851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Usage","metadata":{}},{"cell_type":"code","source":"study_scores=[\n0.39388390121767813,\n0.39234564782357945,\n0.3847761244542205,\n0.37818329809109086,\n0.38300730435642993]","metadata":{"execution":{"iopub.status.busy":"2021-07-08T16:42:54.490775Z","iopub.execute_input":"2021-07-08T16:42:54.491103Z","iopub.status.idle":"2021-07-08T16:42:54.49755Z","shell.execute_reply.started":"2021-07-08T16:42:54.491073Z","shell.execute_reply":"2021-07-08T16:42:54.496694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold_mAP=[]\nfor fold in range(5):\n    covideval=CovidDataEval(image_level_df[image_level_df[\"fold\"]==fold])\n    cocoEvalRes = covideval.evaluate(preds_df_all[fold])\n    fold_mAP.append(cocoEvalRes.stats[1]*1/3)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T16:42:55.32576Z","iopub.execute_input":"2021-07-08T16:42:55.32607Z","iopub.status.idle":"2021-07-08T16:45:30.068592Z","shell.execute_reply.started":"2021-07-08T16:42:55.326039Z","shell.execute_reply":"2021-07-08T16:45:30.067857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold in range(5):\n    print(f\"\\nStudy Level mAP Score fold {fold+1}: {study_scores[fold]}\\nImage Level mAP Score fold {fold+1}: {fold_mAP[fold]}\")\n    \nprint(f\"\\nStudy Level mAP Score: {np.array(study_scores).mean()}\\nImage Level mAP Score: {np.array(fold_mAP).mean()}\\n\\nOverall mAP: {np.array(study_scores).mean()+np.array(fold_mAP).mean()}\")","metadata":{"execution":{"iopub.status.busy":"2021-07-08T16:47:11.716208Z","iopub.execute_input":"2021-07-08T16:47:11.716524Z","iopub.status.idle":"2021-07-08T16:47:11.726165Z","shell.execute_reply.started":"2021-07-08T16:47:11.716494Z","shell.execute_reply":"2021-07-08T16:47:11.725317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}