{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install 'git+https://github.com/facebookresearch/detectron2.git'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-26T14:15:43.167685Z","iopub.execute_input":"2021-07-26T14:15:43.168154Z","iopub.status.idle":"2021-07-26T14:18:12.686166Z","shell.execute_reply.started":"2021-07-26T14:15:43.168063Z","shell.execute_reply":"2021-07-26T14:18:12.685053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!conda install gdcm -c conda-forge -y","metadata":{"execution":{"iopub.status.busy":"2021-07-26T14:18:12.688132Z","iopub.execute_input":"2021-07-26T14:18:12.688500Z","iopub.status.idle":"2021-07-26T14:19:10.879346Z","shell.execute_reply.started":"2021-07-26T14:18:12.688470Z","shell.execute_reply":"2021-07-26T14:19:10.878398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from detectron2 import model_zoo\nfrom detectron2.engine import DefaultTrainer\nfrom detectron2.config import get_cfg\nfrom detectron2.utils.visualizer import Visualizer, ColorMode\nfrom detectron2.evaluation import DatasetEvaluator\n\nfrom detectron2.data import DatasetCatalog, MetadataCatalog\nfrom detectron2.data import build_detection_train_loader, build_detection_test_loader\nfrom detectron2.data import transforms as T\nfrom detectron2.data import detection_utils as utils\nfrom detectron2.utils.logger import setup_logger\nsetup_logger()\n\nimport os\nimport pandas as pd\nimport numpy as np\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nfrom tqdm import tqdm\nimport copy\nimport json\n\nimport torch\nfrom torch.utils.data import random_split","metadata":{"execution":{"iopub.status.busy":"2021-07-26T14:19:10.881454Z","iopub.execute_input":"2021-07-26T14:19:10.881822Z","iopub.status.idle":"2021-07-26T14:19:11.964505Z","shell.execute_reply.started":"2021-07-26T14:19:10.881784Z","shell.execute_reply":"2021-07-26T14:19:11.963635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_xray(path, voi_lut = True, fix_monochrome = True):\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \n    # \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n        \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data\n\ndef load_data_dicts(path):\n    with open(path, 'r') as f:\n        data = json.load(f)\n        \n    for d in data:\n        if len(d['annotations']) > 0:\n            for idx in range(len(d['annotations'])):\n                d['annotations'][idx]['category_id'] = 0\n    \n    return data","metadata":{"execution":{"iopub.status.busy":"2021-07-26T14:23:18.787470Z","iopub.execute_input":"2021-07-26T14:23:18.787884Z","iopub.status.idle":"2021-07-26T14:23:18.798571Z","shell.execute_reply.started":"2021-07-26T14:23:18.787827Z","shell.execute_reply":"2021-07-26T14:23:18.797431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_path = '../input/covid-detectron2-preprocessing/train_data_dicts.json'\nval_path = '../input/covid-detectron2-preprocessing/val_data_dicts.json'\nthing_classes = ['opacity']\n\nDatasetCatalog.register('covid_train', lambda: load_data_dicts(train_path))\nDatasetCatalog.register('covid_val', lambda: load_data_dicts(val_path))\nMetadataCatalog.get(\"covid_train\").set(thing_classes=thing_classes)\nMetadataCatalog.get(\"covid_val\").set(thing_classes=thing_classes)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T14:19:11.978533Z","iopub.execute_input":"2021-07-26T14:19:11.979097Z","iopub.status.idle":"2021-07-26T14:19:11.990933Z","shell.execute_reply.started":"2021-07-26T14:19:11.979055Z","shell.execute_reply":"2021-07-26T14:19:11.989927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def custom_mapper(dataset_dict):\n    dataset_dict = copy.deepcopy(dataset_dict)\n    image = read_xray(dataset_dict['file_name'])\n\n    transform_list = [T.Resize((1024,1024)),\n                      T.RandomBrightness(0.8, 1.2),\n                      T.RandomFlip(prob=0.5, horizontal=False, vertical=True),\n                      T.RandomFlip(prob=0.5, horizontal=True, vertical=False)\n                      ]\n    image, transforms = T.apply_transform_gens(transform_list, image)\n    dataset_dict[\"image\"] = torch.as_tensor(np.expand_dims(image, axis=0).astype(\"float32\"))\n\n    annos = [\n        utils.transform_instance_annotations(obj, transforms, image.shape)\n        for obj in dataset_dict.pop(\"annotations\")\n    ]\n    instances = utils.annotations_to_instances(annos, image.shape)\n    dataset_dict[\"instances\"] = utils.filter_empty_instances(instances)\n    return dataset_dict","metadata":{"execution":{"iopub.status.busy":"2021-07-26T14:19:11.993384Z","iopub.execute_input":"2021-07-26T14:19:11.994321Z","iopub.status.idle":"2021-07-26T14:19:12.003496Z","shell.execute_reply.started":"2021-07-26T14:19:11.994281Z","shell.execute_reply":"2021-07-26T14:19:12.002650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nOriginal code from https://github.com/cocodataset/cocoapi/blob/8c9bcc3cf640524c4c20a9c40e89cb6a2f2fa0e9/PythonAPI/pycocotools/cocoeval.py\nJust modified to show AP@40\n\"\"\"\n# Copyright (c) Facebook, Inc. and its affiliates.\nimport contextlib\nimport copy\nimport io\nimport itertools\nimport json\nimport logging\nimport numpy as np\nimport os\nimport pickle\nfrom collections import OrderedDict\nimport pycocotools.mask as mask_util\nimport torch\nfrom pycocotools.coco import COCO\nfrom pycocotools.cocoeval import COCOeval\nfrom tabulate import tabulate\n\nimport detectron2.utils.comm as comm\nfrom detectron2.config import CfgNode\nfrom detectron2.data import MetadataCatalog\nfrom detectron2.data.datasets.coco import convert_to_coco_json\nfrom detectron2.evaluation.evaluator import DatasetEvaluator\nfrom detectron2.evaluation.fast_eval_api import COCOeval_opt\nfrom detectron2.structures import Boxes, BoxMode, pairwise_iou\nfrom detectron2.utils.file_io import PathManager\nfrom detectron2.utils.logger import create_small_table\n\n\ndef covid_summarize(self):\n    '''\n    Compute and display summary metrics for evaluation results.\n    Note this functin can *only* be applied on the default parameter setting\n    '''\n\n    def _summarize(ap=1, iouThr=None, areaRng='all', maxDets=100):\n        p = self.params\n        iStr = ' {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}'\n        titleStr = 'Average Precision' if ap == 1 else 'Average Recall'\n        typeStr = '(AP)' if ap == 1 else '(AR)'\n        iouStr = '{:0.2f}:{:0.2f}'.format(p.iouThrs[0], p.iouThrs[-1]) \\\n            if iouThr is None else '{:0.2f}'.format(iouThr)\n\n        aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng]\n        mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets]\n        if ap == 1:\n            # dimension of precision: [TxRxKxAxM]\n            s = self.eval['precision']\n            # IoU\n            if iouThr is not None:\n                t = np.where(iouThr == p.iouThrs)[0]\n                s = s[t]\n            s = s[:, :, :, aind, mind]\n        else:\n            # dimension of recall: [TxKxAxM]\n            s = self.eval['recall']\n            if iouThr is not None:\n                t = np.where(iouThr == p.iouThrs)[0]\n                s = s[t]\n            s = s[:, :, aind, mind]\n        if len(s[s > -1]) == 0:\n            mean_s = -1\n        else:\n            mean_s = np.mean(s[s > -1])\n        print(iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, mean_s))\n        return mean_s\n\n    def _summarizeDets():\n        stats = np.zeros((12,))\n        stats[0] = _summarize(1)\n        stats[1] = _summarize(1, iouThr=.5, maxDets=self.params.maxDets[2])\n        stats[2] = _summarize(1, iouThr=.75, maxDets=self.params.maxDets[2])\n        # stats[2] = _summarize(1, iouThr=.4, maxDets=self.params.maxDets[2])\n        stats[3] = _summarize(1, areaRng='small', maxDets=self.params.maxDets[2])\n        stats[4] = _summarize(1, areaRng='medium', maxDets=self.params.maxDets[2])\n        stats[5] = _summarize(1, areaRng='large', maxDets=self.params.maxDets[2])\n        stats[6] = _summarize(0, maxDets=self.params.maxDets[0])\n        stats[7] = _summarize(0, maxDets=self.params.maxDets[1])\n        stats[8] = _summarize(0, maxDets=self.params.maxDets[2])\n        stats[9] = _summarize(0, areaRng='small', maxDets=self.params.maxDets[2])\n        stats[10] = _summarize(0, areaRng='medium', maxDets=self.params.maxDets[2])\n        stats[11] = _summarize(0, areaRng='large', maxDets=self.params.maxDets[2])\n        return stats\n\n    def _summarizeKps():\n        stats = np.zeros((10,))\n        stats[0] = _summarize(1, maxDets=20)\n        stats[1] = _summarize(1, maxDets=20, iouThr=.5)\n        stats[2] = _summarize(1, maxDets=20, iouThr=.75)\n        stats[3] = _summarize(1, maxDets=20, areaRng='medium')\n        stats[4] = _summarize(1, maxDets=20, areaRng='large')\n        stats[5] = _summarize(0, maxDets=20)\n        stats[6] = _summarize(0, maxDets=20, iouThr=.5)\n        stats[7] = _summarize(0, maxDets=20, iouThr=.75)\n        stats[8] = _summarize(0, maxDets=20, areaRng='medium')\n        stats[9] = _summarize(0, maxDets=20, areaRng='large')\n        return stats\n\n    if not self.eval:\n        raise Exception('Please run accumulate() first')\n    iouType = self.params.iouType\n    if iouType == 'segm' or iouType == 'bbox':\n        summarize = _summarizeDets\n    elif iouType == 'keypoints':\n        summarize = _summarizeKps\n    self.stats = summarize()\n\n\n# print(\"HACKING: overriding COCOeval.summarize = vin_summarize...\")\nCOCOeval.summarize = covid_summarize\n\n\nclass CovidEvaluator(DatasetEvaluator):\n    \"\"\"\n    Evaluate AR for object proposals, AP for instance detection/segmentation, AP\n    for keypoint detection outputs using COCO's metrics.\n    See http://cocodataset.org/#detection-eval and\n    http://cocodataset.org/#keypoints-eval to understand its metrics.\n\n    In addition to COCO, this evaluator is able to support any bounding box detection,\n    instance segmentation, or keypoint detection dataset.\n    \"\"\"\n\n    def __init__(\n        self,\n        dataset_name,\n        tasks=None,\n        distributed=True,\n        output_dir=None,\n        *,\n        use_fast_impl=True,\n        kpt_oks_sigmas=(),\n    ):\n        \"\"\"\n        Args:\n            dataset_name (str): name of the dataset to be evaluated.\n                It must have either the following corresponding metadata:\n\n                    \"json_file\": the path to the COCO format annotation\n\n                Or it must be in detectron2's standard dataset format\n                so it can be converted to COCO format automatically.\n            tasks (tuple[str]): tasks that can be evaluated under the given\n                configuration. A task is one of \"bbox\", \"segm\", \"keypoints\".\n                By default, will infer this automatically from predictions.\n            distributed (True): if True, will collect results from all ranks and run evaluation\n                in the main process.\n                Otherwise, will only evaluate the results in the current process.\n            output_dir (str): optional, an output directory to dump all\n                results predicted on the dataset. The dump contains two files:\n\n                1. \"instances_predictions.pth\" a file in torch serialization\n                   format that contains all the raw original predictions.\n                2. \"coco_instances_results.json\" a json file in COCO's result\n                   format.\n            use_fast_impl (bool): use a fast but **unofficial** implementation to compute AP.\n                Although the results should be very close to the official implementation in COCO\n                API, it is still recommended to compute results with the official API for use in\n                papers. The faster implementation also uses more RAM.\n            kpt_oks_sigmas (list[float]): The sigmas used to calculate keypoint OKS.\n                See http://cocodataset.org/#keypoints-eval\n                When empty, it will use the defaults in COCO.\n                Otherwise it should be the same length as ROI_KEYPOINT_HEAD.NUM_KEYPOINTS.\n        \"\"\"\n        self._logger = logging.getLogger(__name__)\n        self._distributed = distributed\n        self._output_dir = output_dir\n        self._use_fast_impl = use_fast_impl\n\n        if tasks is not None and isinstance(tasks, CfgNode):\n            kpt_oks_sigmas = (\n                tasks.TEST.KEYPOINT_OKS_SIGMAS if not kpt_oks_sigmas else kpt_oks_sigmas\n            )\n            self._logger.warn(\n                \"COCO Evaluator instantiated using config, this is deprecated behavior.\"\n                \" Please pass in explicit arguments instead.\"\n            )\n            self._tasks = None  # Infering it from predictions should be better\n        else:\n            self._tasks = tasks\n\n        self._cpu_device = torch.device(\"cpu\")\n\n        self._metadata = MetadataCatalog.get(dataset_name)\n        if not hasattr(self._metadata, \"json_file\"):\n            self._logger.info(\n                f\"'{dataset_name}' is not registered by `register_coco_instances`.\"\n                \" Therefore trying to convert it to COCO format ...\"\n            )\n\n            cache_path = os.path.join(output_dir, f\"{dataset_name}_coco_format.json\")\n            self._metadata.json_file = cache_path\n            convert_to_coco_json(dataset_name, cache_path)\n\n        json_file = PathManager.get_local_path(self._metadata.json_file)\n        with contextlib.redirect_stdout(io.StringIO()):\n            self._coco_api = COCO(json_file)\n\n        # Test set json files do not contain annotations (evaluation must be\n        # performed using the COCO evaluation server).\n        self._do_evaluation = \"annotations\" in self._coco_api.dataset\n        if self._do_evaluation:\n            self._kpt_oks_sigmas = kpt_oks_sigmas\n\n    def reset(self):\n        self._predictions = []\n\n    def process(self, inputs, outputs):\n        \"\"\"\n        Args:\n            inputs: the inputs to a COCO model (e.g., GeneralizedRCNN).\n                It is a list of dict. Each dict corresponds to an image and\n                contains keys like \"height\", \"width\", \"file_name\", \"image_id\".\n            outputs: the outputs of a COCO model. It is a list of dicts with key\n                \"instances\" that contains :class:`Instances`.\n        \"\"\"\n        for input, output in zip(inputs, outputs):\n            prediction = {\"image_id\": input[\"image_id\"]}\n\n            if \"instances\" in output:\n                instances = output[\"instances\"].to(self._cpu_device)\n                prediction[\"instances\"] = instances_to_coco_json(instances, input[\"image_id\"])\n            if \"proposals\" in output:\n                prediction[\"proposals\"] = output[\"proposals\"].to(self._cpu_device)\n            if len(prediction) > 1:\n                self._predictions.append(prediction)\n\n    def evaluate(self, img_ids=None):\n        \"\"\"\n        Args:\n            img_ids: a list of image IDs to evaluate on. Default to None for the whole dataset\n        \"\"\"\n        if self._distributed:\n            comm.synchronize()\n            predictions = comm.gather(self._predictions, dst=0)\n            predictions = list(itertools.chain(*predictions))\n\n            if not comm.is_main_process():\n                return {}\n        else:\n            predictions = self._predictions\n\n        if len(predictions) == 0:\n            self._logger.warning(\"[VinbigdataEvaluator] Did not receive valid predictions.\")\n            return {}\n\n        if self._output_dir:\n            PathManager.mkdirs(self._output_dir)\n            file_path = os.path.join(self._output_dir, \"instances_predictions.pth\")\n            with PathManager.open(file_path, \"wb\") as f:\n                torch.save(predictions, f)\n\n        self._results = OrderedDict()\n        if \"proposals\" in predictions[0]:\n            self._eval_box_proposals(predictions)\n        if \"instances\" in predictions[0]:\n            self._eval_predictions(predictions, img_ids=img_ids)\n        # Copy so the caller can do whatever with results\n        return copy.deepcopy(self._results)\n\n    def _tasks_from_predictions(self, predictions):\n        \"\"\"\n        Get COCO API \"tasks\" (i.e. iou_type) from COCO-format predictions.\n        \"\"\"\n        tasks = {\"bbox\"}\n        for pred in predictions:\n            if \"segmentation\" in pred:\n                tasks.add(\"segm\")\n            if \"keypoints\" in pred:\n                tasks.add(\"keypoints\")\n        return sorted(tasks)\n\n    def _eval_predictions(self, predictions, img_ids=None):\n        \"\"\"\n        Evaluate predictions. Fill self._results with the metrics of the tasks.\n        \"\"\"\n        self._logger.info(\"Preparing results for COCO format ...\")\n        coco_results = list(itertools.chain(*[x[\"instances\"] for x in predictions]))\n        tasks = self._tasks or self._tasks_from_predictions(coco_results)\n\n        # unmap the category ids for COCO\n        if hasattr(self._metadata, \"thing_dataset_id_to_contiguous_id\"):\n            dataset_id_to_contiguous_id = self._metadata.thing_dataset_id_to_contiguous_id\n            all_contiguous_ids = list(dataset_id_to_contiguous_id.values())\n            num_classes = len(all_contiguous_ids)\n            assert min(all_contiguous_ids) == 0 and max(all_contiguous_ids) == num_classes - 1\n\n            reverse_id_mapping = {v: k for k, v in dataset_id_to_contiguous_id.items()}\n            for result in coco_results:\n                category_id = result[\"category_id\"]\n                assert category_id < num_classes, (\n                    f\"A prediction has class={category_id}, \"\n                    f\"but the dataset only has {num_classes} classes and \"\n                    f\"predicted class id should be in [0, {num_classes - 1}].\"\n                )\n                result[\"category_id\"] = reverse_id_mapping[category_id]\n\n        if self._output_dir:\n            file_path = os.path.join(self._output_dir, \"coco_instances_results.json\")\n            self._logger.info(\"Saving results to {}\".format(file_path))\n            with PathManager.open(file_path, \"w\") as f:\n                f.write(json.dumps(coco_results))\n                f.flush()\n\n        if not self._do_evaluation:\n            self._logger.info(\"Annotations are not available for evaluation.\")\n            return\n\n        self._logger.info(\n            \"Evaluating predictions with {} COCO API...\".format(\n                \"unofficial\" if self._use_fast_impl else \"official\"\n            )\n        )\n        for task in sorted(tasks):\n            coco_eval = (\n                _evaluate_predictions_on_coco(\n                    self._coco_api,\n                    coco_results,\n                    task,\n                    kpt_oks_sigmas=self._kpt_oks_sigmas,\n                    use_fast_impl=self._use_fast_impl,\n                    img_ids=img_ids,\n                )\n                if len(coco_results) > 0\n                else None  # cocoapi does not handle empty results very well\n            )\n\n            res = self._derive_coco_results(\n                coco_eval, task, class_names=self._metadata.get(\"thing_classes\")\n            )\n            self._results[task] = res\n\n    def _eval_box_proposals(self, predictions):\n        \"\"\"\n        Evaluate the box proposals in predictions.\n        Fill self._results with the metrics for \"box_proposals\" task.\n        \"\"\"\n        if self._output_dir:\n            # Saving generated box proposals to file.\n            # Predicted box_proposals are in XYXY_ABS mode.\n            bbox_mode = BoxMode.XYXY_ABS.value\n            ids, boxes, objectness_logits = [], [], []\n            for prediction in predictions:\n                ids.append(prediction[\"image_id\"])\n                boxes.append(prediction[\"proposals\"].proposal_boxes.tensor.numpy())\n                objectness_logits.append(prediction[\"proposals\"].objectness_logits.numpy())\n\n            proposal_data = {\n                \"boxes\": boxes,\n                \"objectness_logits\": objectness_logits,\n                \"ids\": ids,\n                \"bbox_mode\": bbox_mode,\n            }\n            with PathManager.open(os.path.join(self._output_dir, \"box_proposals.pkl\"), \"wb\") as f:\n                pickle.dump(proposal_data, f)\n\n        if not self._do_evaluation:\n            self._logger.info(\"Annotations are not available for evaluation.\")\n            return\n\n        self._logger.info(\"Evaluating bbox proposals ...\")\n        res = {}\n        areas = {\"all\": \"\", \"small\": \"s\", \"medium\": \"m\", \"large\": \"l\"}\n        for limit in [100, 1000]:\n            for area, suffix in areas.items():\n                stats = _evaluate_box_proposals(predictions, self._coco_api, area=area, limit=limit)\n                key = \"AR{}@{:d}\".format(suffix, limit)\n                res[key] = float(stats[\"ar\"].item() * 100)\n        self._logger.info(\"Proposal metrics: \\n\" + create_small_table(res))\n        self._results[\"box_proposals\"] = res\n\n    def _derive_coco_results(self, coco_eval, iou_type, class_names=None):\n        \"\"\"\n        Derive the desired score numbers from summarized COCOeval.\n\n        Args:\n            coco_eval (None or COCOEval): None represents no predictions from model.\n            iou_type (str):\n            class_names (None or list[str]): if provided, will use it to predict\n                per-category AP.\n\n        Returns:\n            a dict of {metric name: score}\n        \"\"\"\n\n        metrics = {\n            \"bbox\": [\"AP\", \"AP50\", \"AP75\", \"APs\", \"APm\", \"APl\"],\n            \"segm\": [\"AP\", \"AP50\", \"AP75\", \"APs\", \"APm\", \"APl\"],\n            \"keypoints\": [\"AP\", \"AP50\", \"AP75\", \"APm\", \"APl\"],\n        }[iou_type]\n\n        if coco_eval is None:\n            self._logger.warn(\"No predictions from the model!\")\n            return {metric: float(\"nan\") for metric in metrics}\n\n        # the standard metrics\n        results = {\n            metric: float(coco_eval.stats[idx] * 100 if coco_eval.stats[idx] >= 0 else \"nan\")\n            for idx, metric in enumerate(metrics)\n        }\n        self._logger.info(\n            \"Evaluation results for {}: \\n\".format(iou_type) + create_small_table(results)\n        )\n        if not np.isfinite(sum(results.values())):\n            self._logger.info(\"Some metrics cannot be computed and is shown as NaN.\")\n\n        if class_names is None or len(class_names) <= 1:\n            return results\n        # Compute per-category AP\n        # from https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L222-L252 # noqa\n        precisions = coco_eval.eval[\"precision\"]\n        # precision has dims (iou, recall, cls, area range, max dets)\n        assert len(class_names) == precisions.shape[2]\n\n        results_per_category = []\n        for idx, name in enumerate(class_names):\n            # area range index 0: all area ranges\n            # max dets index -1: typically 100 per image\n            precision = precisions[:, :, idx, 0, -1]\n            precision = precision[precision > -1]\n            ap = np.mean(precision) if precision.size else float(\"nan\")\n            results_per_category.append((\"{}\".format(name), float(ap * 100)))\n\n        # tabulate it\n        N_COLS = min(6, len(results_per_category) * 2)\n        results_flatten = list(itertools.chain(*results_per_category))\n        results_2d = itertools.zip_longest(*[results_flatten[i::N_COLS] for i in range(N_COLS)])\n        table = tabulate(\n            results_2d,\n            tablefmt=\"pipe\",\n            floatfmt=\".3f\",\n            headers=[\"category\", \"AP\"] * (N_COLS // 2),\n            numalign=\"left\",\n        )\n        self._logger.info(\"Per-category {} AP: \\n\".format(iou_type) + table)\n\n        results.update({\"AP-\" + name: ap for name, ap in results_per_category})\n        return results\n\n\ndef instances_to_coco_json(instances, img_id):\n    \"\"\"\n    Dump an \"Instances\" object to a COCO-format json that's used for evaluation.\n\n    Args:\n        instances (Instances):\n        img_id (int): the image id\n\n    Returns:\n        list[dict]: list of json annotations in COCO format.\n    \"\"\"\n    num_instance = len(instances)\n    if num_instance == 0:\n        return []\n\n    boxes = instances.pred_boxes.tensor.numpy()\n    boxes = BoxMode.convert(boxes, BoxMode.XYXY_ABS, BoxMode.XYWH_ABS)\n    boxes = boxes.tolist()\n    scores = instances.scores.tolist()\n    classes = instances.pred_classes.tolist()\n\n    has_mask = instances.has(\"pred_masks\")\n    if has_mask:\n        # use RLE to encode the masks, because they are too large and takes memory\n        # since this evaluator stores outputs of the entire dataset\n        rles = [\n            mask_util.encode(np.array(mask[:, :, None], order=\"F\", dtype=\"uint8\"))[0]\n            for mask in instances.pred_masks\n        ]\n        for rle in rles:\n            # \"counts\" is an array encoded by mask_util as a byte-stream. Python3's\n            # json writer which always produces strings cannot serialize a bytestream\n            # unless you decode it. Thankfully, utf-8 works out (which is also what\n            # the pycocotools/_mask.pyx does).\n            rle[\"counts\"] = rle[\"counts\"].decode(\"utf-8\")\n\n    has_keypoints = instances.has(\"pred_keypoints\")\n    if has_keypoints:\n        keypoints = instances.pred_keypoints\n\n    results = []\n    for k in range(num_instance):\n        result = {\n            \"image_id\": img_id,\n            \"category_id\": classes[k],\n            \"bbox\": boxes[k],\n            \"score\": scores[k],\n        }\n        if has_mask:\n            result[\"segmentation\"] = rles[k]\n        if has_keypoints:\n            # In COCO annotations,\n            # keypoints coordinates are pixel indices.\n            # However our predictions are floating point coordinates.\n            # Therefore we subtract 0.5 to be consistent with the annotation format.\n            # This is the inverse of data loading logic in `datasets/coco.py`.\n            keypoints[k][:, :2] -= 0.5\n            result[\"keypoints\"] = keypoints[k].flatten().tolist()\n        results.append(result)\n    return results\n\n\n# inspired from Detectron:\n# https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L255 # noqa\ndef _evaluate_box_proposals(dataset_predictions, coco_api, thresholds=None, area=\"all\", limit=None):\n    \"\"\"\n    Evaluate detection proposal recall metrics. This function is a much\n    faster alternative to the official COCO API recall evaluation code. However,\n    it produces slightly different results.\n    \"\"\"\n    # Record max overlap value for each gt box\n    # Return vector of overlap values\n    areas = {\n        \"all\": 0,\n        \"small\": 1,\n        \"medium\": 2,\n        \"large\": 3,\n        \"96-128\": 4,\n        \"128-256\": 5,\n        \"256-512\": 6,\n        \"512-inf\": 7,\n    }\n    area_ranges = [\n        [0 ** 2, 1e5 ** 2],  # all\n        [0 ** 2, 32 ** 2],  # small\n        [32 ** 2, 96 ** 2],  # medium\n        [96 ** 2, 1e5 ** 2],  # large\n        [96 ** 2, 128 ** 2],  # 96-128\n        [128 ** 2, 256 ** 2],  # 128-256\n        [256 ** 2, 512 ** 2],  # 256-512\n        [512 ** 2, 1e5 ** 2],\n    ]  # 512-inf\n    assert area in areas, \"Unknown area range: {}\".format(area)\n    area_range = area_ranges[areas[area]]\n    gt_overlaps = []\n    num_pos = 0\n\n    for prediction_dict in dataset_predictions:\n        predictions = prediction_dict[\"proposals\"]\n\n        # sort predictions in descending order\n        # TODO maybe remove this and make it explicit in the documentation\n        inds = predictions.objectness_logits.sort(descending=True)[1]\n        predictions = predictions[inds]\n\n        ann_ids = coco_api.getAnnIds(imgIds=prediction_dict[\"image_id\"])\n        anno = coco_api.loadAnns(ann_ids)\n        gt_boxes = [\n            BoxMode.convert(obj[\"bbox\"], BoxMode.XYWH_ABS, BoxMode.XYXY_ABS)\n            for obj in anno\n            if obj[\"iscrowd\"] == 0\n        ]\n        gt_boxes = torch.as_tensor(gt_boxes).reshape(-1, 4)  # guard against no boxes\n        gt_boxes = Boxes(gt_boxes)\n        gt_areas = torch.as_tensor([obj[\"area\"] for obj in anno if obj[\"iscrowd\"] == 0])\n\n        if len(gt_boxes) == 0 or len(predictions) == 0:\n            continue\n\n        valid_gt_inds = (gt_areas >= area_range[0]) & (gt_areas <= area_range[1])\n        gt_boxes = gt_boxes[valid_gt_inds]\n\n        num_pos += len(gt_boxes)\n\n        if len(gt_boxes) == 0:\n            continue\n\n        if limit is not None and len(predictions) > limit:\n            predictions = predictions[:limit]\n\n        overlaps = pairwise_iou(predictions.proposal_boxes, gt_boxes)\n\n        _gt_overlaps = torch.zeros(len(gt_boxes))\n        for j in range(min(len(predictions), len(gt_boxes))):\n            # find which proposal box maximally covers each gt box\n            # and get the iou amount of coverage for each gt box\n            max_overlaps, argmax_overlaps = overlaps.max(dim=0)\n\n            # find which gt box is 'best' covered (i.e. 'best' = most iou)\n            gt_ovr, gt_ind = max_overlaps.max(dim=0)\n            assert gt_ovr >= 0\n            # find the proposal box that covers the best covered gt box\n            box_ind = argmax_overlaps[gt_ind]\n            # record the iou coverage of this gt box\n            _gt_overlaps[j] = overlaps[box_ind, gt_ind]\n            assert _gt_overlaps[j] == gt_ovr\n            # mark the proposal box and the gt box as used\n            overlaps[box_ind, :] = -1\n            overlaps[:, gt_ind] = -1\n\n        # append recorded iou coverage level\n        gt_overlaps.append(_gt_overlaps)\n    gt_overlaps = (\n        torch.cat(gt_overlaps, dim=0) if len(gt_overlaps) else torch.zeros(0, dtype=torch.float32)\n    )\n    gt_overlaps, _ = torch.sort(gt_overlaps)\n\n    if thresholds is None:\n        step = 0.05\n        thresholds = torch.arange(0.5, 0.95 + 1e-5, step, dtype=torch.float32)\n        # thresholds = torch.arange(0.4, 0.95 + 1e-5, step, dtype=torch.float32)\n    recalls = torch.zeros_like(thresholds)\n    # compute recall for each iou threshold\n    for i, t in enumerate(thresholds):\n        recalls[i] = (gt_overlaps >= t).float().sum() / float(num_pos)\n    # ar = 2 * np.trapz(recalls, thresholds)\n    ar = recalls.mean()\n    return {\n        \"ar\": ar,\n        \"recalls\": recalls,\n        \"thresholds\": thresholds,\n        \"gt_overlaps\": gt_overlaps,\n        \"num_pos\": num_pos,\n    }\n\n\ndef _evaluate_predictions_on_coco(\n    coco_gt, coco_results, iou_type, kpt_oks_sigmas=None, use_fast_impl=True, img_ids=None\n):\n    \"\"\"\n    Evaluate the coco results using COCOEval API.\n    \"\"\"\n    assert len(coco_results) > 0\n\n    if iou_type == \"segm\":\n        coco_results = copy.deepcopy(coco_results)\n        # When evaluating mask AP, if the results contain bbox, cocoapi will\n        # use the box area as the area of the instance, instead of the mask area.\n        # This leads to a different definition of small/medium/large.\n        # We remove the bbox field to let mask AP use mask area.\n        for c in coco_results:\n            c.pop(\"bbox\", None)\n\n    coco_dt = coco_gt.loadRes(coco_results)\n    coco_eval = (COCOeval_opt if use_fast_impl else COCOeval)(coco_gt, coco_dt, iou_type)\n\n    coco_eval.params.iouThrs = np.linspace(\n        .5, 0.95, int(np.round((0.95 - .5) / .05)) + 1, endpoint=True)\n\n    if img_ids is not None:\n        coco_eval.params.imgIds = img_ids\n\n    if iou_type == \"keypoints\":\n        # Use the COCO default keypoint OKS sigmas unless overrides are specified\n        if kpt_oks_sigmas:\n            assert hasattr(coco_eval.params, \"kpt_oks_sigmas\"), \"pycocotools is too old!\"\n            coco_eval.params.kpt_oks_sigmas = np.array(kpt_oks_sigmas)\n        # COCOAPI requires every detection and every gt to have keypoints, so\n        # we just take the first entry from both\n        num_keypoints_dt = len(coco_results[0][\"keypoints\"]) // 3\n        num_keypoints_gt = len(next(iter(coco_gt.anns.values()))[\"keypoints\"]) // 3\n        num_keypoints_oks = len(coco_eval.params.kpt_oks_sigmas)\n#         assert num_keypoints_oks == num_keypoints_dt == num_keypoints_gt, (\n#             f\"[VinbigdataEvaluator] Prediction contain {num_keypoints_dt} keypoints. \"\n#             f\"Ground truth contains {num_keypoints_gt} keypoints. \"\n#             f\"The length of cfg.TEST.KEYPOINT_OKS_SIGMAS is {num_keypoints_oks}. \"\n#             \"They have to agree with each other. For meaning of OKS, please refer to \"\n#             \"http://cocodataset.org/#keypoints-eval.\"\n#         )\n\n    coco_eval.evaluate()\n    coco_eval.accumulate()\n    coco_eval.summarize()\n\n    return coco_eval","metadata":{"execution":{"iopub.status.busy":"2021-07-26T14:19:12.005469Z","iopub.execute_input":"2021-07-26T14:19:12.005964Z","iopub.status.idle":"2021-07-26T14:19:12.096894Z","shell.execute_reply.started":"2021-07-26T14:19:12.005923Z","shell.execute_reply":"2021-07-26T14:19:12.096042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nTo calculate & record validation loss\n\nOriginal code from https://medium.com/@apofeniaco/training-on-detectron2-with-a-validation-set-and-plot-loss-on-it-to-avoid-overfitting-6449418fbf4e\nby @apofeniaco\n\"\"\"\nimport numpy as np\nimport logging\n\nfrom detectron2.engine.hooks import HookBase\nfrom detectron2.utils.logger import log_every_n_seconds\nimport detectron2.utils.comm as comm\nimport torch\nimport time\nimport datetime\n\n\nclass LossEvalHook(HookBase):\n    def __init__(self, eval_period, model, data_loader):\n        self._model = model\n        self._period = eval_period\n        self._data_loader = data_loader\n\n    def _do_loss_eval(self):\n        # Copying inference_on_dataset from evaluator.py\n        total = len(self._data_loader)\n        num_warmup = min(5, total - 1)\n\n        start_time = time.perf_counter()\n        total_compute_time = 0\n        losses = []\n        for idx, inputs in enumerate(self._data_loader):\n            if idx == num_warmup:\n                start_time = time.perf_counter()\n                total_compute_time = 0\n            start_compute_time = time.perf_counter()\n            if torch.cuda.is_available():\n                torch.cuda.synchronize()\n            total_compute_time += time.perf_counter() - start_compute_time\n            iters_after_start = idx + 1 - num_warmup * int(idx >= num_warmup)\n            seconds_per_img = total_compute_time / iters_after_start\n            if idx >= num_warmup * 2 or seconds_per_img > 5:\n                total_seconds_per_img = (time.perf_counter() - start_time) / iters_after_start\n                eta = datetime.timedelta(seconds=int(total_seconds_per_img * (total - idx - 1)))\n                log_every_n_seconds(\n                    logging.INFO,\n                    \"Loss on Validation  done {}/{}. {:.4f} s / img. ETA={}\".format(\n                        idx + 1, total, seconds_per_img, str(eta)\n                    ),\n                    n=5,\n                )\n            loss_batch = self._get_loss(inputs)\n            losses.append(loss_batch)\n        mean_loss = np.mean(losses)\n        # self.trainer.storage.put_scalar('validation_loss', mean_loss)\n        comm.synchronize()\n\n        # return losses\n        return mean_loss\n\n    def _get_loss(self, data):\n        # How loss is calculated on train_loop\n        metrics_dict = self._model(data)\n        metrics_dict = {\n            k: v.detach().cpu().item() if isinstance(v, torch.Tensor) else float(v)\n            for k, v in metrics_dict.items()\n        }\n        total_losses_reduced = sum(loss for loss in metrics_dict.values())\n        return total_losses_reduced\n\n    def after_step(self):\n        next_iter = int(self.trainer.iter) + 1\n        is_final = next_iter == self.trainer.max_iter\n        if is_final or (self._period > 0 and next_iter % self._period == 0):\n            mean_loss = self._do_loss_eval()\n            self.trainer.storage.put_scalars(validation_loss=mean_loss)\n            print(\"validation do loss eval\", mean_loss)\n        else:\n            pass\n            # self.trainer.storage.put_scalars(timetest=11)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T14:19:12.099509Z","iopub.execute_input":"2021-07-26T14:19:12.100076Z","iopub.status.idle":"2021-07-26T14:19:12.117226Z","shell.execute_reply.started":"2021-07-26T14:19:12.100032Z","shell.execute_reply":"2021-07-26T14:19:12.115913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Trainer(DefaultTrainer):\n    @classmethod\n    def build_train_loader(cls, cfg):\n        return build_detection_train_loader(cfg, mapper=custom_mapper)\n    \n    @classmethod\n    def build_test_loader(cls, cfg, dataset_name):\n        return build_detection_test_loader(\n            cfg, dataset_name, mapper=custom_mapper\n        )\n\n    @classmethod\n    def build_evaluator(cls, cfg, dataset_name, output_folder=None):\n        if output_folder is None:\n            output_folder = os.path.join(cfg.OUTPUT_DIR, \"inference\")\n        # return PascalVOCDetectionEvaluator(dataset_name)  # not working\n        # return COCOEvaluator(dataset_name, (\"bbox\",), False, output_dir=output_folder)\n        return CovidEvaluator(dataset_name, (\"bbox\",), False, output_dir=output_folder)\n\n    def build_hooks(self):\n        hooks = super(Trainer, self).build_hooks()\n        cfg = self.cfg\n        if len(cfg.DATASETS.TEST) > 0:\n            loss_eval_hook = LossEvalHook(\n                cfg.TEST.EVAL_PERIOD,\n                self.model,\n                Trainer.build_test_loader(cfg, cfg.DATASETS.TEST[0]),\n            )\n            hooks.insert(-1, loss_eval_hook)\n\n        return hooks","metadata":{"execution":{"iopub.status.busy":"2021-07-26T14:19:12.118965Z","iopub.execute_input":"2021-07-26T14:19:12.119400Z","iopub.status.idle":"2021-07-26T14:19:12.128814Z","shell.execute_reply.started":"2021-07-26T14:19:12.119361Z","shell.execute_reply":"2021-07-26T14:19:12.127794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg = get_cfg()\ncfg.merge_from_file(model_zoo.get_config_file(\"COCO-Detection/faster_rcnn_R_101_FPN_3x.yaml\"))\ncfg.DATASETS.TRAIN = (\"covid_train\",)\ncfg.DATASETS.TEST = (\"covid_val\",)\ncfg.DATALOADER.NUM_WORKERS = 2\ncfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-Detection/faster_rcnn_R_101_FPN_3x.yaml\") \ncfg.SOLVER.IMS_PER_BATCH = 4\ncfg.SOLVER.BASE_LR = 0.00025  # pick a good LR\ncfg.SOLVER.MAX_ITER = 10000\ncfg.SOLVER.CHECKPOINT_PERIOD = 2000\ncfg.SOLVER.STEPS = []        # do not decay learning rate\ncfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 256   \ncfg.MODEL.ROI_HEADS.NUM_CLASSES = 1\ncfg.TEST.EVAL_PERIOD = 2000\n\nos.makedirs(cfg.OUTPUT_DIR, exist_ok=True)\ntrainer = Trainer(cfg) \ntrainer.resume_or_load(resume=False)\ntrainer.train()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T14:25:00.151432Z","iopub.execute_input":"2021-07-26T14:25:00.151776Z","iopub.status.idle":"2021-07-26T14:37:18.713555Z","shell.execute_reply.started":"2021-07-26T14:25:00.151743Z","shell.execute_reply":"2021-07-26T14:37:18.708291Z"},"trusted":true},"execution_count":null,"outputs":[]}]}