{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from pathlib import Path\n\nTRAIN_IMG_DIR = Path(\"/kaggle/input/dlsprint2/badlad/images/train\")\nTRAIN_COCO_PATH = Path(\"/kaggle/input/dlsprint2/badlad/labels/coco_format/train/badlad-train-coco.json\")\nTEST_IMG_DIR = Path(\"/kaggle/input/dlsprint2/badlad/images/test\")\nTEST_METADATA_PATH = Path(\"/kaggle/input/dlsprint2/badlad/badlad-test-metadata.json\")","metadata":{"execution":{"iopub.status.busy":"2023-06-23T17:57:48.738911Z","iopub.execute_input":"2023-06-23T17:57:48.739329Z","iopub.status.idle":"2023-06-23T17:57:48.746119Z","shell.execute_reply.started":"2023-06-23T17:57:48.739298Z","shell.execute_reply":"2023-06-23T17:57:48.744750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom tqdm.notebook import tqdm \nimport matplotlib.pyplot as plt\nimport json\nimport cv2\nimport copy\nfrom typing import Optional","metadata":{"execution":{"iopub.status.busy":"2023-06-23T17:57:56.343380Z","iopub.execute_input":"2023-06-23T17:57:56.343750Z","iopub.status.idle":"2023-06-23T17:57:56.632775Z","shell.execute_reply.started":"2023-06-23T17:57:56.343723Z","shell.execute_reply":"2023-06-23T17:57:56.631622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DATALOAD\nFrom [Starter Notebook](https://www.kaggle.com/code/salmankhondker/starter-notebook-dl-sprint-2-0)","metadata":{}},{"cell_type":"code","source":"class DatareadUtil():\n    def __init__(self):\n        self.load_all_data()\n    \n    def _organize_coco_data(self, data_dict: dict) -> tuple[list[str], list[dict], list[dict]]:\n        thing_classes: list[str] = []\n\n        # Map Category Names to IDs\n        for cat in data_dict['categories']:\n            thing_classes.append(cat['name'])\n\n        # Images\n        images_metadata: list[dict] = data_dict['images']\n\n        # Convert COCO annotations to detectron2 annotations format\n        data_annotations = []\n        for ann in data_dict['annotations']:\n            # coco format -> detectron2 format\n            annot_obj = {\n                # Annotation ID\n                \"id\": ann['id'],\n\n                # Segmentation Polygon (x, y) coords\n                \"gt_masks\": ann['segmentation'],\n\n                # Image ID for this annotation (Which image does this annotation belong to?)\n                \"image_id\": ann['image_id'],\n\n                # Category Label (0: paragraph, 1: text box, 2: image, 3: table)\n                \"category_id\": ann['category_id'],\n\n                \"x_min\": ann['bbox'][0],  # left\n                \"y_min\": ann['bbox'][1],  # top\n                \"x_max\": ann['bbox'][0] + ann['bbox'][2],  # left+width\n                \"y_max\": ann['bbox'][1] + ann['bbox'][3]  # top+height\n            }\n            data_annotations.append(annot_obj)\n\n        print(\"DATA LOAD COMPLETE\")\n        return thing_classes, images_metadata, data_annotations\n\n    def load_all_data(self):\n        with TRAIN_COCO_PATH.open() as f:\n            train_dict = json.load(f)\n\n        with TEST_METADATA_PATH.open() as f:\n            test_dict = json.load(f)\n\n        print(\"#### LABELS AND METADATA LOADED ####\")\n        self.thing_classes, self.images_metadata, self.data_annotations = self._organize_coco_data(\n            train_dict\n        )\n\n        self.thing_classes_test, self.images_metadata_test, _ = self._organize_coco_data(\n            test_dict\n        )\n\ndata = DatareadUtil()","metadata":{"execution":{"iopub.status.busy":"2023-06-23T17:58:35.291111Z","iopub.execute_input":"2023-06-23T17:58:35.291554Z","iopub.status.idle":"2023-06-23T17:58:43.165943Z","shell.execute_reply.started":"2023-06-23T17:58:35.291521Z","shell.execute_reply":"2023-06-23T17:58:43.164715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.thing_classes","metadata":{"execution":{"iopub.status.busy":"2023-06-23T18:03:24.071793Z","iopub.execute_input":"2023-06-23T18:03:24.072200Z","iopub.status.idle":"2023-06-23T18:03:24.080699Z","shell.execute_reply.started":"2023-06-23T18:03:24.072158Z","shell.execute_reply":"2023-06-23T18:03:24.079060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata = pd.DataFrame(data.images_metadata)\ntrain_metadata = train_metadata[['id', 'file_name', 'width', 'height']]\ntrain_metadata = train_metadata.rename(columns={\"id\": \"image_id\"})\nprint(\"train_metadata size=\", len(train_metadata))\ntrain_metadata.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-06-23T17:58:46.875442Z","iopub.execute_input":"2023-06-23T17:58:46.875829Z","iopub.status.idle":"2023-06-23T17:58:47.012768Z","shell.execute_reply.started":"2023-06-23T17:58:46.875798Z","shell.execute_reply":"2023-06-23T17:58:47.006655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_annot_df = pd.DataFrame(data.data_annotations)\nprint(\"train_annot_df size=\", len(train_annot_df))\ntrain_annot_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-06-23T17:58:49.841003Z","iopub.execute_input":"2023-06-23T17:58:49.841409Z","iopub.status.idle":"2023-06-23T17:58:51.481311Z","shell.execute_reply.started":"2023-06-23T17:58:49.841377Z","shell.execute_reply":"2023-06-23T17:58:51.480241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_metadata = pd.DataFrame(data.images_metadata_test)\ntest_metadata = test_metadata[['id', 'file_name', 'width', 'height']]\ntest_metadata = test_metadata.rename(columns={\"id\": \"image_id\"})\nprint(\"test_metadata size=\", len(test_metadata))\ntest_metadata.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-06-23T17:58:53.525625Z","iopub.execute_input":"2023-06-23T17:58:53.526072Z","iopub.status.idle":"2023-06-23T17:58:53.589738Z","shell.execute_reply.started":"2023-06-23T17:58:53.526038Z","shell.execute_reply":"2023-06-23T17:58:53.588421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Classwise Distribution","metadata":{}},{"cell_type":"code","source":"imageAnnotClass = train_annot_df[['image_id','category_id']].copy()\nimageAnnotClass[data.thing_classes[0]] = imageAnnotClass['category_id'] == 0\nimageAnnotClass[data.thing_classes[1]] = imageAnnotClass['category_id'] == 1\nimageAnnotClass[data.thing_classes[2]] = imageAnnotClass['category_id'] == 2\nimageAnnotClass[data.thing_classes[3]] = imageAnnotClass['category_id'] == 3\ndel imageAnnotClass['category_id']\n\nimageAnnotClass = imageAnnotClass.groupby('image_id').sum()\nimageAnnotClass.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-23T17:58:56.025560Z","iopub.execute_input":"2023-06-23T17:58:56.025935Z","iopub.status.idle":"2023-06-23T17:58:56.096520Z","shell.execute_reply.started":"2023-06-23T17:58:56.025907Z","shell.execute_reply":"2023-06-23T17:58:56.095248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imageAnnotClass.describe()","metadata":{"execution":{"iopub.status.busy":"2023-06-23T17:58:59.769060Z","iopub.execute_input":"2023-06-23T17:58:59.769487Z","iopub.status.idle":"2023-06-23T17:58:59.800657Z","shell.execute_reply.started":"2023-06-23T17:58:59.769456Z","shell.execute_reply":"2023-06-23T17:58:59.799519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imageAnnotClass.plot(subplots=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-23T17:59:13.959166Z","iopub.execute_input":"2023-06-23T17:59:13.959583Z","iopub.status.idle":"2023-06-23T17:59:15.288537Z","shell.execute_reply.started":"2023-06-23T17:59:13.959553Z","shell.execute_reply":"2023-06-23T17:59:15.287232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imageAnnotClass.plot(kind='kde', subplots=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-23T17:59:19.850843Z","iopub.execute_input":"2023-06-23T17:59:19.851280Z","iopub.status.idle":"2023-06-23T17:59:23.169653Z","shell.execute_reply.started":"2023-06-23T17:59:19.851246Z","shell.execute_reply":"2023-06-23T17:59:23.168342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imageAnnotClass.plot(kind='box', subplots=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-23T17:59:32.846289Z","iopub.execute_input":"2023-06-23T17:59:32.846711Z","iopub.status.idle":"2023-06-23T17:59:33.485236Z","shell.execute_reply.started":"2023-06-23T17:59:32.846679Z","shell.execute_reply":"2023-06-23T17:59:33.484284Z"},"trusted":true},"execution_count":null,"outputs":[]}]}