{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Using Detectron2\n\n[Detectron2](https://detectron2.readthedocs.io/en/latest/index.html) is a popular open-source software library developed by Facebook AI Research (FAIR) for building computer vision models. It serves as a powerful framework for object detection, instance segmentation, and keypoint detection tasks. Detectron2 is built on top of PyTorch, geared towards a more convenient way to build modular, flexible pipelines for specific Computer Vision Tasks such as object detection, instance segmentation.","metadata":{}},{"cell_type":"markdown","source":"# 1 Installing detectron2","metadata":{}},{"cell_type":"code","source":"%%capture\nimport sys, os, distutils.core\n# Note: This is a faster way to install detectron2 in Colab, but it does not include all functionalities (e.g. compiled operators).\n# See https://detectron2.readthedocs.io/tutorials/install.html for full installation instructions\n!git clone 'https://github.com/facebookresearch/detectron2'\ndist = distutils.core.run_setup(\"./detectron2/setup.py\")\n!python -m pip install {' '.join([f\"'{x}'\" for x in dist.install_requires])}\nsys.path.insert(0, os.path.abspath('./detectron2'))","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:35:52.083128Z","iopub.execute_input":"2023-08-06T15:35:52.08347Z","iopub.status.idle":"2023-08-06T15:36:46.361042Z","shell.execute_reply.started":"2023-08-06T15:35:52.08344Z","shell.execute_reply":"2023-08-06T15:36:46.359647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2 Notebook Config","metadata":{}},{"cell_type":"markdown","source":"## 2.1 Decisions","metadata":{}},{"cell_type":"code","source":"from datetime import datetime\n\n# if False, model is set to `PRETRAINED_PATH` model\nis_train = False\n\n# if True, evaluate on validation dataset\nis_evaluate = False\n\n# if True, run inference on test dataset\nis_inference = True\n\n# if True and `is_train` == True, `PRETRAINED_PATH` model is trained further\nis_resume_training = True\n\n# Perform augmentation\nis_augment = True\n\n\nSEED = int(datetime.now().timestamp())\n\n# FIXED SEED VALUE is used for reproducing similar results, THIS VALUE WAS USED in best submission\n#SEED = 1690218831","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:36:46.363946Z","iopub.execute_input":"2023-08-06T15:36:46.365101Z","iopub.status.idle":"2023-08-06T15:36:46.376361Z","shell.execute_reply.started":"2023-08-06T15:36:46.365054Z","shell.execute_reply":"2023-08-06T15:36:46.373226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.2 Paths","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\n\nTRAIN_IMG_DIR = Path(\"/kaggle/input/dlsprint2/badlad/images/train\")\n\nTRAIN_COCO_PATH = Path(\"/kaggle/input/dlsprint2/badlad/labels/coco_format/train/badlad-train-coco.json\")\n\nTEST_IMG_DIR = Path(\"/kaggle/input/dlsprint2/badlad/images/test\")\n\nTEST_METADATA_PATH = Path(\"/kaggle/input/dlsprint2/badlad/badlad-test-metadata.json\")\n\n# Training output directory\nOUTPUT_DIR = Path(\"./output\")\nOUTPUT_MODEL = OUTPUT_DIR/\"model_final.pth\"\n\n# Path to pretrained model weights\nPRETRAINED_PATH = '/kaggle/input/pretrained-badlad-theemptymodel/model_final.pth'","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:36:46.379789Z","iopub.execute_input":"2023-08-06T15:36:46.380152Z","iopub.status.idle":"2023-08-06T15:36:46.392887Z","shell.execute_reply.started":"2023-08-06T15:36:46.380119Z","shell.execute_reply":"2023-08-06T15:36:46.391732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model path based on Decisions\nMODEL_PATH = OUTPUT_MODEL if is_train else PRETRAINED_PATH","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:36:46.397224Z","iopub.execute_input":"2023-08-06T15:36:46.397601Z","iopub.status.idle":"2023-08-06T15:36:46.404642Z","shell.execute_reply.started":"2023-08-06T15:36:46.397574Z","shell.execute_reply":"2023-08-06T15:36:46.403558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.3 imports","metadata":{}},{"cell_type":"code","source":"# detectron2\nfrom detectron2.utils.memory import retry_if_cuda_oom\nfrom detectron2.utils.logger import setup_logger\nfrom detectron2.checkpoint import DetectionCheckpointer\nfrom detectron2.modeling import build_model\nfrom detectron2.evaluation import COCOEvaluator, inference_on_dataset\nfrom detectron2.engine.hooks import HookBase\nimport detectron2.data.transforms as T\nfrom detectron2.data import detection_utils as utils\nfrom detectron2.data import DatasetCatalog, MetadataCatalog, build_detection_test_loader, build_detection_train_loader, DatasetMapper\nfrom detectron2.utils.visualizer import Visualizer\nfrom detectron2.utils.visualizer import ColorMode\nfrom detectron2.structures import BoxMode\nfrom detectron2.engine import DefaultPredictor, DefaultTrainer\nfrom detectron2.config import get_cfg\nfrom detectron2 import model_zoo\n\nimport pandas as pd\nimport numpy as np\nfrom tqdm.notebook import tqdm  # progress bar\nimport matplotlib.pyplot as plt\nimport json\nimport cv2\nimport copy\nfrom typing import Optional\n\nfrom IPython.display import FileLink\n\n# torch\nimport torch\n\nimport gc\n\nimport warnings\n# Ignore \"future\" warnings and Data-Frame-Slicing warnings.\nwarnings.filterwarnings('ignore')\n\nsetup_logger()","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:36:46.406221Z","iopub.execute_input":"2023-08-06T15:36:46.40659Z","iopub.status.idle":"2023-08-06T15:36:47.38066Z","shell.execute_reply.started":"2023-08-06T15:36:46.406559Z","shell.execute_reply":"2023-08-06T15:36:47.379571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3 COCO Annotations Data","metadata":{}},{"cell_type":"markdown","source":"## 3.1 Load","metadata":{}},{"cell_type":"code","source":"with TRAIN_COCO_PATH.open() as f:\n    train_dict = json.load(f)\n\nwith TEST_METADATA_PATH.open() as f:\n    test_dict = json.load(f)\n\nprint(\"#### LABELS AND METADATA LOADED ####\")","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:36:47.382473Z","iopub.execute_input":"2023-08-06T15:36:47.382845Z","iopub.status.idle":"2023-08-06T15:36:54.111261Z","shell.execute_reply.started":"2023-08-06T15:36:47.38281Z","shell.execute_reply":"2023-08-06T15:36:54.110256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.2 Observing Data","metadata":{}},{"cell_type":"code","source":"def organize_coco_data(data_dict: dict) -> tuple[list[str], list[dict], list[dict]]:\n    thing_classes: list[str] = []\n    \n    # Map Category Names to IDs\n    for cat in data_dict['categories']:\n        thing_classes.append(cat['name'])\n\n    # Images\n    images_metadata: list[dict] = data_dict['images']\n\n    # Convert COCO annotations to detectron2 annotations format\n    data_annotations = []\n    for ann in data_dict['annotations']:\n        # coco format -> detectron2 format\n        annot_obj = {\n            # Annotation ID\n            \"id\": ann['id'],\n\n            # Segmentation Polygon (x, y) coords\n            \"gt_masks\": ann['segmentation'],\n\n            # Image ID for this annotation (Which image does this annotation belong to?)\n            \"image_id\": ann['image_id'],\n\n            # Category Label (0: paragraph, 1: text box, 2: image, 3: table)\n            \"category_id\": ann['category_id'],\n\n            \"x_min\": ann['bbox'][0],  # left\n            \"y_min\": ann['bbox'][1],  # top\n            \"x_max\": ann['bbox'][0] + ann['bbox'][2],  # left+width\n            \"y_max\": ann['bbox'][1] + ann['bbox'][3]  # top+height\n        }\n        data_annotations.append(annot_obj)\n\n    return thing_classes, images_metadata, data_annotations","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:36:54.113554Z","iopub.execute_input":"2023-08-06T15:36:54.114141Z","iopub.status.idle":"2023-08-06T15:36:54.122479Z","shell.execute_reply.started":"2023-08-06T15:36:54.114106Z","shell.execute_reply":"2023-08-06T15:36:54.121462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"thing_classes, images_metadata, data_annotations = organize_coco_data(\n    train_dict\n)\n\nthing_classes_test, images_metadata_test, _ = organize_coco_data(\n    test_dict\n)","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:36:54.12396Z","iopub.execute_input":"2023-08-06T15:36:54.124473Z","iopub.status.idle":"2023-08-06T15:36:55.49266Z","shell.execute_reply.started":"2023-08-06T15:36:54.124442Z","shell.execute_reply":"2023-08-06T15:36:55.491595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"These are the categories we are going to detect.","metadata":{}},{"cell_type":"code","source":"print(thing_classes)","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:36:55.494364Z","iopub.execute_input":"2023-08-06T15:36:55.495093Z","iopub.status.idle":"2023-08-06T15:36:55.500686Z","shell.execute_reply.started":"2023-08-06T15:36:55.495057Z","shell.execute_reply":"2023-08-06T15:36:55.499533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata = pd.DataFrame(images_metadata)\ntrain_metadata = train_metadata[['id', 'file_name', 'width', 'height']]\ntrain_metadata = train_metadata.rename(columns={\"id\": \"image_id\"})\nprint(\"train_metadata size=\", len(train_metadata))\ntrain_metadata.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:36:55.505798Z","iopub.execute_input":"2023-08-06T15:36:55.507175Z","iopub.status.idle":"2023-08-06T15:36:55.593998Z","shell.execute_reply.started":"2023-08-06T15:36:55.507132Z","shell.execute_reply":"2023-08-06T15:36:55.593086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_annot_df = pd.DataFrame(data_annotations)\nprint(\"train_annot_df size=\", len(train_annot_df))\ntrain_annot_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:36:55.595553Z","iopub.execute_input":"2023-08-06T15:36:55.596218Z","iopub.status.idle":"2023-08-06T15:36:57.004325Z","shell.execute_reply.started":"2023-08-06T15:36:55.596185Z","shell.execute_reply":"2023-08-06T15:36:57.003304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here `gt_masks` are the sequence of `(x, y)` coordinates of vertices of the polygon surrounding the target object. ","metadata":{}},{"cell_type":"code","source":"test_metadata = pd.DataFrame(images_metadata_test)\ntest_metadata = test_metadata[['id', 'file_name', 'width', 'height']]\ntest_metadata = test_metadata.rename(columns={\"id\": \"image_id\"})\nprint(\"test_metadata size=\", len(test_metadata))\ntest_metadata.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:36:57.005609Z","iopub.execute_input":"2023-08-06T15:36:57.006757Z","iopub.status.idle":"2023-08-06T15:36:57.060658Z","shell.execute_reply.started":"2023-08-06T15:36:57.006723Z","shell.execute_reply":"2023-08-06T15:36:57.059761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4 Preparing Data for Training","metadata":{}},{"cell_type":"markdown","source":"## 4.1 Train-Validation Split","metadata":{}},{"cell_type":"code","source":"TRAIN_SPLIT = 0.965","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:36:57.062061Z","iopub.execute_input":"2023-08-06T15:36:57.065129Z","iopub.status.idle":"2023-08-06T15:36:57.071337Z","shell.execute_reply.started":"2023-08-06T15:36:57.065092Z","shell.execute_reply":"2023-08-06T15:36:57.069599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_dataset = len(train_metadata)\nn_train = int(n_dataset * TRAIN_SPLIT)\nprint(\"n_dataset\", n_dataset, \"n_train\", n_train, \"n_val\", n_dataset-n_train)\n\nnp.random.seed(SEED)\n\ninds = np.random.permutation(n_dataset)\ntrain_inds, valid_inds = inds[:n_train], inds[n_train:]","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:36:57.073616Z","iopub.execute_input":"2023-08-06T15:36:57.074562Z","iopub.status.idle":"2023-08-06T15:36:57.084748Z","shell.execute_reply.started":"2023-08-06T15:36:57.074502Z","shell.execute_reply":"2023-08-06T15:36:57.083647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4.2 Formatting Data for `detectron2` and Selective Sampling","metadata":{}},{"cell_type":"code","source":"def convert_coco_to_detectron2_format(\n    imgdir: Path,\n    metadata_df: pd.DataFrame,\n    train: True,\n    annot_df: Optional[pd.DataFrame] = None,\n    target_indices: Optional[np.ndarray] = None,\n):\n\n\n    dataset_dicts = []\n    for _, train_meta_row in tqdm(metadata_df.iterrows(), total=len(metadata_df)):\n        # Iterate over each image\n        image_id, filename, width, height = train_meta_row.values\n\n        annotations = []\n\n        # too large text or too small text\n        weird_text = False\n        \n        # if the data has image\n        has_image = False\n        \n        #if the data has table\n        has_table = False\n        \n        \n        # number of paragraphs, texts, tables, and weird texts in the data (single data)\n        num_of_paras = 0\n        num_of_texts = 0\n        num_of_tables = 0\n        num_of_imgs = 0\n        num_of_wtexts = 0\n        \n        # If train/validation data, then there will be annotations\n        if annot_df is not None:\n            for _, ann in annot_df.query(\"image_id == @image_id\").iterrows():\n                # Get annotations of current iteration's image\n                class_id = ann[\"category_id\"]\n                \n                rd = np.random.random()\n                \n                c = int(class_id)\n                if c == 0:\n                    num_of_paras +=1\n                elif c == 1:\n                    num_of_texts += 1\n                elif c == 2:\n                    num_of_imgs += 1\n                    has_image = True\n                else:\n                    num_of_tables += 1\n                    has_table = True\n                    \n                    \n                gt_masks = ann[\"gt_masks\"]\n                bbox_resized = [\n                    float(ann[\"x_min\"]),\n                    float(ann[\"y_min\"]),\n                    float(ann[\"x_max\"]),\n                    float(ann[\"y_max\"]),\n                ]\n\n                \n                # detect if there are texts which are too large/ too small or texts with aspect ratio of too large/ small\n                if c == 1:\n                    width_ann = (bbox_resized[2]-bbox_resized[0])\n                    height_ann = (bbox_resized[3]-bbox_resized[1])\n                    rat = width_ann / height_ann\n                    \n                    #weird dimension (ASPECT RATIO)\n                    if rat > 15 or rat < 1/15:\n                        weird_text = True\n                        \n                    rat = (width_ann * height_ann) / (width * height)\n                    # too large text or too small text compared to IMAGE SIZE\n                    if rat > 0.12 or rat < 0.0025:\n                        weird_text = True\n                \n                    \n                annotation = {\n                    \"bbox\": bbox_resized,\n                    \"bbox_mode\": BoxMode.XYXY_ABS,\n                    \"segmentation\": gt_masks,\n                    \"category_id\": class_id,\n                }\n\n                annotations.append(annotation)\n\n        # coco format -> detectron2 format dict\n        record = {\n            \"file_name\": str(imgdir/filename),\n            \"image_id\": image_id,\n            \"width\": width,\n            \"height\": height,\n            \"annotations\": annotations\n        }\n        \n        \n        # if this contains image/ table randomly/willingly duplicate/ triplet (ONLY TRAINING DATA)\n        if train:\n                \n            #append image, table, weird texts more (large text box or wide text boxes)\n            if has_image or has_table or weird_text:\n                \n                # OVER SAMPLING DATA Containing Tables\n                if has_table:\n                    for i in range(8):\n                        dataset_dicts.append(record)\n                        \n                        \n                # OVER SAMPLING DATA Containing Images\n                elif has_image:\n                    \n                    #how many times to over sample\n                    to_add = 2\n                    \n                    # IF There are more images, Oversample more\n                    if num_of_imgs > 10:\n                        to_add = 4\n                    elif num_of_imgs > 4:\n                        to_add = 3\n                        \n                    for i in range(to_add):\n                        dataset_dicts.append(record)\n                        \n                # OVER SAMPLING DATA Containing weird text boxes\n                else:\n                    dataset_dicts.append(record)\n                    \n            # NORMAL Data Appending\n            else:\n                rd = np.random.random()\n                \n                # Append Large Images more so that they can be used in Augmentations/ Cropped to train\n                if (width > 3500 or height > 3500) and rd > 0.6:\n                    for i in range(3):\n                        dataset_dicts.append(record)\n                \n                \n                # drop 15-20% Data (unless Data contains many paragraphs)\n                if rd > 0.2 or num_of_paras > 9:\n                    dataset_dicts.append(record)\n        \n        # TEST / INFERENCE DATA (Single append)\n        else:\n            dataset_dicts.append(record)\n\n        \n\n    if target_indices is not None and not train:\n        dataset_dicts = [dataset_dicts[i] for i in target_indices]\n\n    print(\"Actual Train Size: \", len(dataset_dicts))\n    return dataset_dicts","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:36:57.086171Z","iopub.execute_input":"2023-08-06T15:36:57.086996Z","iopub.status.idle":"2023-08-06T15:36:57.107993Z","shell.execute_reply.started":"2023-08-06T15:36:57.086961Z","shell.execute_reply":"2023-08-06T15:36:57.106947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4.3 Registering and Loading Data for `detectron2`","metadata":{}},{"cell_type":"code","source":"DATA_REGISTER_TRAINING = \"badlad_train\"\nDATA_REGISTER_VALID    = \"badlad_valid\"\nDATA_REGISTER_TEST     = \"badlad_test\"","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:36:57.109817Z","iopub.execute_input":"2023-08-06T15:36:57.110146Z","iopub.status.idle":"2023-08-06T15:36:57.12395Z","shell.execute_reply.started":"2023-08-06T15:36:57.110115Z","shell.execute_reply":"2023-08-06T15:36:57.123038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Register Training data\nif is_train:\n    DatasetCatalog.register(\n        DATA_REGISTER_TRAINING,\n        lambda: convert_coco_to_detectron2_format(\n            TRAIN_IMG_DIR,\n            train_metadata,\n            True,\n            train_annot_df,\n            target_indices=train_inds,\n        ),\n    )\n\n    # Set Training data categories\n    MetadataCatalog.get(DATA_REGISTER_TRAINING).set(thing_classes=thing_classes)\n\n    dataset_dicts_train = DatasetCatalog.get(DATA_REGISTER_TRAINING)\n    metadata_dicts_train = MetadataCatalog.get(DATA_REGISTER_TRAINING)\n\n    print(\"dicts training size=\", len(dataset_dicts_train))\n    print(\"################\")","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:36:57.125832Z","iopub.execute_input":"2023-08-06T15:36:57.126256Z","iopub.status.idle":"2023-08-06T15:38:39.985501Z","shell.execute_reply.started":"2023-08-06T15:36:57.126224Z","shell.execute_reply":"2023-08-06T15:38:39.984544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sample Training Data Visualization with Annotations","metadata":{}},{"cell_type":"code","source":"if is_train:\n    fig = plt.figure(figsize =(20,20))\n    ax = fig.add_subplot(1,1,1)\n    d=dataset_dicts_train[np.random.randint(20000)]\n    img = cv2.imread(d[\"file_name\"]) \n    v = Visualizer(img[:, :, ::-1], metadata=train_metadata, instance_mode=ColorMode.IMAGE_BW) # remove the colors of unsegmented pixels. This option is only available for segmentation models \n    out = v.draw_dataset_dict(d) \n    ax.grid(False) \n    ax.axis('off') \n    ax.imshow(out.get_image()[:, :, ::-1])","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:38:39.98738Z","iopub.execute_input":"2023-08-06T15:38:39.988042Z","iopub.status.idle":"2023-08-06T15:38:41.444611Z","shell.execute_reply.started":"2023-08-06T15:38:39.988007Z","shell.execute_reply":"2023-08-06T15:38:41.443225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Register Validation data\nif is_train or is_evaluate:\n    DatasetCatalog.register(\n        DATA_REGISTER_VALID,\n        lambda: convert_coco_to_detectron2_format(\n            TRAIN_IMG_DIR,\n            train_metadata,\n            False,\n            train_annot_df,\n            target_indices=valid_inds,\n        ),\n    )\n\n    # Set Validation data categories\n    MetadataCatalog.get(DATA_REGISTER_VALID).set(thing_classes=thing_classes)\n\n    dataset_dicts_valid = DatasetCatalog.get(DATA_REGISTER_VALID)\n    metadata_dicts_valid = MetadataCatalog.get(DATA_REGISTER_VALID)\n\n    print(\"dicts valid size=\", len(dataset_dicts_valid))\n    print(\"################\")","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:38:41.445828Z","iopub.execute_input":"2023-08-06T15:38:41.446148Z","iopub.status.idle":"2023-08-06T15:40:23.455766Z","shell.execute_reply.started":"2023-08-06T15:38:41.446119Z","shell.execute_reply":"2023-08-06T15:40:23.454776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Register Test Inference data\nDatasetCatalog.register(\n    DATA_REGISTER_TEST,\n    lambda: convert_coco_to_detectron2_format(\n        TEST_IMG_DIR,\n        test_metadata,\n        False\n    )\n)\n\n# Set Test data categories\nMetadataCatalog.get(DATA_REGISTER_TEST).set(\n    thing_classes=thing_classes_test\n)\n\ndataset_dicts_test = DatasetCatalog.get(DATA_REGISTER_TEST)\nmetadata_dicts_test = MetadataCatalog.get(DATA_REGISTER_TEST)","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:23.457144Z","iopub.execute_input":"2023-08-06T15:40:23.457803Z","iopub.status.idle":"2023-08-06T15:40:24.312183Z","shell.execute_reply.started":"2023-08-06T15:40:23.457767Z","shell.execute_reply":"2023-08-06T15:40:24.311083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"#### DATA REGISTERED ####\")","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:24.313599Z","iopub.execute_input":"2023-08-06T15:40:24.314036Z","iopub.status.idle":"2023-08-06T15:40:24.320087Z","shell.execute_reply.started":"2023-08-06T15:40:24.314002Z","shell.execute_reply":"2023-08-06T15:40:24.31908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5 Augmentation","metadata":{}},{"cell_type":"code","source":"def custom_mapper(dataset_dict):\n    dataset_dict = copy.deepcopy(dataset_dict)\n    image = utils.read_image(dataset_dict[\"file_name\"], format=\"BGR\")\n    h = image.shape[0]\n    w = image.shape[1]\n    \n    transform_list = []\n    \n    \n    # Randomly Orient (Landscape / Portrait)\n    rd = np.random.random()\n    \n    # CCW or CW Rotation\n    rotation_angle = 90\n    if rd > 0.85:\n        rotation_angle = -90\n    \n    # To change Orientation or not\n    rd = np.random.random()\n    if rd > 0.98:\n        transform_list.append(T.RotationTransform(h, w, rotation_angle))\n        w, h = h, w\n    \n    # Lower High Resolution Images\n    if w > 3500 or h > 3500:\n        transform_list.append(T.ResizeShortestEdge(3500))\n        \n        \n    # Sometimes lower the resolution very much so that MODEL is generalized in predicting lower resolution PICTURES\n    if rd > 0.965:\n        transform_list.append(T.ResizeShortestEdge(400))\n        \n        \n    #randomly Resize or Crop (Crop for Large Image only)\n    maxD = 1024.0  # MAXIMUM ALLOWED dimension size for resize\n    maxC = 800     # Cropping dimension size (1:1 aspect ratio)\n    \n    \n    # Minimum dimension has to be 800 => So if for a picture with longer dimension 1024 has the lower dimension crossing 800, \n    lim = min(w, h) * maxD / max(w, h)\n    if lim > 800:\n        maxD = max(w,h) * 800 / min(w, h)\n            \n    \n    rd = np.random.random()\n    \n    if w < 1500 or h < 1500 or rd > 0.15:\n        if max(w, h) > maxD:\n            if w > h:\n                new_w = maxD\n                new_h = maxD * h / w\n            else:\n                new_h = maxD\n                new_w = maxD * w / h\n\n            \n            # Shrinking Width of The IMAGE (So that for paragraphs, each individual word is not detected as a text box)\n            new_w = int(new_w * 0.8)\n            new_h = int(new_h)\n\n            transform_list.append(T.Resize((new_h,  new_w)))\n            \n    # FOR LARGE IMAGES, SOMETIMES CROP instead of resize\n    else:\n        transform_list.append(T.FixedSizeCrop((maxC, maxC)))\n        \n        \n    # AUGMENTATIONS\n    transform_list.extend([T.RandomBrightness(0.8, 1.15),\n                           T.RandomSaturation(0.8, 1.3),\n                           T.RandomContrast(0.9, 1.55),\n                           #T.RandomFlip(prob=0.5, horizontal=False, vertical=True),\n                           T.RandomFlip(prob=0.15, horizontal=True, vertical=False),\n                           T.RandomRotation(-3, 3)\n                          ])\n    \n    image, transforms = T.apply_transform_gens(transform_list, image)\n    \n    \n    dataset_dict[\"image\"] = torch.as_tensor(\n        image.transpose(2, 0, 1).astype(\"float32\"))\n\n    annos = [\n        utils.transform_instance_annotations(obj, transforms, image.shape[:2])\n        for obj in dataset_dict.pop(\"annotations\")\n    ]\n    instances = utils.annotations_to_instances(annos, image.shape[:2])\n\n    dataset_dict[\"instances\"] = utils.filter_empty_instances(instances)\n\n    return dataset_dict","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:24.322042Z","iopub.execute_input":"2023-08-06T15:40:24.32278Z","iopub.status.idle":"2023-08-06T15:40:24.345675Z","shell.execute_reply.started":"2023-08-06T15:40:24.322748Z","shell.execute_reply":"2023-08-06T15:40:24.344793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Change frequency of loss eval hook","metadata":{}},{"cell_type":"code","source":"class AugTrainer(DefaultTrainer):\n    @classmethod\n    def build_train_loader(cls, cfg):\n        return build_detection_train_loader(cfg, mapper=custom_mapper)\n    \n    def build_evaluator(cls, cfg, dataset_name, output_folder=None):\n        if output_folder is None:\n            output_folder = os.path.join(cfg.OUTPUT_DIR, \"test\")\n        return COCOEvaluator(dataset_name,(\"segm\",), True, output_folder)","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:45:21.238083Z","iopub.execute_input":"2023-08-06T15:45:21.238464Z","iopub.status.idle":"2023-08-06T15:45:21.249422Z","shell.execute_reply.started":"2023-08-06T15:45:21.238432Z","shell.execute_reply":"2023-08-06T15:45:21.24842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6 Hyperparameters\n### (use 10k iterations. Since Kaggle has limit on session run, first a model is pretrained on 5k iterations, then again train on 5k iterations.)","metadata":{}},{"cell_type":"markdown","source":"Detectron2 models need a config file to build the model. This config file has the associated hyperparameters.","metadata":{}},{"cell_type":"code","source":"if is_train:\n    cfg = get_cfg()\n\n    # config_name = \"COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml\"\n    config_name = \"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"\n\n    cfg.merge_from_file(model_zoo.get_config_file(config_name))\n\n    cfg.DATASETS.TRAIN = (DATA_REGISTER_TRAINING,)\n    cfg.DATASETS.TEST = (DATA_REGISTER_VALID,)\n\n    # to evaluate during training, you have to implement `build_evaluator()` method of the trainer.\n    # https://github.com/facebookresearch/detectron2/blob/94113be6e12db36b8c7601e13747587f19ec92fe/detectron2/engine/defaults.py#L561\n    \n    cfg.TEST.EVAL_PERIOD = 80\n\n    cfg.DATALOADER.NUM_WORKERS = 1\n\n    cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(config_name)\n    if (is_resume_training):\n        print(\"#### SETTING PRETRAINED WEIGHTS TO RESUME TRAINING ####\")\n        cfg.MODEL.WEIGHTS = str(PRETRAINED_PATH)\n    else:\n        print(\"#### TRAINING MODEL FROM SCRATCH ####\")\n    \n    cfg.MODEL.RPN.POST_NMS_TOPK_TRAIN = 2000\n    cfg.MODEL.RPN.PRE_NMS_TOPK_TRAIN = 4000\n    \n    cfg.SOLVER.AMP.ENABLED = True\n    cfg.SOLVER.IMS_PER_BATCH = 12\n    \n    # when reiterively trained (First 0.008 is used for quicker convergence, Next time trained on Pretrained model, lower LR (0.0005) is used)\n    cfg.SOLVER.BASE_LR = 0.0005\n\n    cfg.SOLVER.WARMUP_ITERS = 20\n\n    # Maximum number of iterations (THE MODEL is iteratively trained for more iterations since Kaggle has limits on Session Run Time. Best Submission had 13.5k iterations.)\n    cfg.SOLVER.MAX_ITER = 200  # FOR ACTUAL TRAINING 5k iterations\n    \n    cfg.SOLVER.STEPS = (2000, 5000) # must be less than MAX_ITER\n    cfg.SOLVER.GAMMA = 0.25\n\n    # Small value == Frequent save need a lot of storage.\n    cfg.SOLVER.CHECKPOINT_PERIOD = 20000\n    cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 256\n    cfg.MODEL.ROI_HEADS.NUM_CLASSES = 4\n    \n    # ADDED MORE CONVOLUTION LAYERS for both MASK HEAD and BOX Head for better generalization\n    cfg.MODEL.ROI_MASK_HEAD.NUM_CONV = 4\n    cfg.MODEL.ROI_BOX_HEAD.NUM_CONV = 4\n    \n    cfg.MODEL.ROI_BOX_HEAD.POOLER_SAMPLING_RATIO = 2\n    cfg.MODEL.ROI_MASK_HEAD.POOLER_SAMPLING_RATIO = 2\n    \n    \n    # Create Output Directory\n    cfg.OUTPUT_DIR = str(OUTPUT_DIR)\n    print(\"creating cfg.OUTPUT_DIR -> \", cfg.OUTPUT_DIR)\n    OUTPUT_DIR.mkdir(exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:45:24.007714Z","iopub.execute_input":"2023-08-06T15:45:24.008067Z","iopub.status.idle":"2023-08-06T15:45:24.038296Z","shell.execute_reply.started":"2023-08-06T15:45:24.008036Z","shell.execute_reply":"2023-08-06T15:45:24.03719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 7 Training","metadata":{}},{"cell_type":"markdown","source":"## 7.1 Training the model","metadata":{}},{"cell_type":"code","source":"if is_train:\n    trainer = DefaultTrainer(cfg) if not is_augment else AugTrainer(cfg)\n        \n    trainer.resume_or_load(resume=is_resume_training)\n\n    trainer.train()\n    \n    print(\"#### TRAINING COMPLETE ####\")\n    _ = trainer.model.train(False)  # turn off training\n    \n    FileLink(str(OUTPUT_MODEL))","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:45:27.758581Z","iopub.execute_input":"2023-08-06T15:45:27.759605Z","iopub.status.idle":"2023-08-06T15:51:02.862291Z","shell.execute_reply.started":"2023-08-06T15:45:27.759563Z","shell.execute_reply":"2023-08-06T15:51:02.860381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 7.2 Visualizing Training Metrics","metadata":{}},{"cell_type":"code","source":"if is_train:\n    # Load metrics\n    metrics_df = pd.read_json(\n        OUTPUT_DIR/\"metrics.json\", orient=\"records\", lines=True\n    )\n    mdf = metrics_df.sort_values(\"iteration\")\n    print(mdf.head(10).T)\n\n    # Plot loss\n    fig, ax = plt.subplots()\n\n    mdf1 = mdf[~mdf[\"total_loss\"].isna()]\n    ax.plot(mdf1[\"iteration\"], mdf1[\"total_loss\"], c=\"C0\", label=\"train\")\n\n    if \"validation_loss\" in mdf.columns:\n        mdf2 = mdf[~mdf[\"validation_loss\"].isna()]\n        ax.plot(mdf2[\"iteration\"], mdf2[\"validation_loss\"],\n                c=\"C1\", label=\"validation\")\n\n    ax.legend()\n    ax.set_title(\"Loss curve\")\n    plt.show()\n\n    # Plot Accuracy\n    fig, ax = plt.subplots()\n\n    mdf1 = mdf[~mdf[\"fast_rcnn/cls_accuracy\"].isna()]\n    ax.plot(mdf1[\"iteration\"], mdf1[\"fast_rcnn/cls_accuracy\"],\n            c=\"C0\", label=\"train\")\n\n    ax.legend()\n    ax.set_title(\"Accuracy curve\")\n    plt.show()\n\n    # Plot Bounding Box regressor loss\n    fig, ax = plt.subplots()\n\n    mdf1 = mdf[~mdf[\"loss_box_reg\"].isna()]\n    ax.plot(mdf1[\"iteration\"], mdf1[\"loss_box_reg\"], c=\"C0\", label=\"train\")\n\n    ax.legend()\n    ax.set_title(\"loss_box_reg\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:24.663187Z","iopub.status.idle":"2023-08-06T15:40:24.663661Z","shell.execute_reply.started":"2023-08-06T15:40:24.663412Z","shell.execute_reply":"2023-08-06T15:40:24.663434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 8 Evaluation","metadata":{}},{"cell_type":"code","source":"def custom_mapper_eval(dataset_dict):\n    dataset_dict = copy.deepcopy(dataset_dict)\n    image = utils.read_image(dataset_dict[\"file_name\"], format=\"BGR\")\n    h = image.shape[0]\n    w = image.shape[1]\n    new_h = new_w = 1\n    \n    transform_list = []\n    \n    maxD = 1024.0\n    \n    # minimum dimension has to be 800\n    lim = min(w, h) * maxD / max(w, h)\n    if lim > 800:\n        maxD = max(w,h) * 800 / min(w, h)\n            \n    if max(w, h) > maxD:\n        if w > h:\n            new_w = maxD\n            new_h = maxD * h / w\n        else:\n            new_h = maxD\n            new_w = maxD * w/ h\n\n        new_w = new_w * 0.8\n        new_h = int(new_h)\n        new_w = int(new_w)\n\n        transform_list = [T.Resize((new_h,  new_w))]\n        \n    \n    transform_list.extend([T.RandomBrightness(0.88, 0.9),\n                           T.RandomContrast(1.4, 1.42),\n                          ])\n    \n    image, transforms = T.apply_transform_gens(transform_list, image)\n    \n    \n    dataset_dict[\"image\"] = torch.as_tensor(\n        image.transpose(2, 0, 1).astype(\"float32\"))\n\n\n    return dataset_dict","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:24.666157Z","iopub.status.idle":"2023-08-06T15:40:24.66715Z","shell.execute_reply.started":"2023-08-06T15:40:24.6669Z","shell.execute_reply":"2023-08-06T15:40:24.666923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Can evaluate trained model on validation dataset to obtain different metric scores. ","metadata":{}},{"cell_type":"code","source":"if is_evaluate:\n    print(\"### EVALUATING ON VALIDATION DATA ####\")\n    # trained model weights\n    cfg.MODEL.WEIGHTS = str(MODEL_PATH)\n    \n    cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128\n    cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.45\n    \n    cfg.MODEL.RPN.POST_NMS_TOPK_TEST = 1000\n    cfg.MODEL.RPN.PRE_NMS_TOPK_TEST = 6000\n    \n    cfg.SOLVER.IMS_PER_BATCH = 2\n    \n    # DETECTION PER IMAGE on default was 100 which is low for our DATASET\n    cfg.TEST.DETECTIONS_PER_IMAGE = 250\n\n    evaluator = COCOEvaluator(\n        DATA_REGISTER_VALID, cfg, False, output_dir=cfg.OUTPUT_DIR, max_dets_per_image=200, use_fast_impl=True\n    )\n\n    val_loader = build_detection_test_loader(cfg, DATA_REGISTER_VALID, mapper=custom_mapper_eval ) \n\n    results = inference_on_dataset(\n        trainer.model, val_loader, evaluator=evaluator\n    )","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:24.6688Z","iopub.status.idle":"2023-08-06T15:40:24.669288Z","shell.execute_reply.started":"2023-08-06T15:40:24.669039Z","shell.execute_reply":"2023-08-06T15:40:24.669062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 9 Inference","metadata":{}},{"cell_type":"markdown","source":"## 9.1 Setting Up Inference Model","metadata":{}},{"cell_type":"code","source":"cfg = get_cfg()\n\nconfig_name = \"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"\n\ncfg.merge_from_file(model_zoo.get_config_file(config_name))\ncfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128\ncfg.TEST.DETECTIONS_PER_IMAGE = 250\n\ncfg.MODEL.ROI_HEADS.NUM_CLASSES = 4\n\ncfg.MODEL.RPN.POST_NMS_TOPK_TEST = 1000\ncfg.MODEL.RPN.PRE_NMS_TOPK_TEST = 6000\n\ncfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.45\n\ncfg.MODEL.ROI_MASK_HEAD.NUM_CONV = 4\ncfg.MODEL.ROI_BOX_HEAD.NUM_CONV = 4\ncfg.MODEL.ROI_BOX_HEAD.POOLER_SAMPLING_RATIO = 2\ncfg.MODEL.ROI_MASK_HEAD.POOLER_SAMPLING_RATIO = 2\n    \ncfg.MODEL.DEVICE = \"cuda\"\n\ncfg.DATALOADER.NUM_WORKERS = 0  # lower this if CUDA overflow occurs\ncfg.MODEL.WEIGHTS = str(MODEL_PATH)","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:24.670994Z","iopub.status.idle":"2023-08-06T15:40:24.671479Z","shell.execute_reply.started":"2023-08-06T15:40:24.671236Z","shell.execute_reply":"2023-08-06T15:40:24.671258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH = 1\ntest_loader = build_detection_test_loader(cfg, DATA_REGISTER_TEST, batch_size=BATCH, mapper = custom_mapper_eval)","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:24.673204Z","iopub.status.idle":"2023-08-06T15:40:24.67407Z","shell.execute_reply.started":"2023-08-06T15:40:24.673877Z","shell.execute_reply":"2023-08-06T15:40:24.673901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Acceptance Thresholds\nDICE Score varies with this, So values were taken based on Sample Inference Visualization, and Trial and Error.","metadata":{}},{"cell_type":"code","source":"ACCEPTANCE_THRESHOLD = [0.63, 0.58, 0.85, 0.86]  # category wise ACCEPTANCE THRESHOLDS","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:24.676493Z","iopub.status.idle":"2023-08-06T15:40:24.676969Z","shell.execute_reply.started":"2023-08-06T15:40:24.676738Z","shell.execute_reply":"2023-08-06T15:40:24.676761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"#### MODEL: {cfg.MODEL.WEIGHTS} FOR INFERENCE ####\")","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:24.678879Z","iopub.status.idle":"2023-08-06T15:40:24.679367Z","shell.execute_reply.started":"2023-08-06T15:40:24.679122Z","shell.execute_reply":"2023-08-06T15:40:24.679145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 9.2 Sample Inference","metadata":{}},{"cell_type":"code","source":"predictor = DefaultPredictor(cfg)","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:24.681351Z","iopub.status.idle":"2023-08-06T15:40:24.681846Z","shell.execute_reply.started":"2023-08-06T15:40:24.681603Z","shell.execute_reply":"2023-08-06T15:40:24.681626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(8, 1, figsize=(100, 100))\nindices = [ax[0], ax[1], ax[2], ax[3], ax[4], ax[5], ax[6], ax[7]]\n\n# Show some qualitative results by predicting on test set images\nNUM_TEST_SAMPLES = 4\n\n#samples = [dataset_dicts_test[3863], dataset_dicts_test[5664], dataset_dicts_test[12525], dataset_dicts_test[244]]\nsamples = np.random.choice(dataset_dicts_test, NUM_TEST_SAMPLES)\n\n\nfor i, sample in enumerate(samples):\n    img = cv2.imread(sample[\"file_name\"])\n    print(sample[\"image_id\"])\n    \n    outputs = predictor(img)\n    prediction = outputs[\"instances\"]\n    pred_masks = prediction.pred_masks  # Predicted binary masks\n    pred_classes = prediction.pred_classes  # Predicted class labels\n    pred_scores = prediction.scores\n    \n    pred_classes = pred_classes.detach().cpu().numpy()\n    pred_scores = pred_scores.detach().cpu().numpy()\n    \n    # Remove predictions with lower scores\n    for j in range(len(pred_classes)):\n        if pred_scores[j] < ACCEPTANCE_THRESHOLD[pred_classes[j]]:\n            pred_classes[j] = -1  #un-detect\n            \n    prediction = prediction[pred_classes != -1]\n    outputs[\"instances\"] = prediction\n    \n    visualizer = Visualizer(img, metadata=metadata_dicts_test, scale=0.5,)\n    visualizer = visualizer.draw_instance_predictions(\n        outputs[\"instances\"].to(\"cpu\")\n    )\n    display_img = visualizer.get_image()[:, :, ::-1]\n    indices[i].grid(False)\n    indices[2*i].imshow(img)\n    indices[2*i+1].imshow(display_img)","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:24.683873Z","iopub.status.idle":"2023-08-06T15:40:24.684358Z","shell.execute_reply.started":"2023-08-06T15:40:24.684116Z","shell.execute_reply":"2023-08-06T15:40:24.684139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 9.3 Test Data Inference and Submission","metadata":{}},{"cell_type":"markdown","source":"### 9.3.1 Building Inference Model","metadata":{}},{"cell_type":"code","source":"def rebuild_model():\n    model = build_model(cfg)\n    _ = DetectionCheckpointer(model).load(cfg.MODEL.WEIGHTS)\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:24.686084Z","iopub.status.idle":"2023-08-06T15:40:24.687073Z","shell.execute_reply.started":"2023-08-06T15:40:24.686881Z","shell.execute_reply":"2023-08-06T15:40:24.686903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = rebuild_model()","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:24.68931Z","iopub.status.idle":"2023-08-06T15:40:24.690152Z","shell.execute_reply.started":"2023-08-06T15:40:24.689871Z","shell.execute_reply":"2023-08-06T15:40:24.689896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 9.3.2 CUDA Problems","metadata":{}},{"cell_type":"code","source":"!export LRU_CACHE_CAPACITY=1\n!export 'PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:512'","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:24.691781Z","iopub.status.idle":"2023-08-06T15:40:24.69264Z","shell.execute_reply.started":"2023-08-06T15:40:24.692378Z","shell.execute_reply":"2023-08-06T15:40:24.692401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Occassionally CUDA memory overflow occurs. Trying to save as much VRAM as we can. ","metadata":{}},{"cell_type":"code","source":"vars_to_del = [\"trainer\", \"predictor\", \"outputs\"]\n\nfor v in vars_to_del:\n    if v in globals():\n        print(f\"Deleting {v}\")\n        del globals()[v]\n    elif v in locals():\n        print(f\"Deleting {v}\")\n        del locals()[v]","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:24.693914Z","iopub.status.idle":"2023-08-06T15:40:24.69588Z","shell.execute_reply.started":"2023-08-06T15:40:24.695636Z","shell.execute_reply":"2023-08-06T15:40:24.695659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 9.3.3 Inference Utils","metadata":{}},{"cell_type":"code","source":"def rle_encode(mask):\n    pixels = mask.T.flatten()\n    use_padding = False\n    if pixels[0] or pixels[-1]:\n        use_padding = True\n        pixel_padded = np.zeros([len(pixels) + 2], dtype=pixels.dtype)\n        pixel_padded[1:-1] = pixels\n        pixels = pixel_padded\n    rle = np.where(pixels[1:] != pixels[:-1])[0] + 2\n    if use_padding:\n        rle = rle - 1\n    rle[1::2] = rle[1::2] - rle[:-1:2]\n    return ' '.join(str(x) for x in rle)","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:24.698344Z","iopub.status.idle":"2023-08-06T15:40:24.699115Z","shell.execute_reply.started":"2023-08-06T15:40:24.698863Z","shell.execute_reply":"2023-08-06T15:40:24.698886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@retry_if_cuda_oom\ndef get_masks(prediction):\n    \n    \n    pred_masks = prediction.pred_masks  # Predicted binary masks\n    pred_classes = prediction.pred_classes  # Predicted class labels\n    pred_scores = prediction.scores\n    \n    pred_classes = pred_classes.detach().cpu().numpy()\n    pred_scores = pred_scores.detach().cpu().numpy()\n    \n    # REMOVE PREDICTIONS WHICH HAS lower score (Category wise)\n    for i in range(len(pred_classes)):\n        if pred_scores[i] < ACCEPTANCE_THRESHOLD[pred_classes[i]]:\n            pred_classes[i] = -1  #un-detect\n    \n    rles = []\n    for cat in range(len(thing_classes)):\n        # get masks for each category\n        pred_mask = pred_masks[pred_classes == cat]\n        pred_mask = torch.any(pred_mask, dim=0)\n        rles.append(rle_encode(pred_mask.short().to(\"cpu\").numpy()))\n    return rles","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:24.700507Z","iopub.status.idle":"2023-08-06T15:40:24.701331Z","shell.execute_reply.started":"2023-08-06T15:40:24.701062Z","shell.execute_reply":"2023-08-06T15:40:24.701086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_inference(data):\n    results = []\n    with torch.no_grad():\n        outputs = model(data)\n        if torch.cuda.is_available():\n            torch.cuda.synchronize()\n\n        for idx, output in enumerate(outputs):\n            output = output[\"instances\"]\n\n            rles = get_masks(output)\n\n            result = [\n                f\"{data[idx]['image_id']}_{cat},{rles[cat]}\\n\"\n                for cat in range(len(thing_classes))\n            ]\n\n            results.extend(result)\n\n        del outputs, output\n\n    return results","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:24.702799Z","iopub.status.idle":"2023-08-06T15:40:24.703625Z","shell.execute_reply.started":"2023-08-06T15:40:24.703341Z","shell.execute_reply":"2023-08-06T15:40:24.703365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 9.3.4 Running Inference on Test Data and Creating Submission File","metadata":{}},{"cell_type":"code","source":"print(\"#### RUNNING INFERENCE ON TEST DATA ####\")","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:24.705239Z","iopub.status.idle":"2023-08-06T15:40:24.705986Z","shell.execute_reply.started":"2023-08-06T15:40:24.705747Z","shell.execute_reply":"2023-08-06T15:40:24.70577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:24.707376Z","iopub.status.idle":"2023-08-06T15:40:24.708157Z","shell.execute_reply.started":"2023-08-06T15:40:24.707904Z","shell.execute_reply":"2023-08-06T15:40:24.707928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_inference:\n    model.eval()\n    submission_file = open(\"submission.csv\", \"w\")\n    submission_file.write(\"Id,Predicted\\n\")\n\n    results: list[str] = []\n    \n    for i, data in enumerate(tqdm(test_loader)):\n        res = run_inference(data)\n        results.extend(res)\n        \n        if i % (500 // BATCH) == 0:\n            print(f\"Inference on batch {i}/{len(test_loader)} done\")\n            submission_file.writelines(results)\n            results = []\n\n    submission_file.writelines(results)\n    submission_file.close()","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:24.709556Z","iopub.status.idle":"2023-08-06T15:40:24.710314Z","shell.execute_reply.started":"2023-08-06T15:40:24.710059Z","shell.execute_reply":"2023-08-06T15:40:24.710082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if Path(\"submission.csv\").exists:\n    display(FileLink(\"submission.csv\"))","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:24.711743Z","iopub.status.idle":"2023-08-06T15:40:24.712486Z","shell.execute_reply.started":"2023-08-06T15:40:24.712246Z","shell.execute_reply":"2023-08-06T15:40:24.712269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r detectron2/","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:40:24.713887Z","iopub.status.idle":"2023-08-06T15:40:24.714674Z","shell.execute_reply.started":"2023-08-06T15:40:24.714414Z","shell.execute_reply":"2023-08-06T15:40:24.714436Z"},"trusted":true},"execution_count":null,"outputs":[]}]}