{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is combination of [Starter Notebook](https://www.kaggle.com/code/salmankhondker/starter-notebook-dl-sprint-2-0) and [EDA](https://www.kaggle.com/code/umongsain/eda-dl-sprint-2-0) to have everthing is one place and a complete guide to start.\n\nHowever, more tuning will be done in this notebook to have an efficient model.","metadata":{}},{"cell_type":"markdown","source":"## Detectron2\n\n*[Detectron2](https://detectron2.readthedocs.io/en/latest/index.html) is a popular open-source software library developed by Facebook AI Research (FAIR) for building computer vision models. It serves as a powerful framework for object detection, instance segmentation, and keypoint detection tasks. Detectron2 is built on top of PyTorch, geared towards a more convenient way to build modular, flexible pipelines for specific Computer Vision Tasks such as object detection, instance segmentation.*\n\n*Detectron2 has a collection of trained models for these tasks in their [model zoo](https://github.com/facebookresearch/detectron2/blob/main/MODEL_ZOO.md). We can also use detectron2 to train pre-implemented state-of-the-art models from scratch for new datasets, as we have done in this notebook.*\n\n*Read the [Detectron2-Documentation](https://detectron2.readthedocs.io/en/latest/index.html) !!!*","metadata":{}},{"cell_type":"markdown","source":"# 1. Import","metadata":{}},{"cell_type":"markdown","source":"### 1.1 Import Detectron2","metadata":{}},{"cell_type":"code","source":"# !git clone https://github.com/facebookresearch/detectron2.git\n# !python -m pip install -e detectron2","metadata":{"execution":{"iopub.status.busy":"2023-07-01T17:56:17.328482Z","iopub.execute_input":"2023-07-01T17:56:17.328835Z","iopub.status.idle":"2023-07-01T17:56:17.334456Z","shell.execute_reply.started":"2023-07-01T17:56:17.328808Z","shell.execute_reply":"2023-07-01T17:56:17.332441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!pip install 'git+https://github.com/facebookresearch/detectron2.git@5aeb252b194b93dc2879b4ac34bc51a31b5aee13'","metadata":{"execution":{"iopub.status.busy":"2023-07-01T17:56:18.241520Z","iopub.execute_input":"2023-07-01T17:56:18.241878Z","iopub.status.idle":"2023-07-01T18:00:16.328806Z","shell.execute_reply.started":"2023-07-01T17:56:18.241850Z","shell.execute_reply":"2023-07-01T18:00:16.327406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.2 Import Libraries","metadata":{}},{"cell_type":"code","source":"%%capture\n# detectron2\nfrom detectron2.utils.memory import retry_if_cuda_oom\nfrom detectron2.utils.logger import setup_logger\nfrom detectron2.checkpoint import DetectionCheckpointer\nfrom detectron2.modeling import build_model\nfrom detectron2.evaluation import COCOEvaluator, inference_on_dataset\nimport detectron2.data.transforms as T\nfrom detectron2.data import detection_utils as utils\nfrom detectron2.data import DatasetCatalog, MetadataCatalog, build_detection_test_loader, build_detection_train_loader, DatasetMapper\nfrom detectron2.utils.visualizer import Visualizer\nfrom detectron2.structures import BoxMode\nfrom detectron2.engine import DefaultPredictor, DefaultTrainer\nfrom detectron2.config import get_cfg\nfrom detectron2 import model_zoo\n\nimport pandas as pd\nimport numpy as np\nfrom tqdm.notebook import tqdm  # progress bar\nimport matplotlib.pyplot as plt\nimport json\nimport cv2\nimport copy\nfrom typing import Optional\nimport seaborn as sns\n\n\n!pip install -q pycocotools\nfrom pycocotools.coco import COCO\nfrom PIL import Image\nimport random\nfrom pathlib import Path\n%matplotlib inline\nsns.set_theme(style='darkgrid', palette='deep', font='sans-serif', font_scale=1)\n\nfrom IPython.display import FileLink\n# torch\nimport torch\nimport gc\nimport warnings\n# Ignore \"future\" warnings and Data-Frame-Slicing warnings.\nwarnings.filterwarnings('ignore')\n\nsetup_logger()","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:00:16.334913Z","iopub.execute_input":"2023-07-01T18:00:16.341416Z","iopub.status.idle":"2023-07-01T18:00:32.070546Z","shell.execute_reply.started":"2023-07-01T18:00:16.341372Z","shell.execute_reply":"2023-07-01T18:00:32.069329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.3 Data Load","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\n\nTRAIN_IMG_DIR = Path(\"/kaggle/input/dlsprint2/badlad/images/train\")\n\nTRAIN_COCO_PATH = Path(\"/kaggle/input/dlsprint2/badlad/labels/coco_format/train/badlad-train-coco.json\")\n\nTEST_IMG_DIR = Path(\"/kaggle/input/dlsprint2/badlad/images/test\")\n\nTEST_METADATA_PATH = Path(\"/kaggle/input/dlsprint2/badlad/badlad-test-metadata.json\")\n\n# Training output directory\nOUTPUT_DIR = Path(\"./output\")\nOUTPUT_MODEL = OUTPUT_DIR/\"model_final.pth\"\n\n# Path to your pretrained model weights\nPRETRAINED_PATH = Path(\"\")","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:00:32.072325Z","iopub.execute_input":"2023-07-01T18:00:32.072899Z","iopub.status.idle":"2023-07-01T18:00:32.079461Z","shell.execute_reply.started":"2023-07-01T18:00:32.072860Z","shell.execute_reply":"2023-07-01T18:00:32.078543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.4 COCO Annotations Data","metadata":{}},{"cell_type":"code","source":"with TRAIN_COCO_PATH.open() as f:\n    train_dict = json.load(f)\n\nwith TEST_METADATA_PATH.open() as f:\n    test_dict = json.load(f)\n    \ntrain_coco_labels=COCO(annotation_file=TRAIN_COCO_PATH)\n\nprint(\"#### LABELS AND METADATA LOADED ####\")","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:00:32.082450Z","iopub.execute_input":"2023-07-01T18:00:32.083053Z","iopub.status.idle":"2023-07-01T18:00:43.753640Z","shell.execute_reply.started":"2023-07-01T18:00:32.083022Z","shell.execute_reply":"2023-07-01T18:00:43.752620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.5 Create Decisions","metadata":{}},{"cell_type":"code","source":"from datetime import datetime\n\n# if False, model is set to `PRETRAINED_PATH` model\nis_train = True\n\n# if True, evaluate on validation dataset\nis_evaluate = False\n\n# if True, run inference on test dataset\nis_inference = True\n\n# if True and `is_train` == True, `PRETRAINED_PATH` model is trained further\nis_resume_training = False\n\n# Perform augmentation\nis_augment = False\n\nSEED = int(datetime.now().timestamp())\n\n# Model path based on Decisions\nMODEL_PATH = OUTPUT_MODEL if is_train else PRETRAINED_PATH","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:00:43.755360Z","iopub.execute_input":"2023-07-01T18:00:43.755713Z","iopub.status.idle":"2023-07-01T18:00:43.761971Z","shell.execute_reply.started":"2023-07-01T18:00:43.755681Z","shell.execute_reply":"2023-07-01T18:00:43.760835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Dataset","metadata":{}},{"cell_type":"markdown","source":"### 2.1 Dataset Analysis","metadata":{}},{"cell_type":"code","source":"print(\"There are \" + str(len(train_dict['categories'])) + \" categories.\\n\")\nprint(\"There are \" + str(len(test_dict['images']) + len(train_dict['images'])) + \" images in the dataset.\")\nprint(\"There are \" + str(len(train_dict['images'])) + \" images in the train set.\")\nprint(\"There are \" + str(len(test_dict['images'])) + \" images in the test set.\\n\")\nprint(\"There are \" + str(len(train_dict['annotations'])) + \" annotations in the train set.\\n\")\n\nprint(\"We will focus on mainly categories, images and annotations.\")","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:00:43.764059Z","iopub.execute_input":"2023-07-01T18:00:43.764676Z","iopub.status.idle":"2023-07-01T18:00:43.788576Z","shell.execute_reply.started":"2023-07-01T18:00:43.764613Z","shell.execute_reply":"2023-07-01T18:00:43.787635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def organize_coco_data(data_dict: dict) -> tuple[list[str], list[dict], list[dict]]:\n    thing_classes: list[str] = []\n\n    # Map Category Names to IDs\n    for cat in data_dict['categories']:\n        thing_classes.append(cat['name'])\n\n    # Images\n    images_metadata: list[dict] = data_dict['images']\n\n    # Convert COCO annotations to detectron2 annotations format\n    data_annotations = []\n    for ann in data_dict['annotations']:\n        # coco format -> detectron2 format\n        annot_obj = {\n            # Annotation ID\n            \"id\": ann['id'],\n\n            # Segmentation Polygon (x, y) coordinnates\n            \"gt_masks\": ann['segmentation'],\n\n            # Image ID for this annotation (Which image does this annotation belong to?)\n            \"image_id\": ann['image_id'],\n\n            # Category Label (0: paragraph, 1: text box, 2: image, 3: table)\n            \"category_id\": ann['category_id'],\n\n            \"x_min\": ann['bbox'][0],  # left\n            \"y_min\": ann['bbox'][1],  # top\n            \"x_max\": ann['bbox'][0] + ann['bbox'][2],  # left+width\n            \"y_max\": ann['bbox'][1] + ann['bbox'][3]  # top+height\n        }\n        data_annotations.append(annot_obj)\n\n    return thing_classes, images_metadata, data_annotations","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:00:43.790353Z","iopub.execute_input":"2023-07-01T18:00:43.790999Z","iopub.status.idle":"2023-07-01T18:00:43.800872Z","shell.execute_reply.started":"2023-07-01T18:00:43.790968Z","shell.execute_reply":"2023-07-01T18:00:43.799974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"thing_classes, images_metadata, data_annotations = organize_coco_data(train_dict)\n\nthing_classes_test, images_metadata_test, _ = organize_coco_data(test_dict)\n\nprint(thing_classes)","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:00:43.802509Z","iopub.execute_input":"2023-07-01T18:00:43.803071Z","iopub.status.idle":"2023-07-01T18:00:45.530915Z","shell.execute_reply.started":"2023-07-01T18:00:43.803037Z","shell.execute_reply":"2023-07-01T18:00:45.529830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata = pd.DataFrame(images_metadata)\ntrain_metadata = train_metadata[['id', 'file_name', 'width', 'height']]\ntrain_metadata = train_metadata.rename(columns={\"id\": \"image_id\"})\nprint(\"train_metadata size=\", len(train_metadata))\ntrain_metadata.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:00:45.532575Z","iopub.execute_input":"2023-07-01T18:00:45.532947Z","iopub.status.idle":"2023-07-01T18:00:45.630304Z","shell.execute_reply.started":"2023-07-01T18:00:45.532914Z","shell.execute_reply":"2023-07-01T18:00:45.629135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_annot_df = pd.DataFrame(data_annotations)\nprint(\"train_annot_df size=\", len(train_annot_df))\ntrain_annot_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:00:45.635698Z","iopub.execute_input":"2023-07-01T18:00:45.636535Z","iopub.status.idle":"2023-07-01T18:00:47.100598Z","shell.execute_reply.started":"2023-07-01T18:00:45.636506Z","shell.execute_reply":"2023-07-01T18:00:47.099585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Here `gt_masks` are the sequence of `(x, y)` coordinates of vertices of the polygon surrounding the target object. ","metadata":{}},{"cell_type":"code","source":"test_metadata = pd.DataFrame(images_metadata_test)\ntest_metadata = test_metadata[['id', 'file_name', 'width', 'height']]\ntest_metadata = test_metadata.rename(columns={\"id\": \"image_id\"})\nprint(\"test_metadata size=\", len(test_metadata))\ntest_metadata.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:00:47.102208Z","iopub.execute_input":"2023-07-01T18:00:47.102925Z","iopub.status.idle":"2023-07-01T18:00:47.156466Z","shell.execute_reply.started":"2023-07-01T18:00:47.102889Z","shell.execute_reply":"2023-07-01T18:00:47.155483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15, 15))\nfor i in range(6):\n    image_id = random.randint(2, 20364)\n    image_file = train_coco_labels.loadImgs([image_id])[0]['file_name']\n    image = Image.open(TRAIN_IMG_DIR/image_file)\n    code = 231 + i;\n    plt.subplot(code)\n    plt.axis('off')\n    plt.imshow(np.asarray(image))","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:00:47.158084Z","iopub.execute_input":"2023-07-01T18:00:47.158795Z","iopub.status.idle":"2023-07-01T18:00:50.571172Z","shell.execute_reply.started":"2023-07-01T18:00:47.158759Z","shell.execute_reply":"2023-07-01T18:00:50.569269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_image(_image_id):\n    image_file = train_coco_labels.loadImgs([_image_id])[0]['file_name']\n    image = Image.open(TRAIN_IMG_DIR/image_file)\n    plt.figure(figsize=(15, 15))\n    plt.axis('off')\n    plt.imshow(np.asarray(image))","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:00:50.572145Z","iopub.execute_input":"2023-07-01T18:00:50.572467Z","iopub.status.idle":"2023-07-01T18:00:50.579237Z","shell.execute_reply.started":"2023-07-01T18:00:50.572439Z","shell.execute_reply":"2023-07-01T18:00:50.577804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_segmentations(_image_id):\n    show_image(_image_id)\n    annotation_ids = train_coco_labels.getAnnIds(imgIds=[_image_id])\n    annotations = train_coco_labels.loadAnns(annotation_ids)\n    train_coco_labels.showAnns(annotations)","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:00:50.580953Z","iopub.execute_input":"2023-07-01T18:00:50.581853Z","iopub.status.idle":"2023-07-01T18:00:50.606417Z","shell.execute_reply.started":"2023-07-01T18:00:50.581821Z","shell.execute_reply":"2023-07-01T18:00:50.605561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# you can change the image_id\nshow_image(50)","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:00:50.608113Z","iopub.execute_input":"2023-07-01T18:00:50.608973Z","iopub.status.idle":"2023-07-01T18:00:51.310602Z","shell.execute_reply.started":"2023-07-01T18:00:50.608940Z","shell.execute_reply":"2023-07-01T18:00:51.308738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# you can change the image_id\nshow_segmentations(50)","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:00:51.312176Z","iopub.execute_input":"2023-07-01T18:00:51.312929Z","iopub.status.idle":"2023-07-01T18:00:52.067460Z","shell.execute_reply.started":"2023-07-01T18:00:51.312893Z","shell.execute_reply":"2023-07-01T18:00:52.066354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.2 Exploring Annonations","metadata":{}},{"cell_type":"markdown","source":"> *`annotations:` This is a list of the polygon bounding segments for each instance of the categories occurring in each image.*\n\n<br>\nThe contents of this category are the crux of the problem:\n\n* `segmentation:` Polygon (x, y) coordinate pairs of it's vertices.For example, the first annotation has its first vertex at (86.328, 179.007). The following pairs complete the polygon region of the annotation.\n* `image_id:` The image this annotation is for.\n* `category_id:` The category this annotation is bounding. It may be a paragraph, text box, image, or table.\n* `id:` Annotation ID, identifies each annotation.\n* `bbox:` The rectangular bounding box that best estimates the annotated region.\nThis list holds all the annotations for each instance of the categories for all images.","metadata":{}},{"cell_type":"code","source":"train_annotations = pd.DataFrame(train_dict['annotations'])\ntrain_annotations.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:00:52.068961Z","iopub.execute_input":"2023-07-01T18:00:52.069847Z","iopub.status.idle":"2023-07-01T18:00:53.532302Z","shell.execute_reply.started":"2023-07-01T18:00:52.069813Z","shell.execute_reply":"2023-07-01T18:00:53.531201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# adding bbox_area and bbox_aspect_ratio\nprint(\"\\nRenaming id to annotation_id.\\nAdding bbox_area, bbox_aspect_ratio.\\n\")\ntrain_annotations.rename(columns={\"id\":\"annotation_id\"}, inplace=True)\nbbox_area=[]\nbbox_aspect_ratio=[]\nfor idx in train_annotations.index:\n    bbox_area.append(train_annotations['bbox'][idx][3]*train_annotations['bbox'][idx][2])\n    bbox_aspect_ratio.append(train_annotations['bbox'][idx][3]/train_annotations['bbox'][idx][2])\ntrain_annotations['bbox_area']=bbox_area\ntrain_annotations['bbox_aspect_ratio']=bbox_aspect_ratio\nprint(\"train_annotations shape: \" + str(train_annotations.shape) + \"\\n\")\ntrain_annotations.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:00:53.534303Z","iopub.execute_input":"2023-07-01T18:00:53.534719Z","iopub.status.idle":"2023-07-01T18:01:09.878651Z","shell.execute_reply.started":"2023-07-01T18:00:53.534684Z","shell.execute_reply":"2023-07-01T18:01:09.877621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_annotations.describe()","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:01:09.880074Z","iopub.execute_input":"2023-07-01T18:01:09.880935Z","iopub.status.idle":"2023-07-01T18:01:09.992402Z","shell.execute_reply.started":"2023-07-01T18:01:09.880903Z","shell.execute_reply":"2023-07-01T18:01:09.991335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.3 Categorical Calssification","metadata":{}},{"cell_type":"code","source":"train_categories = pd.DataFrame(train_dict['categories'])\n\nprint(\"Dropping supercategory.\\nRenaming id to category_id.\\n\")\ntrain_categories.drop('supercategory', axis=1, inplace=True)\ntrain_categories.rename(columns={\"id\":\"category_id\"}, inplace=True)\ntrain_categories","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:01:09.993777Z","iopub.execute_input":"2023-07-01T18:01:09.994452Z","iopub.status.idle":"2023-07-01T18:01:10.008256Z","shell.execute_reply.started":"2023-07-01T18:01:09.994413Z","shell.execute_reply":"2023-07-01T18:01:10.007259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_cat_count = train_annotations[['image_id', 'category_id']].copy()\ncategory_names=[]\nfor idx in img_cat_count.index:\n    category_names.append(train_categories['name'][img_cat_count['category_id'][idx]])\nimg_cat_count['category_name']=category_names\nimg_cat_count[train_categories['name'][0]] = img_cat_count['category_id'] == train_categories['category_id'][0]\nimg_cat_count[train_categories['name'][1]] = img_cat_count['category_id'] == train_categories['category_id'][1]\nimg_cat_count[train_categories['name'][2]] = img_cat_count['category_id'] == train_categories['category_id'][2]\nimg_cat_count[train_categories['name'][3]] = img_cat_count['category_id'] == train_categories['category_id'][3]\nimg_cat_count.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:01:10.009755Z","iopub.execute_input":"2023-07-01T18:01:10.010397Z","iopub.status.idle":"2023-07-01T18:01:17.792546Z","shell.execute_reply.started":"2023-07-01T18:01:10.010361Z","shell.execute_reply":"2023-07-01T18:01:17.791560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"\\nCategorywise Document Image Count.\\n\")\ncategorywise_image_count=img_cat_count.groupby('category_id', as_index=False)['image_id'].nunique()\ncategorywise_image_count['category_name']=train_categories['name']\ncategorywise_image_count.rename(columns={'image_id':'image_count'}, inplace=True)\ncategorywise_image_count = categorywise_image_count[['category_id', 'category_name', 'image_count']]\ncategorywise_image_count","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:01:17.794028Z","iopub.execute_input":"2023-07-01T18:01:17.794519Z","iopub.status.idle":"2023-07-01T18:01:17.839980Z","shell.execute_reply.started":"2023-07-01T18:01:17.794482Z","shell.execute_reply":"2023-07-01T18:01:17.838767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Insights**\n> * Text box and paragraph appears almost in all the documents.\n> * Image appears almost in one-fourth of the documents of trainset.\n> * Tables are not that much frequent like other 3 classes. 1 in 20 documents contains table.","metadata":{}},{"cell_type":"code","source":"plt.title(\"Categories vs Number of document images they appear\")\nsns.barplot(x=categorywise_image_count['category_name'], y=categorywise_image_count['image_count'])","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:01:17.841670Z","iopub.execute_input":"2023-07-01T18:01:17.842311Z","iopub.status.idle":"2023-07-01T18:01:18.215959Z","shell.execute_reply.started":"2023-07-01T18:01:17.842275Z","shell.execute_reply":"2023-07-01T18:01:18.215019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"\\nImagewise Category Count.\\n\")\nimagewise_category_count=img_cat_count.groupby('image_id', as_index=False)[['paragraph', 'text_box', 'image', 'table']].sum()\nimagewise_category_count.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:01:18.217671Z","iopub.execute_input":"2023-07-01T18:01:18.218021Z","iopub.status.idle":"2023-07-01T18:01:18.258974Z","shell.execute_reply.started":"2023-07-01T18:01:18.217990Z","shell.execute_reply":"2023-07-01T18:01:18.257860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15, 30))\n\nplt.subplot(411)\nplt.title(\"Number of paragraphs per document image\")\nsns.lineplot(x=imagewise_category_count['image_id'], y=imagewise_category_count['paragraph'])\n\nplt.subplot(412)\nplt.title(\"Number of text boxes per document image\")\nsns.lineplot(x=imagewise_category_count['image_id'], y=imagewise_category_count['text_box'])\n\nplt.subplot(413)\nplt.title(\"Number of images per document image\")\nsns.lineplot(x=imagewise_category_count['image_id'], y=imagewise_category_count['image'])\n\nplt.subplot(414)\nplt.title(\"Number of tables per document image\")\nsns.lineplot(x=imagewise_category_count['image_id'], y=imagewise_category_count['table'])","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:01:18.260749Z","iopub.execute_input":"2023-07-01T18:01:18.261100Z","iopub.status.idle":"2023-07-01T18:01:21.269687Z","shell.execute_reply.started":"2023-07-01T18:01:18.261069Z","shell.execute_reply":"2023-07-01T18:01:21.268804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Preparing Data for Training","metadata":{}},{"cell_type":"markdown","source":"### 3.1 Train-Validation Split","metadata":{}},{"cell_type":"code","source":"TRAIN_SPLIT = 0.95","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:01:21.270961Z","iopub.execute_input":"2023-07-01T18:01:21.271840Z","iopub.status.idle":"2023-07-01T18:01:21.281569Z","shell.execute_reply.started":"2023-07-01T18:01:21.271806Z","shell.execute_reply":"2023-07-01T18:01:21.280292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_dataset = len(train_metadata)\nn_train = int(n_dataset * TRAIN_SPLIT)\nprint(\"n_dataset\", n_dataset, \"n_train\", n_train, \"n_val\", n_dataset-n_train)\n\nnp.random.seed(SEED)\n\ninds = np.random.permutation(n_dataset)\ntrain_inds, valid_inds = inds[:n_train], inds[n_train:]","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:01:21.283445Z","iopub.execute_input":"2023-07-01T18:01:21.284138Z","iopub.status.idle":"2023-07-01T18:01:21.295363Z","shell.execute_reply.started":"2023-07-01T18:01:21.284092Z","shell.execute_reply":"2023-07-01T18:01:21.294420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.2 Formatting Data for `detectron2`","metadata":{}},{"cell_type":"code","source":"def convert_coco_to_detectron2_format(\n    imgdir: Path,\n    metadata_df: pd.DataFrame,\n    annot_df: Optional[pd.DataFrame] = None,\n    target_indices: Optional[np.ndarray] = None,\n):\n\n    dataset_dicts = []\n    for _, train_meta_row in tqdm(metadata_df.iterrows(), total=len(metadata_df)):\n        # Iterate over each image\n        image_id, filename, width, height = train_meta_row.values\n\n        annotations = []\n\n        # If train/validation data, then there will be annotations\n        if annot_df is not None:\n            for _, ann in annot_df.query(\"image_id == @image_id\").iterrows():\n                # Get annotations of current iteration's image\n                class_id = ann[\"category_id\"]\n                gt_masks = ann[\"gt_masks\"]\n                bbox_resized = [\n                    float(ann[\"x_min\"]),\n                    float(ann[\"y_min\"]),\n                    float(ann[\"x_max\"]),\n                    float(ann[\"y_max\"]),\n                ]\n\n                annotation = {\n                    \"bbox\": bbox_resized,\n                    \"bbox_mode\": BoxMode.XYXY_ABS,\n                    \"segmentation\": gt_masks,\n                    \"category_id\": class_id,\n                }\n\n                annotations.append(annotation)\n\n        # coco format -> detectron2 format dict\n        record = {\n            \"file_name\": str(imgdir/filename),\n            \"image_id\": image_id,\n            \"width\": width,\n            \"height\": height,\n            \"annotations\": annotations\n        }\n\n        dataset_dicts.append(record)\n\n    if target_indices is not None:\n        dataset_dicts = [dataset_dicts[i] for i in target_indices]\n\n    return dataset_dicts","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:01:21.309565Z","iopub.execute_input":"2023-07-01T18:01:21.310219Z","iopub.status.idle":"2023-07-01T18:01:21.325073Z","shell.execute_reply.started":"2023-07-01T18:01:21.310187Z","shell.execute_reply":"2023-07-01T18:01:21.324046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.3 Registering and Loading Data for `detectron2`","metadata":{}},{"cell_type":"code","source":"DATA_REGISTER_TRAINING = \"badlad_train\"\nDATA_REGISTER_VALID    = \"badlad_valid\"\nDATA_REGISTER_TEST     = \"badlad_test\"","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:01:21.329692Z","iopub.execute_input":"2023-07-01T18:01:21.332565Z","iopub.status.idle":"2023-07-01T18:01:21.338982Z","shell.execute_reply.started":"2023-07-01T18:01:21.332532Z","shell.execute_reply":"2023-07-01T18:01:21.338070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Register Training data\nif is_train:\n    DatasetCatalog.register(\n        DATA_REGISTER_TRAINING,\n        lambda: convert_coco_to_detectron2_format(\n            TRAIN_IMG_DIR,\n            train_metadata,\n            train_annot_df,\n            target_indices=train_inds,\n        ),\n    )\n\n    # Set Training data categories\n    MetadataCatalog.get(DATA_REGISTER_TRAINING).set(thing_classes=thing_classes)\n\n    dataset_dicts_train = DatasetCatalog.get(DATA_REGISTER_TRAINING)\n    metadata_dicts_train = MetadataCatalog.get(DATA_REGISTER_TRAINING)\n\n    print(\"dicts training size=\", len(dataset_dicts_train))\n    print(\"################\")","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:01:21.343571Z","iopub.execute_input":"2023-07-01T18:01:21.346361Z","iopub.status.idle":"2023-07-01T18:03:11.511573Z","shell.execute_reply.started":"2023-07-01T18:01:21.346328Z","shell.execute_reply":"2023-07-01T18:03:11.510584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Register Validation data\nif is_train or is_evaluate:\n    DatasetCatalog.register(\n        DATA_REGISTER_VALID,\n        lambda: convert_coco_to_detectron2_format(\n            TRAIN_IMG_DIR,\n            train_metadata,\n            train_annot_df,\n            target_indices=valid_inds,\n        ),\n    )\n\n    # Set Validation data categories\n    MetadataCatalog.get(DATA_REGISTER_VALID).set(thing_classes=thing_classes)\n\n    dataset_dicts_valid = DatasetCatalog.get(DATA_REGISTER_VALID)\n    metadata_dicts_valid = MetadataCatalog.get(DATA_REGISTER_VALID)\n\n    print(\"dicts valid size=\", len(dataset_dicts_valid))\n    print(\"################\")","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:03:11.513323Z","iopub.execute_input":"2023-07-01T18:03:11.514018Z","iopub.status.idle":"2023-07-01T18:05:00.827505Z","shell.execute_reply.started":"2023-07-01T18:03:11.513967Z","shell.execute_reply":"2023-07-01T18:05:00.826344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Register Test Inference data\nDatasetCatalog.register(\n    DATA_REGISTER_TEST,\n    lambda: convert_coco_to_detectron2_format(\n        TEST_IMG_DIR,\n        test_metadata,\n    )\n)\n\n# Set Test data categories\nMetadataCatalog.get(DATA_REGISTER_TEST).set(\n    thing_classes=thing_classes_test\n)\n\ndataset_dicts_test = DatasetCatalog.get(DATA_REGISTER_TEST)\nmetadata_dicts_test = MetadataCatalog.get(DATA_REGISTER_TEST)","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:05:00.829291Z","iopub.execute_input":"2023-07-01T18:05:00.829944Z","iopub.status.idle":"2023-07-01T18:05:01.634110Z","shell.execute_reply.started":"2023-07-01T18:05:00.829909Z","shell.execute_reply":"2023-07-01T18:05:01.633052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"#### DATA REGISTERED ####\")","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:05:01.635662Z","iopub.execute_input":"2023-07-01T18:05:01.636221Z","iopub.status.idle":"2023-07-01T18:05:01.643886Z","shell.execute_reply.started":"2023-07-01T18:05:01.636187Z","shell.execute_reply":"2023-07-01T18:05:01.642800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Augmentation","metadata":{}},{"cell_type":"code","source":"def custom_mapper(dataset_dict):\n    dataset_dict = copy.deepcopy(dataset_dict)\n    image = utils.read_image(dataset_dict[\"file_name\"], format=\"BGR\")\n\n    transform_list = [T.RandomBrightness(0.8, 1.2),\n                      T.RandomFlip(prob=0.5, horizontal=False, vertical=True),\n                      T.RandomFlip(prob=0.5, horizontal=True, vertical=False)\n                      ]\n    image, transforms = T.apply_transform_gens(transform_list, image)\n\n    dataset_dict[\"image\"] = torch.as_tensor(\n        image.transpose(2, 0, 1).astype(\"float32\"))\n\n    annos = [\n        utils.transform_instance_annotations(obj, transforms, image.shape[:2])\n        for obj in dataset_dict.pop(\"annotations\")\n        if obj.get(\"iscrowd\", 0) == 0\n    ]\n    instances = utils.annotations_to_instances(annos, image.shape[:2])\n\n    dataset_dict[\"instances\"] = utils.filter_empty_instances(instances)\n\n    return dataset_dict","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:05:01.645647Z","iopub.execute_input":"2023-07-01T18:05:01.646285Z","iopub.status.idle":"2023-07-01T18:05:01.656592Z","shell.execute_reply.started":"2023-07-01T18:05:01.646249Z","shell.execute_reply":"2023-07-01T18:05:01.655399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AugTrainer(DefaultTrainer):\n    @classmethod\n    def build_train_loader(cls, cfg):\n        return build_detection_train_loader(cfg, mapper=custom_mapper)","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:05:01.658018Z","iopub.execute_input":"2023-07-01T18:05:01.658528Z","iopub.status.idle":"2023-07-01T18:05:01.671152Z","shell.execute_reply.started":"2023-07-01T18:05:01.658496Z","shell.execute_reply":"2023-07-01T18:05:01.670000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Hyperparameters","metadata":{}},{"cell_type":"markdown","source":"Detectron2 models need a config file to build the model. This config file has the associated hyperparameters.\n\nYou can play with this: https://detectron2.readthedocs.io/en/latest/modules/config.html#yaml-config-references","metadata":{}},{"cell_type":"code","source":"if is_train:\n    cfg = get_cfg()\n\n    # config_name = \"COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml\"\n    config_name = \"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"\n\n    cfg.merge_from_file(model_zoo.get_config_file(config_name))\n\n    cfg.DATASETS.TRAIN = (DATA_REGISTER_TRAINING,)\n    cfg.DATASETS.TEST = (DATA_REGISTER_VALID,)\n\n    # to evaluate during training, you have to implement `build_evaluator()` method of the trainer.\n    # https://github.com/facebookresearch/detectron2/blob/94113be6e12db36b8c7601e13747587f19ec92fe/detectron2/engine/defaults.py#L561\n    # cfg.TEST.EVAL_PERIOD = 500\n\n    cfg.DATALOADER.NUM_WORKERS = 2\n\n    # cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(config_name)\n    if (is_resume_training):\n        print(\"#### SETTING PRETRAINED WEIGHTS TO RESUME TRAINING ####\")\n        cfg.MODEL.WEIGHTS = str(PRETRAINED_PATH)\n    else:\n        print(\"#### TRAINING MODEL FROM SCRATCH ####\")\n\n    cfg.SOLVER.AMP.ENABLED = True\n    cfg.SOLVER.IMS_PER_BATCH = 8\n    cfg.SOLVER.BASE_LR = 0.001\n\n    cfg.SOLVER.WARMUP_ITERS = 5\n\n    # Maximum number of iterations\n    cfg.SOLVER.MAX_ITER = 500\n\n    # cfg.SOLVER.STEPS = (500, 1000) # must be less than MAX_ITER\n\n    cfg.SOLVER.GAMMA = 0.05\n\n    # Small value == Frequent save need a lot of storage.\n    cfg.SOLVER.CHECKPOINT_PERIOD = 500\n    cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128\n    cfg.MODEL.ROI_HEADS.NUM_CLASSES = 4\n\n    # Create Output Directory\n    cfg.OUTPUT_DIR = str(OUTPUT_DIR)\n    print(\"creating cfg.OUTPUT_DIR -> \", cfg.OUTPUT_DIR)\n    OUTPUT_DIR.mkdir(exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:09:51.910428Z","iopub.execute_input":"2023-07-01T18:09:51.910849Z","iopub.status.idle":"2023-07-01T18:09:51.946733Z","shell.execute_reply.started":"2023-07-01T18:09:51.910810Z","shell.execute_reply":"2023-07-01T18:09:51.945799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6. Training","metadata":{}},{"cell_type":"markdown","source":"## 6.1 Training the model","metadata":{}},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:09:54.281987Z","iopub.execute_input":"2023-07-01T18:09:54.282678Z","iopub.status.idle":"2023-07-01T18:09:55.323964Z","shell.execute_reply.started":"2023-07-01T18:09:54.282645Z","shell.execute_reply":"2023-07-01T18:09:55.322579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_train:\n    trainer = DefaultTrainer(cfg) if not is_augment else AugTrainer(cfg)\n        \n    trainer.resume_or_load(resume=is_resume_training)\n\n    trainer.train()\n    \n    print(\"#### TRAINING COMPLETE ####\")\n    _ = trainer.model.train(False)  # turn off training\n    \n    FileLink(str(OUTPUT_MODEL))","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:09:56.360384Z","iopub.execute_input":"2023-07-01T18:09:56.360798Z","iopub.status.idle":"2023-07-01T18:29:04.310937Z","shell.execute_reply.started":"2023-07-01T18:09:56.360758Z","shell.execute_reply":"2023-07-01T18:29:04.309832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 6.2 Visualizing Training Metrics","metadata":{}},{"cell_type":"code","source":"if is_train:\n    # Load metrics\n    metrics_df = pd.read_json(\n        OUTPUT_DIR/\"metrics.json\", orient=\"records\", lines=True\n    )\n    mdf = metrics_df.sort_values(\"iteration\")\n    print(mdf.head(10).T)\n\n    # Plot loss\n    fig, ax = plt.subplots()\n\n    mdf1 = mdf[~mdf[\"total_loss\"].isna()]\n    ax.plot(mdf1[\"iteration\"], mdf1[\"total_loss\"], c=\"C0\", label=\"train\")\n\n    if \"validation_loss\" in mdf.columns:\n        mdf2 = mdf[~mdf[\"validation_loss\"].isna()]\n        ax.plot(mdf2[\"iteration\"], mdf2[\"validation_loss\"],\n                c=\"C1\", label=\"validation\")\n\n    ax.legend()\n    ax.set_title(\"Loss curve\")\n    plt.show()\n\n    # Plot Accuracy\n    fig, ax = plt.subplots()\n\n    mdf1 = mdf[~mdf[\"fast_rcnn/cls_accuracy\"].isna()]\n    ax.plot(mdf1[\"iteration\"], mdf1[\"fast_rcnn/cls_accuracy\"],\n            c=\"C0\", label=\"train\")\n\n    ax.legend()\n    ax.set_title(\"Accuracy curve\")\n    plt.show()\n\n    # Plot Bounding Box regressor loss\n    fig, ax = plt.subplots()\n\n    mdf1 = mdf[~mdf[\"loss_box_reg\"].isna()]\n    ax.plot(mdf1[\"iteration\"], mdf1[\"loss_box_reg\"], c=\"C0\", label=\"train\")\n\n    ax.legend()\n    ax.set_title(\"loss_box_reg\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:29:04.313554Z","iopub.execute_input":"2023-07-01T18:29:04.314444Z","iopub.status.idle":"2023-07-01T18:29:05.418012Z","shell.execute_reply.started":"2023-07-01T18:29:04.314390Z","shell.execute_reply":"2023-07-01T18:29:05.417062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 7. Evaluation","metadata":{}},{"cell_type":"markdown","source":"Can evaluate trained model on validation dataset to obtain different metric scores. ","metadata":{}},{"cell_type":"code","source":"if is_evaluate:\n    print(\"### EVALUATING ON VALIDATION DATA ####\")\n    # trained model weights\n    cfg.MODEL.WEIGHTS = str(MODEL_PATH)\n    cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5   # set a custom testing threshold\n\n    cfg.SOLVER.IMS_PER_BATCH = 64\n\n    evaluator = COCOEvaluator(\n        DATA_REGISTER_VALID, cfg, False, output_dir=cfg.OUTPUT_DIR, use_fast_impl=True\n    )\n\n    val_loader = build_detection_test_loader(cfg, DATA_REGISTER_VALID)\n\n    results = inference_on_dataset(\n        trainer.model, val_loader, evaluator=evaluator\n    )","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:31:02.157849Z","iopub.execute_input":"2023-07-01T18:31:02.158278Z","iopub.status.idle":"2023-07-01T18:31:02.168402Z","shell.execute_reply.started":"2023-07-01T18:31:02.158243Z","shell.execute_reply":"2023-07-01T18:31:02.167205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 8. Inference","metadata":{}},{"cell_type":"markdown","source":"### 8.1 Setting Up Inference Model","metadata":{}},{"cell_type":"code","source":"inf_cfg = get_cfg()\n\nconfig_name = \"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"\n\ninf_cfg.merge_from_file(model_zoo.get_config_file(config_name))\ninf_cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128\ninf_cfg.MODEL.ROI_HEADS.NUM_CLASSES = 4\ninf_cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5\ninf_cfg.MODEL.DEVICE = \"cuda\"\n\ninf_cfg.DATALOADER.NUM_WORKERS = 2  # lower this if CUDA overflow occurs\ninf_cfg.MODEL.WEIGHTS = str(MODEL_PATH)","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:31:18.495491Z","iopub.execute_input":"2023-07-01T18:31:18.495869Z","iopub.status.idle":"2023-07-01T18:31:18.521564Z","shell.execute_reply.started":"2023-07-01T18:31:18.495839Z","shell.execute_reply":"2023-07-01T18:31:18.520618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH = 8  # lower this if CUDA overflow occurs\ntest_loader = build_detection_test_loader(inf_cfg, DATA_REGISTER_TEST, batch_size=BATCH)","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:31:23.333512Z","iopub.execute_input":"2023-07-01T18:31:23.333882Z","iopub.status.idle":"2023-07-01T18:31:24.616986Z","shell.execute_reply.started":"2023-07-01T18:31:23.333852Z","shell.execute_reply":"2023-07-01T18:31:24.615990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"How confident should the model be for you to accept the predicted mask?","metadata":{}},{"cell_type":"code","source":"ACCEPTANCE_THRESHOLD = 0.6  # for all categories","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:31:34.723024Z","iopub.execute_input":"2023-07-01T18:31:34.724044Z","iopub.status.idle":"2023-07-01T18:31:34.728946Z","shell.execute_reply.started":"2023-07-01T18:31:34.724000Z","shell.execute_reply":"2023-07-01T18:31:34.727919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"#### MODEL: {inf_cfg.MODEL.WEIGHTS} FOR INFERENCE ####\")","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:31:39.460007Z","iopub.execute_input":"2023-07-01T18:31:39.460455Z","iopub.status.idle":"2023-07-01T18:31:39.465481Z","shell.execute_reply.started":"2023-07-01T18:31:39.460423Z","shell.execute_reply":"2023-07-01T18:31:39.464393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 8.2 Sample Inference","metadata":{}},{"cell_type":"code","source":"predictor = DefaultPredictor(inf_cfg)","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:31:51.094274Z","iopub.execute_input":"2023-07-01T18:31:51.095252Z","iopub.status.idle":"2023-07-01T18:31:52.256822Z","shell.execute_reply.started":"2023-07-01T18:31:51.095204Z","shell.execute_reply":"2023-07-01T18:31:52.255829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(2, 2, figsize=(20, 20))\nindices = [ax[0][0], ax[1][0], ax[0][1], ax[1][1]]\n\n# Show some qualitative results by predicting on test set images\nNUM_TEST_SAMPLES = 4\nsamples = np.random.choice(dataset_dicts_test, NUM_TEST_SAMPLES)\n\nfor i, sample in enumerate(samples):\n    img = cv2.imread(sample[\"file_name\"])\n    outputs = predictor(img)\n    visualizer = Visualizer(img, metadata=metadata_dicts_test, scale=0.5,)\n    visualizer = visualizer.draw_instance_predictions(\n        outputs[\"instances\"].to(\"cpu\")\n    )\n    display_img = visualizer.get_image()[:, :, ::-1]\n    indices[i].grid(False)\n    indices[i].imshow(display_img)","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:31:55.794996Z","iopub.execute_input":"2023-07-01T18:31:55.795990Z","iopub.status.idle":"2023-07-01T18:32:06.697625Z","shell.execute_reply.started":"2023-07-01T18:31:55.795953Z","shell.execute_reply":"2023-07-01T18:32:06.696772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 8.3 Test Data Inference and Submission","metadata":{}},{"cell_type":"markdown","source":"### 8.3.1 Building Inference Model","metadata":{}},{"cell_type":"code","source":"def rebuild_model():\n    model = build_model(inf_cfg)\n    _ = DetectionCheckpointer(model).load(inf_cfg.MODEL.WEIGHTS)\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:33:42.865623Z","iopub.execute_input":"2023-07-01T18:33:42.866029Z","iopub.status.idle":"2023-07-01T18:33:42.872061Z","shell.execute_reply.started":"2023-07-01T18:33:42.865994Z","shell.execute_reply":"2023-07-01T18:33:42.870987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = rebuild_model()","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:33:48.578820Z","iopub.execute_input":"2023-07-01T18:33:48.579204Z","iopub.status.idle":"2023-07-01T18:33:49.680396Z","shell.execute_reply.started":"2023-07-01T18:33:48.579173Z","shell.execute_reply":"2023-07-01T18:33:49.679410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 8.3.2 CUDA Problems","metadata":{}},{"cell_type":"code","source":"!export LRU_CACHE_CAPACITY=1\n!export 'PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:512'","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:33:56.396155Z","iopub.execute_input":"2023-07-01T18:33:56.396511Z","iopub.status.idle":"2023-07-01T18:33:58.592795Z","shell.execute_reply.started":"2023-07-01T18:33:56.396481Z","shell.execute_reply":"2023-07-01T18:33:58.591428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Occassionally CUDA memory overflow occurs. Trying to save as much VRAM as we can. ","metadata":{}},{"cell_type":"code","source":"vars_to_del = [\"trainer\", \"predictor\", \"outputs\"]\n\nfor v in vars_to_del:\n    if v in globals():\n        print(f\"Deleting {v}\")\n        del globals()[v]\n    elif v in locals():\n        print(f\"Deleting {v}\")\n        del locals()[v]","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:34:12.002628Z","iopub.execute_input":"2023-07-01T18:34:12.003125Z","iopub.status.idle":"2023-07-01T18:34:12.166659Z","shell.execute_reply.started":"2023-07-01T18:34:12.003083Z","shell.execute_reply":"2023-07-01T18:34:12.165658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 8.3.3 Inference Utils","metadata":{}},{"cell_type":"code","source":"def rle_encode(mask):\n    pixels = mask.T.flatten()\n    use_padding = False\n    if pixels[0] or pixels[-1]:\n        use_padding = True\n        pixel_padded = np.zeros([len(pixels) + 2], dtype=pixels.dtype)\n        pixel_padded[1:-1] = pixels\n        pixels = pixel_padded\n    rle = np.where(pixels[1:] != pixels[:-1])[0] + 2\n    if use_padding:\n        rle = rle - 1\n    rle[1::2] = rle[1::2] - rle[:-1:2]\n    return ' '.join(str(x) for x in rle)","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:34:28.849916Z","iopub.execute_input":"2023-07-01T18:34:28.850326Z","iopub.status.idle":"2023-07-01T18:34:28.858471Z","shell.execute_reply.started":"2023-07-01T18:34:28.850293Z","shell.execute_reply":"2023-07-01T18:34:28.857339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@retry_if_cuda_oom\ndef get_masks(prediction):\n    # get masks for each category\n    take = prediction.scores >= ACCEPTANCE_THRESHOLD\n    pred_masks = (prediction.pred_masks[take] != 0)\n    pred_classes = prediction.pred_classes[take]\n  \n    rles = []\n    for cat in range(len(thing_classes)):\n        pred_mask = pred_masks[pred_classes == cat]\n        \n        # pred_mask = retry_if_cuda_oom(torch.any)(pred_mask, dim=0)\n        pred_mask = torch.any(pred_mask, dim=0)\n        rles.append(rle_encode(pred_mask.short().to(\"cpu\").numpy()))\n\n    return rles","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:34:29.517912Z","iopub.execute_input":"2023-07-01T18:34:29.518292Z","iopub.status.idle":"2023-07-01T18:34:29.525313Z","shell.execute_reply.started":"2023-07-01T18:34:29.518263Z","shell.execute_reply":"2023-07-01T18:34:29.524227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_inference(data):\n    results = []\n    with torch.no_grad():\n        outputs = model(data)\n        if torch.cuda.is_available():\n            torch.cuda.synchronize()\n\n        for idx, output in enumerate(outputs):\n            output = output[\"instances\"]\n\n            rles = get_masks(output)\n\n            result = [\n                f\"{data[idx]['image_id']}_{cat},{rles[cat]}\\n\"\n                for cat in range(len(thing_classes))\n            ]\n\n            results.extend(result)\n\n        del outputs, output\n\n    return results","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:34:39.649530Z","iopub.execute_input":"2023-07-01T18:34:39.649894Z","iopub.status.idle":"2023-07-01T18:34:39.656640Z","shell.execute_reply.started":"2023-07-01T18:34:39.649865Z","shell.execute_reply":"2023-07-01T18:34:39.655609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 8.3.4 Running Inference on Test Data and Creating Submission File\nThis is super slow. ","metadata":{}},{"cell_type":"code","source":"print(\"#### RUNNING INFERENCE ON TEST DATA ####\")","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:34:51.777684Z","iopub.execute_input":"2023-07-01T18:34:51.778081Z","iopub.status.idle":"2023-07-01T18:34:51.783670Z","shell.execute_reply.started":"2023-07-01T18:34:51.778048Z","shell.execute_reply":"2023-07-01T18:34:51.782182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:34:57.500588Z","iopub.execute_input":"2023-07-01T18:34:57.500956Z","iopub.status.idle":"2023-07-01T18:34:59.603010Z","shell.execute_reply.started":"2023-07-01T18:34:57.500925Z","shell.execute_reply":"2023-07-01T18:34:59.602079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_inference:\n    model.eval()\n    submission_file = open(\"submission.csv\", \"w\")\n    submission_file.write(\"Id,Predicted\\n\")\n\n    results: list[str] = []\n    \n    for i, data in enumerate(tqdm(test_loader)):\n        res = run_inference(data)\n        results.extend(res)\n        \n        if i % (500 // BATCH) == 0:\n            print(f\"Inference on batch {i}/{len(test_loader)} done\")\n            submission_file.writelines(results)\n            results = []\n\n    submission_file.writelines(results)\n    submission_file.close()","metadata":{"execution":{"iopub.status.busy":"2023-07-01T18:35:06.298777Z","iopub.execute_input":"2023-07-01T18:35:06.299856Z","iopub.status.idle":"2023-07-01T19:18:05.245350Z","shell.execute_reply.started":"2023-07-01T18:35:06.299814Z","shell.execute_reply":"2023-07-01T19:18:05.244107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if Path(\"submission.csv\").exists:\n    display(FileLink(\"submission.csv\"))","metadata":{"execution":{"iopub.status.busy":"2023-07-01T19:18:12.394861Z","iopub.execute_input":"2023-07-01T19:18:12.395346Z","iopub.status.idle":"2023-07-01T19:18:12.404582Z","shell.execute_reply.started":"2023-07-01T19:18:12.395301Z","shell.execute_reply":"2023-07-01T19:18:12.402861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r detectron2/","metadata":{},"execution_count":null,"outputs":[]}]}