{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is combination of [Starter Notebook](https://www.kaggle.com/code/salmankhondker/starter-notebook-dl-sprint-2-0) and [EDA](https://www.kaggle.com/code/umongsain/eda-dl-sprint-2-0) to have everthing is one place and a complete guide to start.\n\nHowever, more tuning will be done in this notebook to have an efficient model.","metadata":{"papermill":{"duration":0.020207,"end_time":"2023-07-24T12:56:55.860631","exception":false,"start_time":"2023-07-24T12:56:55.840424","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Detectron2\n\n*[Detectron2](https://detectron2.readthedocs.io/en/latest/index.html) is a popular open-source software library developed by Facebook AI Research (FAIR) for building computer vision models. It serves as a powerful framework for object detection, instance segmentation, and keypoint detection tasks. Detectron2 is built on top of PyTorch, geared towards a more convenient way to build modular, flexible pipelines for specific Computer Vision Tasks such as object detection, instance segmentation.*\n\n*Detectron2 has a collection of trained models for these tasks in their [model zoo](https://github.com/facebookresearch/detectron2/blob/main/MODEL_ZOO.md). We can also use detectron2 to train pre-implemented state-of-the-art models from scratch for new datasets, as we have done in this notebook.*\n\n*Read the [Detectron2-Documentation](https://detectron2.readthedocs.io/en/latest/index.html) !!!*","metadata":{"papermill":{"duration":0.01901,"end_time":"2023-07-24T12:56:55.899216","exception":false,"start_time":"2023-07-24T12:56:55.880206","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# 1. Import","metadata":{"papermill":{"duration":0.019107,"end_time":"2023-07-24T12:56:55.941389","exception":false,"start_time":"2023-07-24T12:56:55.922282","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 1.1 Import Detectron2","metadata":{"papermill":{"duration":0.019272,"end_time":"2023-07-24T12:56:55.979821","exception":false,"start_time":"2023-07-24T12:56:55.960549","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# !git clone https://github.com/facebookresearch/detectron2.git\n# !python -m pip install -e detectron2","metadata":{"execution":{"iopub.execute_input":"2023-07-24T12:56:56.021457Z","iopub.status.busy":"2023-07-24T12:56:56.02063Z","iopub.status.idle":"2023-07-24T12:56:56.025621Z","shell.execute_reply":"2023-07-24T12:56:56.0248Z"},"papermill":{"duration":0.029588,"end_time":"2023-07-24T12:56:56.029061","exception":false,"start_time":"2023-07-24T12:56:55.999473","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!pip install 'git+https://github.com/facebookresearch/detectron2.git@5aeb252b194b93dc2879b4ac34bc51a31b5aee13'","metadata":{"execution":{"iopub.execute_input":"2023-07-24T12:56:56.069475Z","iopub.status.busy":"2023-07-24T12:56:56.068555Z","iopub.status.idle":"2023-07-24T13:00:51.146599Z","shell.execute_reply":"2023-07-24T13:00:51.145371Z"},"papermill":{"duration":235.101,"end_time":"2023-07-24T13:00:51.149261","exception":false,"start_time":"2023-07-24T12:56:56.048261","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.2 Import Libraries","metadata":{"papermill":{"duration":0.019238,"end_time":"2023-07-24T13:00:51.18838","exception":false,"start_time":"2023-07-24T13:00:51.169142","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%capture\n# detectron2\nfrom detectron2.utils.memory import retry_if_cuda_oom\nfrom detectron2.utils.logger import setup_logger\nfrom detectron2.checkpoint import DetectionCheckpointer\nfrom detectron2.modeling import build_model\nfrom detectron2.evaluation import COCOEvaluator, inference_on_dataset\nimport detectron2.data.transforms as T\nfrom detectron2.data import detection_utils as utils\nfrom detectron2.data import DatasetCatalog, MetadataCatalog, build_detection_test_loader, build_detection_train_loader, DatasetMapper\nfrom detectron2.utils.visualizer import Visualizer\nfrom detectron2.structures import BoxMode\nfrom detectron2.engine import DefaultPredictor, DefaultTrainer\nfrom detectron2.config import get_cfg\nfrom detectron2 import model_zoo\n\nimport pandas as pd\nimport numpy as np\nfrom tqdm.notebook import tqdm  # progress bar\nimport matplotlib.pyplot as plt\nimport json\nimport cv2\nimport copy\nfrom typing import Optional\nimport seaborn as sns\n\n\n!pip install -q pycocotools\nfrom pycocotools.coco import COCO\nfrom PIL import Image\nimport random\nfrom pathlib import Path\n%matplotlib inline\nsns.set_theme(style='darkgrid', palette='deep', font='sans-serif', font_scale=1)\n\nfrom IPython.display import FileLink\n# torch\nimport torch\nimport gc\nimport warnings\n# Ignore \"future\" warnings and Data-Frame-Slicing warnings.\nwarnings.filterwarnings('ignore')\n\nsetup_logger()","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:00:51.229063Z","iopub.status.busy":"2023-07-24T13:00:51.228039Z","iopub.status.idle":"2023-07-24T13:01:06.760324Z","shell.execute_reply":"2023-07-24T13:01:06.759116Z"},"papermill":{"duration":15.555306,"end_time":"2023-07-24T13:01:06.762883","exception":false,"start_time":"2023-07-24T13:00:51.207577","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.3 Data Load","metadata":{"papermill":{"duration":0.018874,"end_time":"2023-07-24T13:01:06.801043","exception":false,"start_time":"2023-07-24T13:01:06.782169","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from pathlib import Path\n\nTRAIN_IMG_DIR = Path(\"/kaggle/input/dlsprint2/badlad/images/train\")\n\nTRAIN_COCO_PATH = Path(\"/kaggle/input/dlsprint2/badlad/labels/coco_format/train/badlad-train-coco.json\")\n\nTEST_IMG_DIR = Path(\"/kaggle/input/dlsprint2/badlad/images/test\")\n\nTEST_METADATA_PATH = Path(\"/kaggle/input/dlsprint2/badlad/badlad-test-metadata.json\")\n\n# Training output directory\nOUTPUT_DIR = Path(\"./output\")\nOUTPUT_MODEL = OUTPUT_DIR/\"model_final.pth\"\n\n# Path to your pretrained model weights\nPRETRAINED_PATH = Path(\"/kaggle/input/shrestha110k88894/shrestha110k88894.pth\")","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:01:06.841172Z","iopub.status.busy":"2023-07-24T13:01:06.840802Z","iopub.status.idle":"2023-07-24T13:01:06.846692Z","shell.execute_reply":"2023-07-24T13:01:06.845865Z"},"papermill":{"duration":0.028677,"end_time":"2023-07-24T13:01:06.84862","exception":false,"start_time":"2023-07-24T13:01:06.819943","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.4 COCO Annotations Data","metadata":{"papermill":{"duration":0.018611,"end_time":"2023-07-24T13:01:06.886278","exception":false,"start_time":"2023-07-24T13:01:06.867667","status":"completed"},"tags":[]}},{"cell_type":"code","source":"with TRAIN_COCO_PATH.open() as f:\n    train_dict = json.load(f)\n\nwith TEST_METADATA_PATH.open() as f:\n    test_dict = json.load(f)\n    \ntrain_coco_labels=COCO(annotation_file=TRAIN_COCO_PATH)\n\nprint(\"#### LABELS AND METADATA LOADED ####\")","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:01:06.925343Z","iopub.status.busy":"2023-07-24T13:01:06.925004Z","iopub.status.idle":"2023-07-24T13:01:19.720101Z","shell.execute_reply":"2023-07-24T13:01:19.71875Z"},"papermill":{"duration":12.817,"end_time":"2023-07-24T13:01:19.722243","exception":false,"start_time":"2023-07-24T13:01:06.905243","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.5 Create Decisions","metadata":{"papermill":{"duration":0.01913,"end_time":"2023-07-24T13:01:19.760819","exception":false,"start_time":"2023-07-24T13:01:19.741689","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from datetime import datetime\n\n# if False, model is set to `PRETRAINED_PATH` model\nis_train = True\n\n# if True, evaluate on validation dataset\nis_evaluate = False\n\n# if True, run inference on test dataset\nis_inference = True\n\n# if True and `is_train` == True, `PRETRAINED_PATH` model is trained further\nis_resume_training = True\n\n# Perform augmentation\nis_augment = False\n\nSEED = int(datetime.now().timestamp())\n\n# Model path based on Decisions\nMODEL_PATH = OUTPUT_MODEL if is_train else PRETRAINED_PATH","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:01:19.800476Z","iopub.status.busy":"2023-07-24T13:01:19.79961Z","iopub.status.idle":"2023-07-24T13:01:19.805325Z","shell.execute_reply":"2023-07-24T13:01:19.804295Z"},"papermill":{"duration":0.027994,"end_time":"2023-07-24T13:01:19.807658","exception":false,"start_time":"2023-07-24T13:01:19.779664","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Dataset","metadata":{"papermill":{"duration":0.018918,"end_time":"2023-07-24T13:01:19.845814","exception":false,"start_time":"2023-07-24T13:01:19.826896","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 2.1 Dataset Analysis","metadata":{"papermill":{"duration":0.018879,"end_time":"2023-07-24T13:01:19.884108","exception":false,"start_time":"2023-07-24T13:01:19.865229","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(\"There are \" + str(len(train_dict['categories'])) + \" categories.\\n\")\nprint(\"There are \" + str(len(test_dict['images']) + len(train_dict['images'])) + \" images in the dataset.\")\nprint(\"There are \" + str(len(train_dict['images'])) + \" images in the train set.\")\nprint(\"There are \" + str(len(test_dict['images'])) + \" images in the test set.\\n\")\nprint(\"There are \" + str(len(train_dict['annotations'])) + \" annotations in the train set.\\n\")\n\nprint(\"We will focus on mainly categories, images and annotations.\")","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:01:19.924081Z","iopub.status.busy":"2023-07-24T13:01:19.923742Z","iopub.status.idle":"2023-07-24T13:01:19.930845Z","shell.execute_reply":"2023-07-24T13:01:19.929955Z"},"papermill":{"duration":0.029712,"end_time":"2023-07-24T13:01:19.933138","exception":false,"start_time":"2023-07-24T13:01:19.903426","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def organize_coco_data(data_dict: dict) -> tuple[list[str], list[dict], list[dict]]:\n    thing_classes: list[str] = []\n\n    # Map Category Names to IDs\n    for cat in data_dict['categories']:\n        thing_classes.append(cat['name'])\n\n    # Images\n    images_metadata: list[dict] = data_dict['images']\n\n    # Convert COCO annotations to detectron2 annotations format\n    data_annotations = []\n    for ann in data_dict['annotations']:\n        # coco format -> detectron2 format\n        annot_obj = {\n            # Annotation ID\n            \"id\": ann['id'],\n\n            # Segmentation Polygon (x, y) coordinnates\n            \"gt_masks\": ann['segmentation'],\n\n            # Image ID for this annotation (Which image does this annotation belong to?)\n            \"image_id\": ann['image_id'],\n\n            # Category Label (0: paragraph, 1: text box, 2: image, 3: table)\n            \"category_id\": ann['category_id'],\n\n            \"x_min\": ann['bbox'][0],  # left\n            \"y_min\": ann['bbox'][1],  # top\n            \"x_max\": ann['bbox'][0] + ann['bbox'][2],  # left+width\n            \"y_max\": ann['bbox'][1] + ann['bbox'][3]  # top+height\n        }\n        data_annotations.append(annot_obj)\n\n    return thing_classes, images_metadata, data_annotations","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:01:19.973159Z","iopub.status.busy":"2023-07-24T13:01:19.972836Z","iopub.status.idle":"2023-07-24T13:01:19.99661Z","shell.execute_reply":"2023-07-24T13:01:19.995715Z"},"papermill":{"duration":0.046327,"end_time":"2023-07-24T13:01:19.998577","exception":false,"start_time":"2023-07-24T13:01:19.95225","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"thing_classes, images_metadata, data_annotations = organize_coco_data(train_dict)\n\nthing_classes_test, images_metadata_test, _ = organize_coco_data(test_dict)\n\nprint(thing_classes)","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:01:20.038842Z","iopub.status.busy":"2023-07-24T13:01:20.038066Z","iopub.status.idle":"2023-07-24T13:01:21.724624Z","shell.execute_reply":"2023-07-24T13:01:21.723459Z"},"papermill":{"duration":1.709626,"end_time":"2023-07-24T13:01:21.727507","exception":false,"start_time":"2023-07-24T13:01:20.017881","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata = pd.DataFrame(images_metadata)\ntrain_metadata = train_metadata[['id', 'file_name', 'width', 'height']]\ntrain_metadata = train_metadata.rename(columns={\"id\": \"image_id\"})\nprint(\"train_metadata size=\", len(train_metadata))\ntrain_metadata.head(5)","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:01:21.768336Z","iopub.status.busy":"2023-07-24T13:01:21.767451Z","iopub.status.idle":"2023-07-24T13:01:21.855978Z","shell.execute_reply":"2023-07-24T13:01:21.855006Z"},"papermill":{"duration":0.111667,"end_time":"2023-07-24T13:01:21.858677","exception":false,"start_time":"2023-07-24T13:01:21.74701","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_annot_df = pd.DataFrame(data_annotations)\nprint(\"train_annot_df size=\", len(train_annot_df))\ntrain_annot_df.head(5)","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:01:21.900631Z","iopub.status.busy":"2023-07-24T13:01:21.900298Z","iopub.status.idle":"2023-07-24T13:01:23.350955Z","shell.execute_reply":"2023-07-24T13:01:23.350025Z"},"papermill":{"duration":1.473772,"end_time":"2023-07-24T13:01:23.35327","exception":false,"start_time":"2023-07-24T13:01:21.879498","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Here `gt_masks` are the sequence of `(x, y)` coordinates of vertices of the polygon surrounding the target object. ","metadata":{"papermill":{"duration":0.019816,"end_time":"2023-07-24T13:01:23.393258","exception":false,"start_time":"2023-07-24T13:01:23.373442","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test_metadata = pd.DataFrame(images_metadata_test)\ntest_metadata = test_metadata[['id', 'file_name', 'width', 'height']]\ntest_metadata = test_metadata.rename(columns={\"id\": \"image_id\"})\nprint(\"test_metadata size=\", len(test_metadata))\ntest_metadata.head(5)","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:01:23.434966Z","iopub.status.busy":"2023-07-24T13:01:23.43435Z","iopub.status.idle":"2023-07-24T13:01:23.506145Z","shell.execute_reply":"2023-07-24T13:01:23.504713Z"},"papermill":{"duration":0.09619,"end_time":"2023-07-24T13:01:23.509578","exception":false,"start_time":"2023-07-24T13:01:23.413388","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15, 15))\nfor i in range(6):\n    image_id = random.randint(2, 20364)\n    image_file = train_coco_labels.loadImgs([image_id])[0]['file_name']\n    image = Image.open(TRAIN_IMG_DIR/image_file)\n    code = 231 + i;\n    plt.subplot(code)\n    plt.axis('off')\n    plt.imshow(np.asarray(image))","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:01:23.565082Z","iopub.status.busy":"2023-07-24T13:01:23.564742Z","iopub.status.idle":"2023-07-24T13:01:29.250621Z","shell.execute_reply":"2023-07-24T13:01:29.249782Z"},"papermill":{"duration":5.718024,"end_time":"2023-07-24T13:01:29.261112","exception":false,"start_time":"2023-07-24T13:01:23.543088","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_image(_image_id):\n    image_file = train_coco_labels.loadImgs([_image_id])[0]['file_name']\n    image = Image.open(TRAIN_IMG_DIR/image_file)\n    plt.figure(figsize=(15, 15))\n    plt.axis('off')\n    plt.imshow(np.asarray(image))","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:01:29.325303Z","iopub.status.busy":"2023-07-24T13:01:29.324942Z","iopub.status.idle":"2023-07-24T13:01:29.330628Z","shell.execute_reply":"2023-07-24T13:01:29.329763Z"},"papermill":{"duration":0.039267,"end_time":"2023-07-24T13:01:29.332719","exception":false,"start_time":"2023-07-24T13:01:29.293452","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_segmentations(_image_id):\n    show_image(_image_id)\n    annotation_ids = train_coco_labels.getAnnIds(imgIds=[_image_id])\n    annotations = train_coco_labels.loadAnns(annotation_ids)\n    train_coco_labels.showAnns(annotations)","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:01:29.394943Z","iopub.status.busy":"2023-07-24T13:01:29.394019Z","iopub.status.idle":"2023-07-24T13:01:29.399396Z","shell.execute_reply":"2023-07-24T13:01:29.398571Z"},"papermill":{"duration":0.038639,"end_time":"2023-07-24T13:01:29.401317","exception":false,"start_time":"2023-07-24T13:01:29.362678","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# you can change the image_id\nshow_image(50)","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:01:29.463461Z","iopub.status.busy":"2023-07-24T13:01:29.462589Z","iopub.status.idle":"2023-07-24T13:01:30.139435Z","shell.execute_reply":"2023-07-24T13:01:30.138614Z"},"papermill":{"duration":0.716733,"end_time":"2023-07-24T13:01:30.147873","exception":false,"start_time":"2023-07-24T13:01:29.43114","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# you can change the image_id\nshow_segmentations(50)","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:01:30.22592Z","iopub.status.busy":"2023-07-24T13:01:30.225545Z","iopub.status.idle":"2023-07-24T13:01:30.942609Z","shell.execute_reply":"2023-07-24T13:01:30.941806Z"},"papermill":{"duration":0.765577,"end_time":"2023-07-24T13:01:30.951157","exception":false,"start_time":"2023-07-24T13:01:30.18558","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.2 Exploring Annonations","metadata":{"papermill":{"duration":0.047007,"end_time":"2023-07-24T13:01:31.045648","exception":false,"start_time":"2023-07-24T13:01:30.998641","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"> *`annotations:` This is a list of the polygon bounding segments for each instance of the categories occurring in each image.*\n\n<br>\nThe contents of this category are the crux of the problem:\n\n* `segmentation:` Polygon (x, y) coordinate pairs of it's vertices.For example, the first annotation has its first vertex at (86.328, 179.007). The following pairs complete the polygon region of the annotation.\n* `image_id:` The image this annotation is for.\n* `category_id:` The category this annotation is bounding. It may be a paragraph, text box, image, or table.\n* `id:` Annotation ID, identifies each annotation.\n* `bbox:` The rectangular bounding box that best estimates the annotated region.\nThis list holds all the annotations for each instance of the categories for all images.","metadata":{"papermill":{"duration":0.045971,"end_time":"2023-07-24T13:01:31.13783","exception":false,"start_time":"2023-07-24T13:01:31.091859","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_annotations = pd.DataFrame(train_dict['annotations'])\ntrain_annotations.head()","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:01:31.23321Z","iopub.status.busy":"2023-07-24T13:01:31.231932Z","iopub.status.idle":"2023-07-24T13:01:32.693691Z","shell.execute_reply":"2023-07-24T13:01:32.692656Z"},"papermill":{"duration":1.5116,"end_time":"2023-07-24T13:01:32.696001","exception":false,"start_time":"2023-07-24T13:01:31.184401","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# adding bbox_area and bbox_aspect_ratio\nprint(\"\\nRenaming id to annotation_id.\\nAdding bbox_area, bbox_aspect_ratio.\\n\")\ntrain_annotations.rename(columns={\"id\":\"annotation_id\"}, inplace=True)\nbbox_area=[]\nbbox_aspect_ratio=[]\nfor idx in train_annotations.index:\n    bbox_area.append(train_annotations['bbox'][idx][3]*train_annotations['bbox'][idx][2])\n    bbox_aspect_ratio.append(train_annotations['bbox'][idx][3]/train_annotations['bbox'][idx][2])\ntrain_annotations['bbox_area']=bbox_area\ntrain_annotations['bbox_aspect_ratio']=bbox_aspect_ratio\nprint(\"train_annotations shape: \" + str(train_annotations.shape) + \"\\n\")\ntrain_annotations.head()","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:01:32.792633Z","iopub.status.busy":"2023-07-24T13:01:32.791625Z","iopub.status.idle":"2023-07-24T13:01:49.102052Z","shell.execute_reply":"2023-07-24T13:01:49.099785Z"},"papermill":{"duration":16.361777,"end_time":"2023-07-24T13:01:49.104248","exception":false,"start_time":"2023-07-24T13:01:32.742471","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_annotations.describe()","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:01:49.199702Z","iopub.status.busy":"2023-07-24T13:01:49.198698Z","iopub.status.idle":"2023-07-24T13:01:49.305199Z","shell.execute_reply":"2023-07-24T13:01:49.304287Z"},"papermill":{"duration":0.15641,"end_time":"2023-07-24T13:01:49.307195","exception":false,"start_time":"2023-07-24T13:01:49.150785","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.3 Categorical Calssification","metadata":{"papermill":{"duration":0.048838,"end_time":"2023-07-24T13:01:49.402846","exception":false,"start_time":"2023-07-24T13:01:49.354008","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_categories = pd.DataFrame(train_dict['categories'])\n\nprint(\"Dropping supercategory.\\nRenaming id to category_id.\\n\")\ntrain_categories.drop('supercategory', axis=1, inplace=True)\ntrain_categories.rename(columns={\"id\":\"category_id\"}, inplace=True)\ntrain_categories","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:01:49.500064Z","iopub.status.busy":"2023-07-24T13:01:49.499365Z","iopub.status.idle":"2023-07-24T13:01:49.511475Z","shell.execute_reply":"2023-07-24T13:01:49.51056Z"},"papermill":{"duration":0.065409,"end_time":"2023-07-24T13:01:49.515099","exception":false,"start_time":"2023-07-24T13:01:49.44969","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_cat_count = train_annotations[['image_id', 'category_id']].copy()\ncategory_names=[]\nfor idx in img_cat_count.index:\n    category_names.append(train_categories['name'][img_cat_count['category_id'][idx]])\nimg_cat_count['category_name']=category_names\nimg_cat_count[train_categories['name'][0]] = img_cat_count['category_id'] == train_categories['category_id'][0]\nimg_cat_count[train_categories['name'][1]] = img_cat_count['category_id'] == train_categories['category_id'][1]\nimg_cat_count[train_categories['name'][2]] = img_cat_count['category_id'] == train_categories['category_id'][2]\nimg_cat_count[train_categories['name'][3]] = img_cat_count['category_id'] == train_categories['category_id'][3]\nimg_cat_count.head()","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:01:49.609566Z","iopub.status.busy":"2023-07-24T13:01:49.609195Z","iopub.status.idle":"2023-07-24T13:01:57.374682Z","shell.execute_reply":"2023-07-24T13:01:57.373651Z"},"papermill":{"duration":7.815897,"end_time":"2023-07-24T13:01:57.376919","exception":false,"start_time":"2023-07-24T13:01:49.561022","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"\\nCategorywise Document Image Count.\\n\")\ncategorywise_image_count=img_cat_count.groupby('category_id', as_index=False)['image_id'].nunique()\ncategorywise_image_count['category_name']=train_categories['name']\ncategorywise_image_count.rename(columns={'image_id':'image_count'}, inplace=True)\ncategorywise_image_count = categorywise_image_count[['category_id', 'category_name', 'image_count']]\ncategorywise_image_count","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:01:57.476217Z","iopub.status.busy":"2023-07-24T13:01:57.475862Z","iopub.status.idle":"2023-07-24T13:01:57.516884Z","shell.execute_reply":"2023-07-24T13:01:57.515649Z"},"papermill":{"duration":0.093332,"end_time":"2023-07-24T13:01:57.518963","exception":false,"start_time":"2023-07-24T13:01:57.425631","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Insights**\n> * Text box and paragraph appears almost in all the documents.\n> * Image appears almost in one-fourth of the documents of trainset.\n> * Tables are not that much frequent like other 3 classes. 1 in 20 documents contains table.","metadata":{"papermill":{"duration":0.046714,"end_time":"2023-07-24T13:01:57.613036","exception":false,"start_time":"2023-07-24T13:01:57.566322","status":"completed"},"tags":[]}},{"cell_type":"code","source":"plt.title(\"Categories vs Number of document images they appear\")\nsns.barplot(x=categorywise_image_count['category_name'], y=categorywise_image_count['image_count'])","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:01:57.714527Z","iopub.status.busy":"2023-07-24T13:01:57.713456Z","iopub.status.idle":"2023-07-24T13:01:58.106542Z","shell.execute_reply":"2023-07-24T13:01:58.105643Z"},"papermill":{"duration":0.448198,"end_time":"2023-07-24T13:01:58.108901","exception":false,"start_time":"2023-07-24T13:01:57.660703","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"\\nImagewise Category Count.\\n\")\nimagewise_category_count=img_cat_count.groupby('image_id', as_index=False)[['paragraph', 'text_box', 'image', 'table']].sum()\nimagewise_category_count.head()","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:01:58.210162Z","iopub.status.busy":"2023-07-24T13:01:58.208982Z","iopub.status.idle":"2023-07-24T13:01:58.248883Z","shell.execute_reply":"2023-07-24T13:01:58.247799Z"},"papermill":{"duration":0.093566,"end_time":"2023-07-24T13:01:58.251026","exception":false,"start_time":"2023-07-24T13:01:58.15746","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15, 30))\n\nplt.subplot(411)\nplt.title(\"Number of paragraphs per document image\")\nsns.lineplot(x=imagewise_category_count['image_id'], y=imagewise_category_count['paragraph'])\n\nplt.subplot(412)\nplt.title(\"Number of text boxes per document image\")\nsns.lineplot(x=imagewise_category_count['image_id'], y=imagewise_category_count['text_box'])\n\nplt.subplot(413)\nplt.title(\"Number of images per document image\")\nsns.lineplot(x=imagewise_category_count['image_id'], y=imagewise_category_count['image'])\n\nplt.subplot(414)\nplt.title(\"Number of tables per document image\")\nsns.lineplot(x=imagewise_category_count['image_id'], y=imagewise_category_count['table'])","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:01:58.352482Z","iopub.status.busy":"2023-07-24T13:01:58.352127Z","iopub.status.idle":"2023-07-24T13:02:00.687645Z","shell.execute_reply":"2023-07-24T13:02:00.686766Z"},"papermill":{"duration":2.388988,"end_time":"2023-07-24T13:02:00.691129","exception":false,"start_time":"2023-07-24T13:01:58.302141","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Preparing Data for Training","metadata":{"papermill":{"duration":0.050738,"end_time":"2023-07-24T13:02:00.794233","exception":false,"start_time":"2023-07-24T13:02:00.743495","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 3.1 Train-Validation Split","metadata":{"papermill":{"duration":0.050685,"end_time":"2023-07-24T13:02:00.895541","exception":false,"start_time":"2023-07-24T13:02:00.844856","status":"completed"},"tags":[]}},{"cell_type":"code","source":"TRAIN_SPLIT = 0.99","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:02:01.000345Z","iopub.status.busy":"2023-07-24T13:02:00.99997Z","iopub.status.idle":"2023-07-24T13:02:01.004428Z","shell.execute_reply":"2023-07-24T13:02:01.003555Z"},"papermill":{"duration":0.059527,"end_time":"2023-07-24T13:02:01.00647","exception":false,"start_time":"2023-07-24T13:02:00.946943","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_dataset = len(train_metadata)\nn_train = int(n_dataset * TRAIN_SPLIT)\nprint(\"n_dataset\", n_dataset, \"n_train\", n_train, \"n_val\", n_dataset-n_train)\n\nnp.random.seed(SEED)\n\ninds = np.random.permutation(n_dataset)\ntrain_inds, valid_inds = inds[:n_train], inds[n_train:]","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:02:01.111033Z","iopub.status.busy":"2023-07-24T13:02:01.109989Z","iopub.status.idle":"2023-07-24T13:02:01.117744Z","shell.execute_reply":"2023-07-24T13:02:01.116647Z"},"papermill":{"duration":0.063065,"end_time":"2023-07-24T13:02:01.119772","exception":false,"start_time":"2023-07-24T13:02:01.056707","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.2 Formatting Data for `detectron2`","metadata":{"papermill":{"duration":0.050729,"end_time":"2023-07-24T13:02:01.220699","exception":false,"start_time":"2023-07-24T13:02:01.16997","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def convert_coco_to_detectron2_format(\n    imgdir: Path,\n    metadata_df: pd.DataFrame,\n    annot_df: Optional[pd.DataFrame] = None,\n    target_indices: Optional[np.ndarray] = None,\n):\n\n    dataset_dicts = []\n    for _, train_meta_row in tqdm(metadata_df.iterrows(), total=len(metadata_df)):\n        # Iterate over each image\n        image_id, filename, width, height = train_meta_row.values\n\n        annotations = []\n\n        # If train/validation data, then there will be annotations\n        if annot_df is not None:\n            for _, ann in annot_df.query(\"image_id == @image_id\").iterrows():\n                # Get annotations of current iteration's image\n                class_id = ann[\"category_id\"]\n                gt_masks = ann[\"gt_masks\"]\n                bbox_resized = [\n                    float(ann[\"x_min\"]),\n                    float(ann[\"y_min\"]),\n                    float(ann[\"x_max\"]),\n                    float(ann[\"y_max\"]),\n                ]\n\n                annotation = {\n                    \"bbox\": bbox_resized,\n                    \"bbox_mode\": BoxMode.XYXY_ABS,\n                    \"segmentation\": gt_masks,\n                    \"category_id\": class_id,\n                }\n\n                annotations.append(annotation)\n\n        # coco format -> detectron2 format dict\n        record = {\n            \"file_name\": str(imgdir/filename),\n            \"image_id\": image_id,\n            \"width\": width,\n            \"height\": height,\n            \"annotations\": annotations\n        }\n\n        dataset_dicts.append(record)\n\n    if target_indices is not None:\n        dataset_dicts = [dataset_dicts[i] for i in target_indices]\n\n    return dataset_dicts","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:02:01.325866Z","iopub.status.busy":"2023-07-24T13:02:01.325279Z","iopub.status.idle":"2023-07-24T13:02:01.336393Z","shell.execute_reply":"2023-07-24T13:02:01.335494Z"},"papermill":{"duration":0.066879,"end_time":"2023-07-24T13:02:01.338357","exception":false,"start_time":"2023-07-24T13:02:01.271478","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.3 Registering and Loading Data for `detectron2`","metadata":{"papermill":{"duration":0.051491,"end_time":"2023-07-24T13:02:01.440028","exception":false,"start_time":"2023-07-24T13:02:01.388537","status":"completed"},"tags":[]}},{"cell_type":"code","source":"DATA_REGISTER_TRAINING = \"badlad_train\"\nDATA_REGISTER_VALID    = \"badlad_valid\"\nDATA_REGISTER_TEST     = \"badlad_test\"","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:02:01.547929Z","iopub.status.busy":"2023-07-24T13:02:01.546822Z","iopub.status.idle":"2023-07-24T13:02:01.552071Z","shell.execute_reply":"2023-07-24T13:02:01.551209Z"},"papermill":{"duration":0.060721,"end_time":"2023-07-24T13:02:01.554042","exception":false,"start_time":"2023-07-24T13:02:01.493321","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Register Training data\nif is_train:\n    DatasetCatalog.register(\n        DATA_REGISTER_TRAINING,\n        lambda: convert_coco_to_detectron2_format(\n            TRAIN_IMG_DIR,\n            train_metadata,\n            train_annot_df,\n            target_indices=train_inds,\n        ),\n    )\n\n    # Set Training data categories\n    MetadataCatalog.get(DATA_REGISTER_TRAINING).set(thing_classes=thing_classes)\n\n    dataset_dicts_train = DatasetCatalog.get(DATA_REGISTER_TRAINING)\n    metadata_dicts_train = MetadataCatalog.get(DATA_REGISTER_TRAINING)\n\n    print(\"dicts training size=\", len(dataset_dicts_train))\n    print(\"################\")","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:02:01.657082Z","iopub.status.busy":"2023-07-24T13:02:01.656751Z","iopub.status.idle":"2023-07-24T13:03:38.598579Z","shell.execute_reply":"2023-07-24T13:03:38.597621Z"},"papermill":{"duration":96.997352,"end_time":"2023-07-24T13:03:38.600758","exception":false,"start_time":"2023-07-24T13:02:01.603406","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Register Validation data\nif is_train or is_evaluate:\n    DatasetCatalog.register(\n        DATA_REGISTER_VALID,\n        lambda: convert_coco_to_detectron2_format(\n            TRAIN_IMG_DIR,\n            train_metadata,\n            train_annot_df,\n            target_indices=valid_inds,\n        ),\n    )\n\n    # Set Validation data categories\n    MetadataCatalog.get(DATA_REGISTER_VALID).set(thing_classes=thing_classes)\n\n    dataset_dicts_valid = DatasetCatalog.get(DATA_REGISTER_VALID)\n    metadata_dicts_valid = MetadataCatalog.get(DATA_REGISTER_VALID)\n\n    print(\"dicts valid size=\", len(dataset_dicts_valid))\n    print(\"################\")","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:03:38.706877Z","iopub.status.busy":"2023-07-24T13:03:38.706021Z","iopub.status.idle":"2023-07-24T13:05:16.672023Z","shell.execute_reply":"2023-07-24T13:05:16.671045Z"},"papermill":{"duration":98.022814,"end_time":"2023-07-24T13:05:16.674613","exception":false,"start_time":"2023-07-24T13:03:38.651799","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Register Test Inference data\nDatasetCatalog.register(\n    DATA_REGISTER_TEST,\n    lambda: convert_coco_to_detectron2_format(\n        TEST_IMG_DIR,\n        test_metadata,\n    )\n)\n\n# Set Test data categories\nMetadataCatalog.get(DATA_REGISTER_TEST).set(\n    thing_classes=thing_classes_test\n)\n\ndataset_dicts_test = DatasetCatalog.get(DATA_REGISTER_TEST)\nmetadata_dicts_test = MetadataCatalog.get(DATA_REGISTER_TEST)","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:05:16.77999Z","iopub.status.busy":"2023-07-24T13:05:16.779126Z","iopub.status.idle":"2023-07-24T13:05:17.563006Z","shell.execute_reply":"2023-07-24T13:05:17.561933Z"},"papermill":{"duration":0.840666,"end_time":"2023-07-24T13:05:17.566319","exception":false,"start_time":"2023-07-24T13:05:16.725653","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"#### DATA REGISTERED ####\")","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:05:17.674219Z","iopub.status.busy":"2023-07-24T13:05:17.672267Z","iopub.status.idle":"2023-07-24T13:05:17.680321Z","shell.execute_reply":"2023-07-24T13:05:17.67939Z"},"papermill":{"duration":0.064787,"end_time":"2023-07-24T13:05:17.682411","exception":false,"start_time":"2023-07-24T13:05:17.617624","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Augmentation","metadata":{"papermill":{"duration":0.051117,"end_time":"2023-07-24T13:05:17.785201","exception":false,"start_time":"2023-07-24T13:05:17.734084","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def custom_mapper(dataset_dict):\n    dataset_dict = copy.deepcopy(dataset_dict)\n    image = utils.read_image(dataset_dict[\"file_name\"], format=\"BGR\")\n\n    transform_list = [T.RandomBrightness(0.8, 1.2),\n                      T.RandomFlip(prob=0.5, horizontal=False, vertical=True),\n                      T.RandomFlip(prob=0.5, horizontal=True, vertical=False)\n                      ]\n    image, transforms = T.apply_transform_gens(transform_list, image)\n\n    dataset_dict[\"image\"] = torch.as_tensor(\n        image.transpose(2, 0, 1).astype(\"float32\"))\n\n    annos = [\n        utils.transform_instance_annotations(obj, transforms, image.shape[:2])\n        for obj in dataset_dict.pop(\"annotations\")\n        if obj.get(\"iscrowd\", 0) == 0\n    ]\n    instances = utils.annotations_to_instances(annos, image.shape[:2])\n\n    dataset_dict[\"instances\"] = utils.filter_empty_instances(instances)\n\n    return dataset_dict","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:05:17.889911Z","iopub.status.busy":"2023-07-24T13:05:17.889203Z","iopub.status.idle":"2023-07-24T13:05:17.89762Z","shell.execute_reply":"2023-07-24T13:05:17.89666Z"},"papermill":{"duration":0.063559,"end_time":"2023-07-24T13:05:17.899674","exception":false,"start_time":"2023-07-24T13:05:17.836115","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AugTrainer(DefaultTrainer):\n    @classmethod\n    def build_train_loader(cls, cfg):\n        return build_detection_train_loader(cfg, mapper=custom_mapper)","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:05:18.004667Z","iopub.status.busy":"2023-07-24T13:05:18.004328Z","iopub.status.idle":"2023-07-24T13:05:18.00953Z","shell.execute_reply":"2023-07-24T13:05:18.008508Z"},"papermill":{"duration":0.061665,"end_time":"2023-07-24T13:05:18.011889","exception":false,"start_time":"2023-07-24T13:05:17.950224","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Hyperparameters","metadata":{"papermill":{"duration":0.051564,"end_time":"2023-07-24T13:05:18.115159","exception":false,"start_time":"2023-07-24T13:05:18.063595","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Detectron2 models need a config file to build the model. This config file has the associated hyperparameters.\n\nYou can play with this: https://detectron2.readthedocs.io/en/latest/modules/config.html#yaml-config-references","metadata":{"papermill":{"duration":0.050903,"end_time":"2023-07-24T13:05:18.217802","exception":false,"start_time":"2023-07-24T13:05:18.166899","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if is_train:\n    cfg = get_cfg()\n\n    # config_name = \"COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml\"\n    config_name = \"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"\n\n    cfg.merge_from_file(model_zoo.get_config_file(config_name))\n\n    cfg.DATASETS.TRAIN = (DATA_REGISTER_TRAINING,)\n    cfg.DATASETS.TEST = (DATA_REGISTER_VALID,)\n\n    # to evaluate during training, you have to implement `build_evaluator()` method of the trainer.\n    # https://github.com/facebookresearch/detectron2/blob/94113be6e12db36b8c7601e13747587f19ec92fe/detectron2/engine/defaults.py#L561\n    # cfg.TEST.EVAL_PERIOD = 500\n\n    cfg.DATALOADER.NUM_WORKERS = 2\n\n    # cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(config_name)\n    if (is_resume_training):\n        print(\"#### SETTING PRETRAINED WEIGHTS TO RESUME TRAINING ####\")\n        cfg.MODEL.WEIGHTS = str(PRETRAINED_PATH)\n    else:\n        print(\"#### TRAINING MODEL FROM SCRATCH ####\")\n\n    cfg.SOLVER.AMP.ENABLED = True\n    cfg.SOLVER.IMS_PER_BATCH = 8\n    cfg.SOLVER.BASE_LR = 0.000001\n\n    cfg.SOLVER.WARMUP_ITERS = 0\n\n    # Maximum number of iterations\n    cfg.SOLVER.MAX_ITER = 5000\n\n    # cfg.SOLVER.STEPS = (500, 1000) # must be less than MAX_ITER\n\n    cfg.SOLVER.GAMMA = 0.00001\n\n    # Small value == Frequent save need a lot of storage.\n    cfg.SOLVER.CHECKPOINT_PERIOD = 2000\n    cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128\n    cfg.MODEL.ROI_HEADS.NUM_CLASSES = 4\n\n    # Create Output Directory\n    cfg.OUTPUT_DIR = str(OUTPUT_DIR)\n    print(\"creating cfg.OUTPUT_DIR -> \", cfg.OUTPUT_DIR)\n    OUTPUT_DIR.mkdir(exist_ok=True)","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:05:18.322379Z","iopub.status.busy":"2023-07-24T13:05:18.322021Z","iopub.status.idle":"2023-07-24T13:05:18.347928Z","shell.execute_reply":"2023-07-24T13:05:18.34688Z"},"papermill":{"duration":0.081609,"end_time":"2023-07-24T13:05:18.350206","exception":false,"start_time":"2023-07-24T13:05:18.268597","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6. Training","metadata":{"papermill":{"duration":0.054516,"end_time":"2023-07-24T13:05:18.456179","exception":false,"start_time":"2023-07-24T13:05:18.401663","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## 6.1 Training the model","metadata":{"papermill":{"duration":0.052583,"end_time":"2023-07-24T13:05:18.561354","exception":false,"start_time":"2023-07-24T13:05:18.508771","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:05:18.666216Z","iopub.status.busy":"2023-07-24T13:05:18.665855Z","iopub.status.idle":"2023-07-24T13:05:19.695234Z","shell.execute_reply":"2023-07-24T13:05:19.694036Z"},"papermill":{"duration":1.085343,"end_time":"2023-07-24T13:05:19.697969","exception":false,"start_time":"2023-07-24T13:05:18.612626","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_train:\n    trainer = DefaultTrainer(cfg) if not is_augment else AugTrainer(cfg)\n        \n    trainer.resume_or_load(resume=is_resume_training)\n\n    trainer.train()\n    \n    print(\"#### TRAINING COMPLETE ####\")\n    _ = trainer.model.train(False)  # turn off training\n    \n    FileLink(str(OUTPUT_MODEL))","metadata":{"execution":{"iopub.execute_input":"2023-07-24T13:05:19.80441Z","iopub.status.busy":"2023-07-24T13:05:19.804009Z","iopub.status.idle":"2023-07-24T23:57:08.58134Z","shell.execute_reply":"2023-07-24T23:57:08.580108Z"},"papermill":{"duration":39108.835172,"end_time":"2023-07-24T23:57:08.585047","exception":false,"start_time":"2023-07-24T13:05:19.749875","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 6.2 Visualizing Training Metrics","metadata":{"papermill":{"duration":0.059587,"end_time":"2023-07-24T23:57:08.706442","exception":false,"start_time":"2023-07-24T23:57:08.646855","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if is_train:\n    # Load metrics\n    metrics_df = pd.read_json(\n        OUTPUT_DIR/\"metrics.json\", orient=\"records\", lines=True\n    )\n    mdf = metrics_df.sort_values(\"iteration\")\n    print(mdf.head(10).T)\n\n    # Plot loss\n    fig, ax = plt.subplots()\n\n    mdf1 = mdf[~mdf[\"total_loss\"].isna()]\n    ax.plot(mdf1[\"iteration\"], mdf1[\"total_loss\"], c=\"C0\", label=\"train\")\n\n    if \"validation_loss\" in mdf.columns:\n        mdf2 = mdf[~mdf[\"validation_loss\"].isna()]\n        ax.plot(mdf2[\"iteration\"], mdf2[\"validation_loss\"],\n                c=\"C1\", label=\"validation\")\n\n    ax.legend()\n    ax.set_title(\"Loss curve\")\n    plt.show()\n\n    # Plot Accuracy\n    fig, ax = plt.subplots()\n\n    mdf1 = mdf[~mdf[\"fast_rcnn/cls_accuracy\"].isna()]\n    ax.plot(mdf1[\"iteration\"], mdf1[\"fast_rcnn/cls_accuracy\"],\n            c=\"C0\", label=\"train\")\n\n    ax.legend()\n    ax.set_title(\"Accuracy curve\")\n    plt.show()\n\n    # Plot Bounding Box regressor loss\n    fig, ax = plt.subplots()\n\n    mdf1 = mdf[~mdf[\"loss_box_reg\"].isna()]\n    ax.plot(mdf1[\"iteration\"], mdf1[\"loss_box_reg\"], c=\"C0\", label=\"train\")\n\n    ax.legend()\n    ax.set_title(\"loss_box_reg\")\n    plt.show()","metadata":{"execution":{"iopub.execute_input":"2023-07-24T23:57:08.833178Z","iopub.status.busy":"2023-07-24T23:57:08.832276Z","iopub.status.idle":"2023-07-24T23:57:09.99625Z","shell.execute_reply":"2023-07-24T23:57:09.99528Z"},"papermill":{"duration":1.23036,"end_time":"2023-07-24T23:57:09.998428","exception":false,"start_time":"2023-07-24T23:57:08.768068","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 7. Evaluation","metadata":{"papermill":{"duration":0.064578,"end_time":"2023-07-24T23:57:10.129967","exception":false,"start_time":"2023-07-24T23:57:10.065389","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Can evaluate trained model on validation dataset to obtain different metric scores. ","metadata":{"papermill":{"duration":0.063262,"end_time":"2023-07-24T23:57:10.256703","exception":false,"start_time":"2023-07-24T23:57:10.193441","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if is_evaluate:\n    print(\"### EVALUATING ON VALIDATION DATA ####\")\n    # trained model weights\n    cfg.MODEL.WEIGHTS = str(MODEL_PATH)\n    cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5   # set a custom testing threshold\n\n    cfg.SOLVER.IMS_PER_BATCH = 64\n\n    evaluator = COCOEvaluator(\n        DATA_REGISTER_VALID, cfg, False, output_dir=cfg.OUTPUT_DIR, use_fast_impl=True\n    )\n\n    val_loader = build_detection_test_loader(cfg, DATA_REGISTER_VALID)\n\n    results = inference_on_dataset(\n        trainer.model, val_loader, evaluator=evaluator\n    )","metadata":{"execution":{"iopub.execute_input":"2023-07-24T23:57:10.386628Z","iopub.status.busy":"2023-07-24T23:57:10.386236Z","iopub.status.idle":"2023-07-24T23:57:10.392658Z","shell.execute_reply":"2023-07-24T23:57:10.391608Z"},"papermill":{"duration":0.074464,"end_time":"2023-07-24T23:57:10.395069","exception":false,"start_time":"2023-07-24T23:57:10.320605","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 8. Inference","metadata":{"papermill":{"duration":0.062002,"end_time":"2023-07-24T23:57:10.519045","exception":false,"start_time":"2023-07-24T23:57:10.457043","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 8.1 Setting Up Inference Model","metadata":{"papermill":{"duration":0.062672,"end_time":"2023-07-24T23:57:10.644585","exception":false,"start_time":"2023-07-24T23:57:10.581913","status":"completed"},"tags":[]}},{"cell_type":"code","source":"inf_cfg = get_cfg()\n\nconfig_name = \"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"\n\ninf_cfg.merge_from_file(model_zoo.get_config_file(config_name))\ninf_cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128\ninf_cfg.MODEL.ROI_HEADS.NUM_CLASSES = 4\ninf_cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5\ninf_cfg.MODEL.DEVICE = \"cuda\"\n\ninf_cfg.DATALOADER.NUM_WORKERS = 2  # lower this if CUDA overflow occurs\ninf_cfg.MODEL.WEIGHTS = str(MODEL_PATH)","metadata":{"execution":{"iopub.execute_input":"2023-07-24T23:57:10.774508Z","iopub.status.busy":"2023-07-24T23:57:10.77393Z","iopub.status.idle":"2023-07-24T23:57:10.797916Z","shell.execute_reply":"2023-07-24T23:57:10.797042Z"},"papermill":{"duration":0.089628,"end_time":"2023-07-24T23:57:10.799837","exception":false,"start_time":"2023-07-24T23:57:10.710209","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH = 8  # lower this if CUDA overflow occurs\ntest_loader = build_detection_test_loader(inf_cfg, DATA_REGISTER_TEST, batch_size=BATCH)","metadata":{"execution":{"iopub.execute_input":"2023-07-24T23:57:10.941943Z","iopub.status.busy":"2023-07-24T23:57:10.94153Z","iopub.status.idle":"2023-07-24T23:57:12.245131Z","shell.execute_reply":"2023-07-24T23:57:12.244173Z"},"papermill":{"duration":1.373139,"end_time":"2023-07-24T23:57:12.247616","exception":false,"start_time":"2023-07-24T23:57:10.874477","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"How confident should the model be for you to accept the predicted mask?","metadata":{"papermill":{"duration":0.063139,"end_time":"2023-07-24T23:57:12.373146","exception":false,"start_time":"2023-07-24T23:57:12.310007","status":"completed"},"tags":[]}},{"cell_type":"code","source":"ACCEPTANCE_THRESHOLD = 0.6  # for all categories","metadata":{"execution":{"iopub.execute_input":"2023-07-24T23:57:12.504653Z","iopub.status.busy":"2023-07-24T23:57:12.503569Z","iopub.status.idle":"2023-07-24T23:57:12.508889Z","shell.execute_reply":"2023-07-24T23:57:12.507978Z"},"papermill":{"duration":0.073492,"end_time":"2023-07-24T23:57:12.511008","exception":false,"start_time":"2023-07-24T23:57:12.437516","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"#### MODEL: {inf_cfg.MODEL.WEIGHTS} FOR INFERENCE ####\")","metadata":{"execution":{"iopub.execute_input":"2023-07-24T23:57:12.638774Z","iopub.status.busy":"2023-07-24T23:57:12.638386Z","iopub.status.idle":"2023-07-24T23:57:12.645261Z","shell.execute_reply":"2023-07-24T23:57:12.643756Z"},"papermill":{"duration":0.073878,"end_time":"2023-07-24T23:57:12.647144","exception":false,"start_time":"2023-07-24T23:57:12.573266","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 8.2 Sample Inference","metadata":{"papermill":{"duration":0.064286,"end_time":"2023-07-24T23:57:12.775602","exception":false,"start_time":"2023-07-24T23:57:12.711316","status":"completed"},"tags":[]}},{"cell_type":"code","source":"predictor = DefaultPredictor(inf_cfg)","metadata":{"execution":{"iopub.execute_input":"2023-07-24T23:57:12.90989Z","iopub.status.busy":"2023-07-24T23:57:12.909517Z","iopub.status.idle":"2023-07-24T23:57:14.218653Z","shell.execute_reply":"2023-07-24T23:57:14.21762Z"},"papermill":{"duration":1.381531,"end_time":"2023-07-24T23:57:14.221541","exception":false,"start_time":"2023-07-24T23:57:12.84001","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(2, 2, figsize=(20, 20))\nindices = [ax[0][0], ax[1][0], ax[0][1], ax[1][1]]\n\n# Show some qualitative results by predicting on test set images\nNUM_TEST_SAMPLES = 4\nsamples = np.random.choice(dataset_dicts_test, NUM_TEST_SAMPLES)\n\nfor i, sample in enumerate(samples):\n    img = cv2.imread(sample[\"file_name\"])\n    outputs = predictor(img)\n    visualizer = Visualizer(img, metadata=metadata_dicts_test, scale=0.5,)\n    visualizer = visualizer.draw_instance_predictions(\n        outputs[\"instances\"].to(\"cpu\")\n    )\n    display_img = visualizer.get_image()[:, :, ::-1]\n    indices[i].grid(False)\n    indices[i].imshow(display_img)","metadata":{"execution":{"iopub.execute_input":"2023-07-24T23:57:14.349956Z","iopub.status.busy":"2023-07-24T23:57:14.349589Z","iopub.status.idle":"2023-07-24T23:57:19.4142Z","shell.execute_reply":"2023-07-24T23:57:19.413123Z"},"papermill":{"duration":5.141374,"end_time":"2023-07-24T23:57:19.425491","exception":false,"start_time":"2023-07-24T23:57:14.284117","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 8.3 Test Data Inference and Submission","metadata":{"papermill":{"duration":0.07703,"end_time":"2023-07-24T23:57:19.583921","exception":false,"start_time":"2023-07-24T23:57:19.506891","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 8.3.1 Building Inference Model","metadata":{"papermill":{"duration":0.078246,"end_time":"2023-07-24T23:57:19.740309","exception":false,"start_time":"2023-07-24T23:57:19.662063","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def rebuild_model():\n    model = build_model(inf_cfg)\n    _ = DetectionCheckpointer(model).load(inf_cfg.MODEL.WEIGHTS)\n    return model","metadata":{"execution":{"iopub.execute_input":"2023-07-24T23:57:19.915888Z","iopub.status.busy":"2023-07-24T23:57:19.915349Z","iopub.status.idle":"2023-07-24T23:57:19.920715Z","shell.execute_reply":"2023-07-24T23:57:19.919718Z"},"papermill":{"duration":0.098992,"end_time":"2023-07-24T23:57:19.922811","exception":false,"start_time":"2023-07-24T23:57:19.823819","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = rebuild_model()","metadata":{"execution":{"iopub.execute_input":"2023-07-24T23:57:20.097888Z","iopub.status.busy":"2023-07-24T23:57:20.097118Z","iopub.status.idle":"2023-07-24T23:57:21.323116Z","shell.execute_reply":"2023-07-24T23:57:21.322102Z"},"papermill":{"duration":1.318567,"end_time":"2023-07-24T23:57:21.325607","exception":false,"start_time":"2023-07-24T23:57:20.00704","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 8.3.2 CUDA Problems","metadata":{"papermill":{"duration":0.077808,"end_time":"2023-07-24T23:57:21.48032","exception":false,"start_time":"2023-07-24T23:57:21.402512","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!export LRU_CACHE_CAPACITY=1\n!export 'PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:512'","metadata":{"execution":{"iopub.execute_input":"2023-07-24T23:57:21.638466Z","iopub.status.busy":"2023-07-24T23:57:21.638034Z","iopub.status.idle":"2023-07-24T23:57:23.973908Z","shell.execute_reply":"2023-07-24T23:57:23.972456Z"},"papermill":{"duration":2.418267,"end_time":"2023-07-24T23:57:23.976668","exception":false,"start_time":"2023-07-24T23:57:21.558401","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Occassionally CUDA memory overflow occurs. Trying to save as much VRAM as we can. ","metadata":{"papermill":{"duration":0.078457,"end_time":"2023-07-24T23:57:24.132996","exception":false,"start_time":"2023-07-24T23:57:24.054539","status":"completed"},"tags":[]}},{"cell_type":"code","source":"vars_to_del = [\"trainer\", \"predictor\", \"outputs\"]\n\nfor v in vars_to_del:\n    if v in globals():\n        print(f\"Deleting {v}\")\n        del globals()[v]\n    elif v in locals():\n        print(f\"Deleting {v}\")\n        del locals()[v]","metadata":{"execution":{"iopub.execute_input":"2023-07-24T23:57:24.299176Z","iopub.status.busy":"2023-07-24T23:57:24.298157Z","iopub.status.idle":"2023-07-24T23:57:24.55242Z","shell.execute_reply":"2023-07-24T23:57:24.551489Z"},"papermill":{"duration":0.345385,"end_time":"2023-07-24T23:57:24.554411","exception":false,"start_time":"2023-07-24T23:57:24.209026","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 8.3.3 Inference Utils","metadata":{"papermill":{"duration":0.082944,"end_time":"2023-07-24T23:57:24.715908","exception":false,"start_time":"2023-07-24T23:57:24.632964","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def rle_encode(mask):\n    pixels = mask.T.flatten()\n    use_padding = False\n    if pixels[0] or pixels[-1]:\n        use_padding = True\n        pixel_padded = np.zeros([len(pixels) + 2], dtype=pixels.dtype)\n        pixel_padded[1:-1] = pixels\n        pixels = pixel_padded\n    rle = np.where(pixels[1:] != pixels[:-1])[0] + 2\n    if use_padding:\n        rle = rle - 1\n    rle[1::2] = rle[1::2] - rle[:-1:2]\n    return ' '.join(str(x) for x in rle)","metadata":{"execution":{"iopub.execute_input":"2023-07-24T23:57:24.880398Z","iopub.status.busy":"2023-07-24T23:57:24.879468Z","iopub.status.idle":"2023-07-24T23:57:24.887028Z","shell.execute_reply":"2023-07-24T23:57:24.886035Z"},"papermill":{"duration":0.092409,"end_time":"2023-07-24T23:57:24.889105","exception":false,"start_time":"2023-07-24T23:57:24.796696","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@retry_if_cuda_oom\ndef get_masks(prediction):\n    # get masks for each category\n    take = prediction.scores >= ACCEPTANCE_THRESHOLD\n    pred_masks = (prediction.pred_masks[take] != 0)\n    pred_classes = prediction.pred_classes[take]\n  \n    rles = []\n    for cat in range(len(thing_classes)):\n        pred_mask = pred_masks[pred_classes == cat]\n        \n        # pred_mask = retry_if_cuda_oom(torch.any)(pred_mask, dim=0)\n        pred_mask = torch.any(pred_mask, dim=0)\n        rles.append(rle_encode(pred_mask.short().to(\"cpu\").numpy()))\n\n    return rles","metadata":{"execution":{"iopub.execute_input":"2023-07-24T23:57:25.048297Z","iopub.status.busy":"2023-07-24T23:57:25.047504Z","iopub.status.idle":"2023-07-24T23:57:25.05491Z","shell.execute_reply":"2023-07-24T23:57:25.053949Z"},"papermill":{"duration":0.089129,"end_time":"2023-07-24T23:57:25.057078","exception":false,"start_time":"2023-07-24T23:57:24.967949","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_inference(data):\n    results = []\n    with torch.no_grad():\n        outputs = model(data)\n        if torch.cuda.is_available():\n            torch.cuda.synchronize()\n\n        for idx, output in enumerate(outputs):\n            output = output[\"instances\"]\n\n            rles = get_masks(output)\n\n            result = [\n                f\"{data[idx]['image_id']}_{cat},{rles[cat]}\\n\"\n                for cat in range(len(thing_classes))\n            ]\n\n            results.extend(result)\n\n        del outputs, output\n\n    return results","metadata":{"execution":{"iopub.execute_input":"2023-07-24T23:57:25.214263Z","iopub.status.busy":"2023-07-24T23:57:25.213895Z","iopub.status.idle":"2023-07-24T23:57:25.220595Z","shell.execute_reply":"2023-07-24T23:57:25.219692Z"},"papermill":{"duration":0.088504,"end_time":"2023-07-24T23:57:25.222641","exception":false,"start_time":"2023-07-24T23:57:25.134137","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 8.3.4 Running Inference on Test Data and Creating Submission File\nThis is super slow. ","metadata":{"papermill":{"duration":0.077176,"end_time":"2023-07-24T23:57:25.379053","exception":false,"start_time":"2023-07-24T23:57:25.301877","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(\"#### RUNNING INFERENCE ON TEST DATA ####\")","metadata":{"execution":{"iopub.execute_input":"2023-07-24T23:57:25.536205Z","iopub.status.busy":"2023-07-24T23:57:25.535834Z","iopub.status.idle":"2023-07-24T23:57:25.541156Z","shell.execute_reply":"2023-07-24T23:57:25.540218Z"},"papermill":{"duration":0.087382,"end_time":"2023-07-24T23:57:25.543696","exception":false,"start_time":"2023-07-24T23:57:25.456314","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()\ngc.collect()","metadata":{"execution":{"iopub.execute_input":"2023-07-24T23:57:25.702479Z","iopub.status.busy":"2023-07-24T23:57:25.701778Z","iopub.status.idle":"2023-07-24T23:57:27.776475Z","shell.execute_reply":"2023-07-24T23:57:27.775551Z"},"papermill":{"duration":2.15819,"end_time":"2023-07-24T23:57:27.778446","exception":false,"start_time":"2023-07-24T23:57:25.620256","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_inference:\n    model.eval()\n    submission_file = open(\"submission.csv\", \"w\")\n    submission_file.write(\"Id,Predicted\\n\")\n\n    results: list[str] = []\n    \n    for i, data in enumerate(tqdm(test_loader)):\n        res = run_inference(data)\n        results.extend(res)\n        \n        if i % (500 // BATCH) == 0:\n            print(f\"Inference on batch {i}/{len(test_loader)} done\")\n            submission_file.writelines(results)\n            results = []\n\n    submission_file.writelines(results)\n    submission_file.close()","metadata":{"execution":{"iopub.execute_input":"2023-07-24T23:57:27.938085Z","iopub.status.busy":"2023-07-24T23:57:27.937699Z","iopub.status.idle":"2023-07-25T00:41:40.469589Z","shell.execute_reply":"2023-07-25T00:41:40.468311Z"},"papermill":{"duration":2652.615537,"end_time":"2023-07-25T00:41:40.473448","exception":false,"start_time":"2023-07-24T23:57:27.857911","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if Path(\"submission.csv\").exists:\n    display(FileLink(\"submission.csv\"))","metadata":{"execution":{"iopub.execute_input":"2023-07-25T00:41:40.63894Z","iopub.status.busy":"2023-07-25T00:41:40.638517Z","iopub.status.idle":"2023-07-25T00:41:40.645864Z","shell.execute_reply":"2023-07-25T00:41:40.644993Z"},"papermill":{"duration":0.093131,"end_time":"2023-07-25T00:41:40.647743","exception":false,"start_time":"2023-07-25T00:41:40.554612","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r detectron2/","metadata":{"execution":{"iopub.execute_input":"2023-07-25T00:41:40.812166Z","iopub.status.busy":"2023-07-25T00:41:40.811689Z","iopub.status.idle":"2023-07-25T00:41:41.977995Z","shell.execute_reply":"2023-07-25T00:41:41.976647Z"},"papermill":{"duration":1.25355,"end_time":"2023-07-25T00:41:41.980428","exception":false,"start_time":"2023-07-25T00:41:40.726878","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.103004,"end_time":"2023-07-25T00:41:42.182505","exception":false,"start_time":"2023-07-25T00:41:42.079501","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}