{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport json\nimport cv2\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\nimport shutil\nimport random\nimport torch\nimport pandas as pd\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2023-08-01T20:00:05.591117Z","iopub.execute_input":"2023-08-01T20:00:05.591431Z","iopub.status.idle":"2023-08-01T20:00:11.586898Z","shell.execute_reply.started":"2023-08-01T20:00:05.591405Z","shell.execute_reply":"2023-08-01T20:00:11.585753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"COCO_PATH = Path(\"/kaggle/input/dlsprint2/badlad/labels/coco_format/train/badlad-train-coco.json\")\nwith open(COCO_PATH, \"r\") as f:\n    coco_data = json.load(f)","metadata":{"execution":{"iopub.status.busy":"2023-08-01T20:00:11.592567Z","iopub.execute_input":"2023-08-01T20:00:11.593192Z","iopub.status.idle":"2023-08-01T20:00:18.931088Z","shell.execute_reply.started":"2023-08-01T20:00:11.593157Z","shell.execute_reply":"2023-08-01T20:00:18.929987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"td=pd.DataFrame(coco_data['images'])\nprint(len(td))\ntd=pd.DataFrame(coco_data['annotations'])\nprint(len(td))","metadata":{"execution":{"iopub.status.busy":"2023-08-01T20:00:18.932684Z","iopub.execute_input":"2023-08-01T20:00:18.933135Z","iopub.status.idle":"2023-08-01T20:00:20.482114Z","shell.execute_reply.started":"2023-08-01T20:00:18.933094Z","shell.execute_reply":"2023-08-01T20:00:20.480993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#image Aspect Ratio\ntrain_images = pd.DataFrame( coco_data[\"images\"])\n\ntrain_images['aspect_ratio']=train_images['width']/train_images['height']\n\ntrain_images.rename(columns={\"id\":\"image_id\"}, inplace=True)\n\ntrain_images_aspect_ratio_count = train_images.groupby('aspect_ratio', as_index=False)['image_id'].nunique()\ntrain_images_aspect_ratio_count.rename(columns={'image_id':'image_count'}, inplace=True)\ntrain_images_aspect_ratio_count.sort_values(by='image_count', ascending=False,inplace=True)\n\nplt.figure(figsize=(15, 6))\nplt.title('Area vs Image Count Line Plot')\nsns.lineplot(x=train_images_aspect_ratio_count['aspect_ratio'], y = train_images_aspect_ratio_count['image_count'])","metadata":{"execution":{"iopub.status.busy":"2023-08-01T20:00:20.484397Z","iopub.execute_input":"2023-08-01T20:00:20.485336Z","iopub.status.idle":"2023-08-01T20:00:21.039423Z","shell.execute_reply.started":"2023-08-01T20:00:20.4853Z","shell.execute_reply":"2023-08-01T20:00:21.038336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#bbox Aspect Ratio\ntrain_annotations = pd.DataFrame(coco_data['annotations'])\n\ntrain_annotations.rename(columns={\"id\":\"annotation_id\"}, inplace=True)\nbbox_aspect_ratio=[]\nfor idx in train_annotations.index:\n    bbox_aspect_ratio.append(train_annotations['bbox'][idx][3]/train_annotations['bbox'][idx][2])\ntrain_annotations['bbox_aspect_ratio']=bbox_aspect_ratio\n\n\nbbox_aspect_ratio_count = train_annotations.groupby('bbox_aspect_ratio', as_index=False)['annotation_id'].count()\nbbox_aspect_ratio_count.rename(columns={'annotation_id':'annotation_count'}, inplace=True)\n\nplt.figure(figsize=(15, 6))\nplt.title('bbox aspect ratio vs Annotation Count Line Plot')\nsns.lineplot(x=bbox_aspect_ratio_count['bbox_aspect_ratio'], y = bbox_aspect_ratio_count['annotation_count'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nlow_aspect_ratio_threshold = 0.1\nhigh_aspect_ratio_threshold = 2.0\nhigh_bbox_aspect_ratio_threshold= 10.0","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:50:39.906258Z","iopub.execute_input":"2023-07-26T21:50:39.906925Z","iopub.status.idle":"2023-07-26T21:50:39.941349Z","shell.execute_reply.started":"2023-07-26T21:50:39.906892Z","shell.execute_reply":"2023-07-26T21:50:39.939895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_to_keep = []\nimages_to_remove = []\nanno_to_keep = []\nanno_to_remove = []","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:50:46.721956Z","iopub.execute_input":"2023-07-26T21:50:46.722406Z","iopub.status.idle":"2023-07-26T21:50:46.727691Z","shell.execute_reply.started":"2023-07-26T21:50:46.722339Z","shell.execute_reply":"2023-07-26T21:50:46.726723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for image_info in coco_data[\"images\"]:\n    image_id = image_info[\"id\"]\n    image_aspect_count = image_info[\"width\"] / image_info[\"height\"]\n    \n    image_aspect_count_tensor = torch.tensor(image_aspect_count, dtype=torch.float32, device=device)\n\n    if (image_aspect_count_tensor > high_aspect_ratio_threshold) or (image_aspect_count_tensor < low_aspect_ratio_threshold):\n        images_to_remove.append(image_id)\n    else:\n        images_to_keep.append(image_id)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:50:48.937966Z","iopub.execute_input":"2023-07-26T21:50:48.938326Z","iopub.status.idle":"2023-07-26T21:50:53.027406Z","shell.execute_reply.started":"2023-07-26T21:50:48.938296Z","shell.execute_reply":"2023-07-26T21:50:53.026426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for anno in coco_data[\"annotations\"]:\n    anno_id = anno['id']\n    bbox_aspect_ratio = anno['bbox'][3]/ anno['bbox'][2]\n    bbox_aspect_ratio_tensor = torch.tensor( bbox_aspect_ratio, dtype=torch.float32, device=device)\n    if bbox_aspect_ratio_tensor > high_bbox_aspect_ratio_threshold:\n        anno_to_remove.append(anno_id)\n    else:\n        anno_to_keep.append(anno_id)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:50:58.041244Z","iopub.execute_input":"2023-07-26T21:50:58.041923Z","iopub.status.idle":"2023-07-26T21:51:19.527551Z","shell.execute_reply.started":"2023-07-26T21:50:58.041891Z","shell.execute_reply":"2023-07-26T21:51:19.526504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(len(anno_to_remove))\n# print(anno_to_remove[0:100])","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:51:19.530085Z","iopub.execute_input":"2023-07-26T21:51:19.530529Z","iopub.status.idle":"2023-07-26T21:51:19.536681Z","shell.execute_reply.started":"2023-07-26T21:51:19.530497Z","shell.execute_reply":"2023-07-26T21:51:19.535689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_category_counts = {}\nfor image_id in images_to_keep:\n        image_category_counts[image_id] = [0, 0, 0, 0]","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:51:19.538157Z","iopub.execute_input":"2023-07-26T21:51:19.538615Z","iopub.status.idle":"2023-07-26T21:51:19.588225Z","shell.execute_reply.started":"2023-07-26T21:51:19.538585Z","shell.execute_reply":"2023-07-26T21:51:19.58691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for annotation in coco_data['annotations']:\n    image_id = annotation['image_id']\n    category_id = annotation['category_id']\n    if image_id in images_to_keep:\n        image_category_counts[image_id][category_id] += 1","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:51:19.59299Z","iopub.execute_input":"2023-07-26T21:51:19.593715Z","iopub.status.idle":"2023-07-26T21:51:56.287233Z","shell.execute_reply.started":"2023-07-26T21:51:19.593688Z","shell.execute_reply":"2023-07-26T21:51:56.286275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_image_category_counts = pd.DataFrame.from_dict(image_category_counts, orient='index', columns=['Paragraph', 'Text-Box', 'Image', 'Table'])","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:51:56.288729Z","iopub.execute_input":"2023-07-26T21:51:56.289069Z","iopub.status.idle":"2023-07-26T21:51:56.336251Z","shell.execute_reply.started":"2023-07-26T21:51:56.289038Z","shell.execute_reply":"2023-07-26T21:51:56.335413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(df_image_category_counts)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:51:56.337463Z","iopub.execute_input":"2023-07-26T21:51:56.338029Z","iopub.status.idle":"2023-07-26T21:51:56.349038Z","shell.execute_reply.started":"2023-07-26T21:51:56.337998Z","shell.execute_reply":"2023-07-26T21:51:56.348015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_images = df_image_category_counts[\n    (df_image_category_counts['Paragraph'] != 0) & (df_image_category_counts['Text-Box'] !=0) & (df_image_category_counts['Image'] ==0) & (df_image_category_counts['Table'] ==0)\n]","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:51:56.350748Z","iopub.execute_input":"2023-07-26T21:51:56.351448Z","iopub.status.idle":"2023-07-26T21:51:56.359939Z","shell.execute_reply.started":"2023-07-26T21:51:56.351407Z","shell.execute_reply":"2023-07-26T21:51:56.3591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(filtered_images)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:51:56.361167Z","iopub.execute_input":"2023-07-26T21:51:56.361527Z","iopub.status.idle":"2023-07-26T21:51:56.372488Z","shell.execute_reply.started":"2023-07-26T21:51:56.361497Z","shell.execute_reply":"2023-07-26T21:51:56.371447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_images_final = filtered_images[\n    ((filtered_images['Paragraph'] > 80) | (filtered_images['Text-Box'] > 150))\n]","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:51:56.375818Z","iopub.execute_input":"2023-07-26T21:51:56.37662Z","iopub.status.idle":"2023-07-26T21:51:56.381988Z","shell.execute_reply.started":"2023-07-26T21:51:56.37659Z","shell.execute_reply":"2023-07-26T21:51:56.381125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(filtered_images_final)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:51:56.383006Z","iopub.execute_input":"2023-07-26T21:51:56.383714Z","iopub.status.idle":"2023-07-26T21:51:56.395954Z","shell.execute_reply.started":"2023-07-26T21:51:56.383683Z","shell.execute_reply":"2023-07-26T21:51:56.395066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(len(filtered_images))\n# print(len(filtered_images_final))\n# print(list(filtered_images_final.index))","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:51:56.397075Z","iopub.execute_input":"2023-07-26T21:51:56.398117Z","iopub.status.idle":"2023-07-26T21:51:56.407654Z","shell.execute_reply.started":"2023-07-26T21:51:56.398087Z","shell.execute_reply":"2023-07-26T21:51:56.406655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(len(images_to_remove))\n# print(images_to_remove[0:100])","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:51:56.408626Z","iopub.execute_input":"2023-07-26T21:51:56.40887Z","iopub.status.idle":"2023-07-26T21:51:56.417941Z","shell.execute_reply.started":"2023-07-26T21:51:56.408849Z","shell.execute_reply":"2023-07-26T21:51:56.417082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_ids_to_delete = list(filtered_images_final.index)\nimage_ids_to_delete.extend(images_to_remove)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:51:56.418955Z","iopub.execute_input":"2023-07-26T21:51:56.419311Z","iopub.status.idle":"2023-07-26T21:51:56.426965Z","shell.execute_reply.started":"2023-07-26T21:51:56.419279Z","shell.execute_reply":"2023-07-26T21:51:56.426021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(len(image_ids_to_delete))\n# print(image_ids_to_delete[0:100])","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:51:56.427889Z","iopub.execute_input":"2023-07-26T21:51:56.429316Z","iopub.status.idle":"2023-07-26T21:51:56.437588Z","shell.execute_reply.started":"2023-07-26T21:51:56.429292Z","shell.execute_reply":"2023-07-26T21:51:56.436539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_ids_to_delete=set(image_ids_to_delete)\nimage_ids_to_delete=list(image_ids_to_delete)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:51:56.439652Z","iopub.execute_input":"2023-07-26T21:51:56.439958Z","iopub.status.idle":"2023-07-26T21:51:56.447085Z","shell.execute_reply.started":"2023-07-26T21:51:56.439935Z","shell.execute_reply":"2023-07-26T21:51:56.445963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_ids_to_delete=sorted(image_ids_to_delete)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:51:56.448716Z","iopub.execute_input":"2023-07-26T21:51:56.449078Z","iopub.status.idle":"2023-07-26T21:51:56.456453Z","shell.execute_reply.started":"2023-07-26T21:51:56.449049Z","shell.execute_reply":"2023-07-26T21:51:56.455742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(len(image_ids_to_delete))\n# print(image_ids_to_delete[0:100])","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:51:56.457814Z","iopub.execute_input":"2023-07-26T21:51:56.458509Z","iopub.status.idle":"2023-07-26T21:51:56.466232Z","shell.execute_reply.started":"2023-07-26T21:51:56.45846Z","shell.execute_reply":"2023-07-26T21:51:56.465338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"coco_data['images'] = [image_info for image_info in coco_data['images'] if image_info['id'] not in image_ids_to_delete]\ncoco_data['annotations'] = [annotation for annotation in coco_data['annotations'] if (annotation['image_id'] not in image_ids_to_delete)]","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:51:56.467596Z","iopub.execute_input":"2023-07-26T21:51:56.46821Z","iopub.status.idle":"2023-07-26T21:52:09.185306Z","shell.execute_reply.started":"2023-07-26T21:51:56.46816Z","shell.execute_reply":"2023-07-26T21:52:09.184183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"coco_data['annotations'] = [annotation for annotation in coco_data['annotations'] if (annotation['id'] not in anno_to_remove)]","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:52:47.922418Z","iopub.execute_input":"2023-07-26T21:52:47.922771Z","iopub.status.idle":"2023-07-26T21:52:56.823847Z","shell.execute_reply.started":"2023-07-26T21:52:47.922745Z","shell.execute_reply":"2023-07-26T21:52:56.822823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"td=pd.DataFrame(coco_data['annotations'])\nprint(len(td))\ntd=pd.DataFrame(coco_data['images'])\nprint(len(td))","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:53:24.866276Z","iopub.execute_input":"2023-07-26T21:53:24.866664Z","iopub.status.idle":"2023-07-26T21:53:26.249702Z","shell.execute_reply.started":"2023-07-26T21:53:24.866636Z","shell.execute_reply":"2023-07-26T21:53:26.248532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_json_file_path = 'filtered_badlad-train-coco.json'\nwith open(filtered_json_file_path, 'w') as filtered_json_file:\n    json.dump(coco_data, filtered_json_file)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:54:03.767197Z","iopub.execute_input":"2023-07-26T21:54:03.767559Z","iopub.status.idle":"2023-07-26T21:54:21.090494Z","shell.execute_reply.started":"2023-07-26T21:54:03.767529Z","shell.execute_reply":"2023-07-26T21:54:21.089436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kaggle_working_directory = '/kaggle/working/'\nshutil.move(filtered_json_file_path, Path(kaggle_working_directory) / filtered_json_file_path)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:56:13.306459Z","iopub.execute_input":"2023-07-26T21:56:13.307413Z","iopub.status.idle":"2023-07-26T21:56:13.314457Z","shell.execute_reply.started":"2023-07-26T21:56:13.307352Z","shell.execute_reply":"2023-07-26T21:56:13.313406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Preprocessing Done\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # **Starter Notebook**","metadata":{}},{"cell_type":"code","source":"%%capture\nimport sys, os, distutils.core\n!git clone 'https://github.com/facebookresearch/detectron2'\ndist = distutils.core.run_setup(\"./detectron2/setup.py\")\n!python -m pip install {' '.join([f\"'{x}'\" for x in dist.install_requires])}\nsys.path.insert(0, os.path.abspath('./detectron2'))","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:56:22.542637Z","iopub.execute_input":"2023-07-26T21:56:22.542995Z","iopub.status.idle":"2023-07-26T21:57:11.120083Z","shell.execute_reply.started":"2023-07-26T21:56:22.542967Z","shell.execute_reply":"2023-07-26T21:57:11.118741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import datetime\n\n# if False, model is set to `PRETRAINED_PATH` model\nis_train = True\n\n# if True, evaluate on validation dataset\nis_evaluate = False\n\n# if True, run inference on test dataset\nis_inference = True\n\n# if True and `is_train` == True, `PRETRAINED_PATH` model is trained further\nis_resume_training = False\n\n# Perform augmentation\nis_augment = False\n\nSEED = int(datetime.now().timestamp())","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:57:11.1229Z","iopub.execute_input":"2023-07-26T21:57:11.123742Z","iopub.status.idle":"2023-07-26T21:57:11.134079Z","shell.execute_reply.started":"2023-07-26T21:57:11.123703Z","shell.execute_reply":"2023-07-26T21:57:11.131842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\n\nTRAIN_IMG_DIR = Path(\"/kaggle/input/dlsprint2/badlad/images/train\")\n\nTRAIN_COCO_PATH = Path(\"/kaggle/working/filtered_badlad-train-coco.json\")\n\nTEST_IMG_DIR = Path(\"/kaggle/input/dlsprint2/badlad/images/test\")\n\nTEST_METADATA_PATH = Path(\"/kaggle/input/dlsprint2/badlad/badlad-test-metadata.json\")\n\n# Training output directory\nOUTPUT_DIR = Path(\"./output\")\nOUTPUT_MODEL = OUTPUT_DIR/\"model_final.pth\"\n\n# Path to your pretrained model weights\nPRETRAINED_PATH = Path(\"\")","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:57:11.13522Z","iopub.execute_input":"2023-07-26T21:57:11.135856Z","iopub.status.idle":"2023-07-26T21:57:11.152891Z","shell.execute_reply.started":"2023-07-26T21:57:11.135571Z","shell.execute_reply":"2023-07-26T21:57:11.151899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model path based on Decisions\nMODEL_PATH = OUTPUT_MODEL if is_train else PRETRAINED_PATH","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:57:11.155566Z","iopub.execute_input":"2023-07-26T21:57:11.156245Z","iopub.status.idle":"2023-07-26T21:57:11.163822Z","shell.execute_reply.started":"2023-07-26T21:57:11.156213Z","shell.execute_reply":"2023-07-26T21:57:11.162913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# detectron2\nfrom detectron2.utils.memory import retry_if_cuda_oom\nfrom detectron2.utils.logger import setup_logger\nfrom detectron2.checkpoint import DetectionCheckpointer\nfrom detectron2.modeling import build_model\nfrom detectron2.evaluation import COCOEvaluator, inference_on_dataset\nimport detectron2.data.transforms as T\nfrom detectron2.data import detection_utils as utils\nfrom detectron2.data import DatasetCatalog, MetadataCatalog, build_detection_test_loader, build_detection_train_loader, DatasetMapper\nfrom detectron2.utils.visualizer import Visualizer\nfrom detectron2.structures import BoxMode\nfrom detectron2.engine import DefaultPredictor, DefaultTrainer\nfrom detectron2.config import get_cfg\nfrom detectron2 import model_zoo\n\nimport pandas as pd\nimport numpy as np\nfrom tqdm.notebook import tqdm  # progress bar\nimport matplotlib.pyplot as plt\nimport json\nimport cv2\nimport copy\nfrom typing import Optional\n\nfrom IPython.display import FileLink\n\n# torch\nimport torch\n\nimport gc\n\nimport warnings\n# Ignore \"future\" warnings and Data-Frame-Slicing warnings.\nwarnings.filterwarnings('ignore')\n\nsetup_logger()","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:57:11.167022Z","iopub.execute_input":"2023-07-26T21:57:11.167276Z","iopub.status.idle":"2023-07-26T21:57:11.904259Z","shell.execute_reply.started":"2023-07-26T21:57:11.167255Z","shell.execute_reply":"2023-07-26T21:57:11.903307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with TRAIN_COCO_PATH.open() as f:\n    train_dict = json.load(f)\n\nwith TEST_METADATA_PATH.open() as f:\n    test_dict = json.load(f)\n\nprint(\"#### LABELS AND METADATA LOADED ####\")","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:57:11.905775Z","iopub.execute_input":"2023-07-26T21:57:11.906132Z","iopub.status.idle":"2023-07-26T21:57:16.001275Z","shell.execute_reply.started":"2023-07-26T21:57:11.906098Z","shell.execute_reply":"2023-07-26T21:57:16.000187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def organize_coco_data(data_dict: dict) -> tuple[list[str], list[dict], list[dict]]:\n    thing_classes: list[str] = []\n\n    # Map Category Names to IDs\n    for cat in data_dict['categories']:\n        thing_classes.append(cat['name'])\n\n    # Images\n    images_metadata: list[dict] = data_dict['images']\n\n    # Convert COCO annotations to detectron2 annotations format\n    data_annotations = []\n    for ann in data_dict['annotations']:\n        # coco format -> detectron2 format\n        annot_obj = {\n            # Annotation ID\n            \"id\": ann['id'],\n\n            # Segmentation Polygon (x, y) coords\n            \"gt_masks\": ann['segmentation'],\n\n            # Image ID for this annotation (Which image does this annotation belong to?)\n            \"image_id\": ann['image_id'],\n\n            # Category Label (0: paragraph, 1: text box, 2: image, 3: table)\n            \"category_id\": ann['category_id'],\n\n            \"x_min\": ann['bbox'][0],  # left\n            \"y_min\": ann['bbox'][1],  # top\n            \"x_max\": ann['bbox'][0] + ann['bbox'][2],  # left+width\n            \"y_max\": ann['bbox'][1] + ann['bbox'][3]  # top+height\n        }\n        data_annotations.append(annot_obj)\n\n    return thing_classes, images_metadata, data_annotations","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:57:21.393383Z","iopub.execute_input":"2023-07-26T21:57:21.393744Z","iopub.status.idle":"2023-07-26T21:57:21.402283Z","shell.execute_reply.started":"2023-07-26T21:57:21.393716Z","shell.execute_reply":"2023-07-26T21:57:21.401073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"thing_classes, images_metadata, data_annotations = organize_coco_data(\n    train_dict\n)\n\nthing_classes_test, images_metadata_test, _ = organize_coco_data(\n    test_dict\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:57:24.887269Z","iopub.execute_input":"2023-07-26T21:57:24.887976Z","iopub.status.idle":"2023-07-26T21:57:25.514824Z","shell.execute_reply.started":"2023-07-26T21:57:24.887942Z","shell.execute_reply":"2023-07-26T21:57:25.513803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(thing_classes)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:57:28.314935Z","iopub.execute_input":"2023-07-26T21:57:28.315293Z","iopub.status.idle":"2023-07-26T21:57:28.320455Z","shell.execute_reply.started":"2023-07-26T21:57:28.315264Z","shell.execute_reply":"2023-07-26T21:57:28.319074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata = pd.DataFrame(images_metadata)\ntrain_metadata = train_metadata[['id', 'file_name', 'width', 'height']]\ntrain_metadata = train_metadata.rename(columns={\"id\": \"image_id\"})\nprint(\"train_metadata size=\", len(train_metadata))\ntrain_metadata.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:57:33.480687Z","iopub.execute_input":"2023-07-26T21:57:33.481043Z","iopub.status.idle":"2023-07-26T21:57:33.551933Z","shell.execute_reply.started":"2023-07-26T21:57:33.481016Z","shell.execute_reply":"2023-07-26T21:57:33.550795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_annot_df = pd.DataFrame(data_annotations)\nprint(\"train_annot_df size=\", len(train_annot_df))\ntrain_annot_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:57:56.29606Z","iopub.execute_input":"2023-07-26T21:57:56.296441Z","iopub.status.idle":"2023-07-26T21:57:57.514441Z","shell.execute_reply.started":"2023-07-26T21:57:56.29641Z","shell.execute_reply":"2023-07-26T21:57:57.513414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_metadata = pd.DataFrame(images_metadata_test)\ntest_metadata = test_metadata[['id', 'file_name', 'width', 'height']]\ntest_metadata = test_metadata.rename(columns={\"id\": \"image_id\"})\nprint(\"test_metadata size=\", len(test_metadata))\ntest_metadata.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:58:12.988853Z","iopub.execute_input":"2023-07-26T21:58:12.989266Z","iopub.status.idle":"2023-07-26T21:58:13.045828Z","shell.execute_reply.started":"2023-07-26T21:58:12.989235Z","shell.execute_reply":"2023-07-26T21:58:13.044941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_SPLIT = 0.85","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:58:33.258285Z","iopub.execute_input":"2023-07-26T21:58:33.258707Z","iopub.status.idle":"2023-07-26T21:58:33.263792Z","shell.execute_reply.started":"2023-07-26T21:58:33.258674Z","shell.execute_reply":"2023-07-26T21:58:33.26269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_dataset = len(train_metadata)\nn_train = int(n_dataset * TRAIN_SPLIT)\nprint(\"n_dataset\", n_dataset, \"n_train\", n_train, \"n_val\", n_dataset-n_train)\n\nnp.random.seed(SEED)\n\ninds = np.random.permutation(n_dataset)\ntrain_inds, valid_inds = inds[:n_train], inds[n_train:]","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:58:40.443481Z","iopub.execute_input":"2023-07-26T21:58:40.443839Z","iopub.status.idle":"2023-07-26T21:58:40.451298Z","shell.execute_reply.started":"2023-07-26T21:58:40.44381Z","shell.execute_reply":"2023-07-26T21:58:40.450159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_coco_to_detectron2_format(\n    imgdir: Path,\n    metadata_df: pd.DataFrame,\n    annot_df: Optional[pd.DataFrame] = None,\n    target_indices: Optional[np.ndarray] = None,\n):\n\n    dataset_dicts = []\n    for _, train_meta_row in tqdm(metadata_df.iterrows(), total=len(metadata_df)):\n        # Iterate over each image\n        image_id, filename, width, height = train_meta_row.values\n\n        annotations = []\n\n        # If train/validation data, then there will be annotations\n        if annot_df is not None:\n            for _, ann in annot_df.query(\"image_id == @image_id\").iterrows():\n                # Get annotations of current iteration's image\n                class_id = ann[\"category_id\"]\n                gt_masks = ann[\"gt_masks\"]\n                bbox_resized = [\n                    float(ann[\"x_min\"]),\n                    float(ann[\"y_min\"]),\n                    float(ann[\"x_max\"]),\n                    float(ann[\"y_max\"]),\n                ]\n\n                annotation = {\n                    \"bbox\": bbox_resized,\n                    \"bbox_mode\": BoxMode.XYXY_ABS,\n                    \"segmentation\": gt_masks,\n                    \"category_id\": class_id,\n                }\n\n                annotations.append(annotation)\n\n        # coco format -> detectron2 format dict\n        record = {\n            \"file_name\": str(imgdir/filename),\n            \"image_id\": image_id,\n            \"width\": width,\n            \"height\": height,\n            \"annotations\": annotations\n        }\n\n        dataset_dicts.append(record)\n\n    if target_indices is not None:\n        dataset_dicts = [dataset_dicts[i] for i in target_indices]\n\n    return dataset_dicts","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:59:19.525127Z","iopub.execute_input":"2023-07-26T21:59:19.525535Z","iopub.status.idle":"2023-07-26T21:59:19.537849Z","shell.execute_reply.started":"2023-07-26T21:59:19.525504Z","shell.execute_reply":"2023-07-26T21:59:19.536767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_REGISTER_TRAINING = \"badlad_train\"\nDATA_REGISTER_VALID    = \"badlad_valid\"\nDATA_REGISTER_TEST     = \"badlad_test\"","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:59:42.59261Z","iopub.execute_input":"2023-07-26T21:59:42.592979Z","iopub.status.idle":"2023-07-26T21:59:42.598307Z","shell.execute_reply.started":"2023-07-26T21:59:42.592948Z","shell.execute_reply":"2023-07-26T21:59:42.597163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Register Training data\nif is_train:\n    DatasetCatalog.register(\n        DATA_REGISTER_TRAINING,\n        lambda: convert_coco_to_detectron2_format(\n            TRAIN_IMG_DIR,\n            train_metadata,\n            train_annot_df,\n            target_indices=train_inds,\n        ),\n    )\n\n    # Set Training data categories\n    MetadataCatalog.get(DATA_REGISTER_TRAINING).set(thing_classes=thing_classes)\n\n    dataset_dicts_train = DatasetCatalog.get(DATA_REGISTER_TRAINING)\n    metadata_dicts_train = MetadataCatalog.get(DATA_REGISTER_TRAINING)\n\n    print(\"dicts training size=\", len(dataset_dicts_train))\n    print(\"################\")","metadata":{"execution":{"iopub.status.busy":"2023-07-26T21:59:53.597848Z","iopub.execute_input":"2023-07-26T21:59:53.598215Z","iopub.status.idle":"2023-07-26T22:01:15.482685Z","shell.execute_reply.started":"2023-07-26T21:59:53.598186Z","shell.execute_reply":"2023-07-26T22:01:15.48165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Register Validation data\nif is_train or is_evaluate:\n    DatasetCatalog.register(\n        DATA_REGISTER_VALID,\n        lambda: convert_coco_to_detectron2_format(\n            TRAIN_IMG_DIR,\n            train_metadata,\n            train_annot_df,\n            target_indices=valid_inds,\n        ),\n    )\n\n    # Set Validation data categories\n    MetadataCatalog.get(DATA_REGISTER_VALID).set(thing_classes=thing_classes)\n\n    dataset_dicts_valid = DatasetCatalog.get(DATA_REGISTER_VALID)\n    metadata_dicts_valid = MetadataCatalog.get(DATA_REGISTER_VALID)\n\n    print(\"dicts valid size=\", len(dataset_dicts_valid))\n    print(\"################\")","metadata":{"execution":{"iopub.status.busy":"2023-07-26T22:01:15.484792Z","iopub.execute_input":"2023-07-26T22:01:15.485563Z","iopub.status.idle":"2023-07-26T22:02:36.966812Z","shell.execute_reply.started":"2023-07-26T22:01:15.485528Z","shell.execute_reply":"2023-07-26T22:02:36.96347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Register Test Inference data\nDatasetCatalog.register(\n    DATA_REGISTER_TEST,\n    lambda: convert_coco_to_detectron2_format(\n        TEST_IMG_DIR,\n        test_metadata,\n    )\n)\n\n# Set Test data categories\nMetadataCatalog.get(DATA_REGISTER_TEST).set(\n    thing_classes=thing_classes_test\n)\n\ndataset_dicts_test = DatasetCatalog.get(DATA_REGISTER_TEST)\nmetadata_dicts_test = MetadataCatalog.get(DATA_REGISTER_TEST)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T22:02:36.968916Z","iopub.execute_input":"2023-07-26T22:02:36.969696Z","iopub.status.idle":"2023-07-26T22:02:38.152825Z","shell.execute_reply.started":"2023-07-26T22:02:36.969654Z","shell.execute_reply":"2023-07-26T22:02:38.151877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"#### DATA REGISTERED ####\")","metadata":{"execution":{"iopub.status.busy":"2023-07-26T22:02:38.15656Z","iopub.execute_input":"2023-07-26T22:02:38.159621Z","iopub.status.idle":"2023-07-26T22:02:38.167724Z","shell.execute_reply.started":"2023-07-26T22:02:38.159586Z","shell.execute_reply":"2023-07-26T22:02:38.166745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def custom_mapper(dataset_dict):\n    dataset_dict = copy.deepcopy(dataset_dict)\n    image = utils.read_image(dataset_dict[\"file_name\"], format=\"BGR\")\n\n    transform_list = [  T.ColorJitter(brightness= 0.8, contrast=0.4, saturation=0.2, hue=0.1),\n#                         T.RandomRotation(angle=[-5, 5]),\n#                         T.RandomScale(scale=(0.2, 1.2)),\n                        T.RandomFlip(prob=0.5, horizontal=False, vertical=True),\n                        T.RandomFlip(prob=0.5, horizontal=True, vertical=False),\n#                         T.RandomBrightness(0.5, 1.2),\n#                         T.RandomContrast(0.5, 1.2),\n#                         T.RandomSaturation(0.5, 1.2),\n#                         T.RandomHue(0.1)\n                    ]    \n                    \n    image, transforms = T.apply_transform_gens(transform_list, image)\n\n    dataset_dict[\"image\"] = torch.as_tensor(\n        image.transpose(2, 0, 1).astype(\"float32\"))\n\n    annos = [\n        utils.transform_instance_annotations(obj, transforms, image.shape[:2])\n        for obj in dataset_dict.pop(\"annotations\")\n        if obj.get(\"iscrowd\", 0) == 0\n    ]\n    instances = utils.annotations_to_instances(annos, image.shape[:2])\n\n    dataset_dict[\"instances\"] = utils.filter_empty_instances(instances)\n\n    return dataset_dict","metadata":{"execution":{"iopub.status.busy":"2023-07-25T19:12:54.228597Z","iopub.execute_input":"2023-07-25T19:12:54.229664Z","iopub.status.idle":"2023-07-25T19:12:54.240892Z","shell.execute_reply.started":"2023-07-25T19:12:54.229628Z","shell.execute_reply":"2023-07-25T19:12:54.23967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AugTrainer(DefaultTrainer):\n    @classmethod\n    def build_train_loader(cls, cfg):\n        return build_detection_train_loader(cfg, mapper=custom_mapper)","metadata":{"execution":{"iopub.status.busy":"2023-07-25T19:12:59.339422Z","iopub.execute_input":"2023-07-25T19:12:59.339823Z","iopub.status.idle":"2023-07-25T19:12:59.345541Z","shell.execute_reply.started":"2023-07-25T19:12:59.339791Z","shell.execute_reply":"2023-07-25T19:12:59.344578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_train:\n    cfg = get_cfg()\n\n    # config_name = \"COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml\"\n    config_name = \"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"\n\n    cfg.merge_from_file(model_zoo.get_config_file(config_name))\n\n    cfg.DATASETS.TRAIN = (DATA_REGISTER_TRAINING,)\n    cfg.DATASETS.TEST = (DATA_REGISTER_VALID,)\n\n    # to evaluate during training, you have to implement `build_evaluator()` method of the trainer.\n    # https://github.com/facebookresearch/detectron2/blob/94113be6e12db36b8c7601e13747587f19ec92fe/detectron2/engine/defaults.py#L561\n    # cfg.TEST.EVAL_PERIOD = 500\n\n    cfg.DATALOADER.NUM_WORKERS = 2\n\n    # cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(config_name)\n    if (is_resume_training):\n        print(\"#### SETTING PRETRAINED WEIGHTS TO RESUME TRAINING ####\")\n        cfg.MODEL.WEIGHTS = str(PRETRAINED_PATH)\n    else:\n        print(\"#### TRAINING MODEL FROM SCRATCH ####\")\n\n    cfg.SOLVER.AMP.ENABLED = True\n    cfg.SOLVER.IMS_PER_BATCH = 8\n    cfg.SOLVER.BASE_LR = 0.00001\n\n    cfg.SOLVER.WARMUP_ITERS = 10\n\n    # Maximum number of iterations\n    cfg.SOLVER.MAX_ITER = 7000\n\n    # cfg.SOLVER.STEPS = (500, 1000) # must be less than MAX_ITER\n\n    cfg.SOLVER.GAMMA = 0.001\n\n    # Small value == Frequent save need a lot of storage.\n    cfg.SOLVER.CHECKPOINT_PERIOD = 500\n    cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128\n    cfg.MODEL.ROI_HEADS.NUM_CLASSES = 4\n\n    # Create Output Directory\n    cfg.OUTPUT_DIR = str(OUTPUT_DIR)\n    print(\"creating cfg.OUTPUT_DIR -> \", cfg.OUTPUT_DIR)\n    OUTPUT_DIR.mkdir(exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-25T19:13:58.666225Z","iopub.execute_input":"2023-07-25T19:13:58.666623Z","iopub.status.idle":"2023-07-25T19:13:58.696419Z","shell.execute_reply.started":"2023-07-25T19:13:58.666591Z","shell.execute_reply":"2023-07-25T19:13:58.695356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_train:\n    trainer = DefaultTrainer(cfg) if not is_augment else AugTrainer(cfg)\n        \n    trainer.resume_or_load(resume=is_resume_training)\n\n    trainer.train()\n    \n    print(\"#### TRAINING COMPLETE ####\")\n    _ = trainer.model.train(False)  # turn off training\n    \n    FileLink(str(OUTPUT_MODEL))","metadata":{"execution":{"iopub.status.busy":"2023-07-25T19:14:12.390484Z","iopub.execute_input":"2023-07-25T19:14:12.391216Z","iopub.status.idle":"2023-07-25T19:17:55.062993Z","shell.execute_reply.started":"2023-07-25T19:14:12.39118Z","shell.execute_reply":"2023-07-25T19:17:55.061636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_train:\n    # Load metrics\n    metrics_df = pd.read_json(\n        OUTPUT_DIR/\"metrics.json\", orient=\"records\", lines=True\n    )\n    mdf = metrics_df.sort_values(\"iteration\")\n    print(mdf.head(10).T)\n\n    # Plot loss\n    fig, ax = plt.subplots()\n\n    mdf1 = mdf[~mdf[\"total_loss\"].isna()]\n    ax.plot(mdf1[\"iteration\"], mdf1[\"total_loss\"], c=\"C0\", label=\"train\")\n\n    if \"validation_loss\" in mdf.columns:\n        mdf2 = mdf[~mdf[\"validation_loss\"].isna()]\n        ax.plot(mdf2[\"iteration\"], mdf2[\"validation_loss\"],\n                c=\"C1\", label=\"validation\")\n\n    ax.legend()\n    ax.set_title(\"Loss curve\")\n    plt.show()\n\n    # Plot Accuracy\n    fig, ax = plt.subplots()\n\n    mdf1 = mdf[~mdf[\"fast_rcnn/cls_accuracy\"].isna()]\n    ax.plot(mdf1[\"iteration\"], mdf1[\"fast_rcnn/cls_accuracy\"],\n            c=\"C0\", label=\"train\")\n\n    ax.legend()\n    ax.set_title(\"Accuracy curve\")\n    plt.show()\n\n    # Plot Bounding Box regressor loss\n    fig, ax = plt.subplots()\n\n    mdf1 = mdf[~mdf[\"loss_box_reg\"].isna()]\n    ax.plot(mdf1[\"iteration\"], mdf1[\"loss_box_reg\"], c=\"C0\", label=\"train\")\n\n    ax.legend()\n    ax.set_title(\"loss_box_reg\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-19T11:34:37.084328Z","iopub.execute_input":"2023-07-19T11:34:37.084803Z","iopub.status.idle":"2023-07-19T11:34:38.018868Z","shell.execute_reply.started":"2023-07-19T11:34:37.084758Z","shell.execute_reply":"2023-07-19T11:34:38.014491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_evaluate:\n    print(\"### EVALUATING ON VALIDATION DATA ####\")\n    # trained model weights\n    cfg.MODEL.WEIGHTS = str(MODEL_PATH)\n    cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5   # set a custom testing threshold\n\n    cfg.SOLVER.IMS_PER_BATCH = 64\n\n    evaluator = COCOEvaluator(\n        DATA_REGISTER_VALID, cfg, False, output_dir=cfg.OUTPUT_DIR, use_fast_impl=True\n    )\n\n    val_loader = build_detection_test_loader(cfg, DATA_REGISTER_VALID)\n\n    results = inference_on_dataset(\n        trainer.model, val_loader, evaluator=evaluator\n    )","metadata":{"execution":{"iopub.status.busy":"2023-07-19T11:35:45.307512Z","iopub.execute_input":"2023-07-19T11:35:45.30794Z","iopub.status.idle":"2023-07-19T11:35:45.31461Z","shell.execute_reply.started":"2023-07-19T11:35:45.307909Z","shell.execute_reply":"2023-07-19T11:35:45.313265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inf_cfg = get_cfg()\n\nconfig_name = \"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"\n\ninf_cfg.merge_from_file(model_zoo.get_config_file(config_name))\ninf_cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128\ninf_cfg.MODEL.ROI_HEADS.NUM_CLASSES = 4\ninf_cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5\ninf_cfg.MODEL.DEVICE = \"cuda\"\n\ninf_cfg.DATALOADER.NUM_WORKERS = 2  # lower this if CUDA overflow occurs\ninf_cfg.MODEL.WEIGHTS = str(MODEL_PATH)","metadata":{"execution":{"iopub.status.busy":"2023-07-19T11:35:51.147799Z","iopub.execute_input":"2023-07-19T11:35:51.148211Z","iopub.status.idle":"2023-07-19T11:35:51.1773Z","shell.execute_reply.started":"2023-07-19T11:35:51.148178Z","shell.execute_reply":"2023-07-19T11:35:51.176267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH = 8  # lower this if CUDA overflow occurs\ntest_loader = build_detection_test_loader(inf_cfg, DATA_REGISTER_TEST, batch_size=BATCH)","metadata":{"execution":{"iopub.status.busy":"2023-07-19T11:35:56.875121Z","iopub.execute_input":"2023-07-19T11:35:56.875777Z","iopub.status.idle":"2023-07-19T11:35:58.134593Z","shell.execute_reply.started":"2023-07-19T11:35:56.875745Z","shell.execute_reply":"2023-07-19T11:35:58.133511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ACCEPTANCE_THRESHOLD = 0.6  # for all categories","metadata":{"execution":{"iopub.status.busy":"2023-07-19T11:36:15.771041Z","iopub.execute_input":"2023-07-19T11:36:15.772119Z","iopub.status.idle":"2023-07-19T11:36:15.77715Z","shell.execute_reply.started":"2023-07-19T11:36:15.772074Z","shell.execute_reply":"2023-07-19T11:36:15.775883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"#### MODEL: {inf_cfg.MODEL.WEIGHTS} FOR INFERENCE ####\")","metadata":{"execution":{"iopub.status.busy":"2023-07-19T11:36:18.412255Z","iopub.execute_input":"2023-07-19T11:36:18.412641Z","iopub.status.idle":"2023-07-19T11:36:18.41828Z","shell.execute_reply.started":"2023-07-19T11:36:18.412609Z","shell.execute_reply":"2023-07-19T11:36:18.417095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictor = DefaultPredictor(inf_cfg)","metadata":{"execution":{"iopub.status.busy":"2023-07-19T11:36:23.138342Z","iopub.execute_input":"2023-07-19T11:36:23.138728Z","iopub.status.idle":"2023-07-19T11:36:24.049514Z","shell.execute_reply.started":"2023-07-19T11:36:23.138697Z","shell.execute_reply":"2023-07-19T11:36:24.047956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(2, 2, figsize=(20, 20))\nindices = [ax[0][0], ax[1][0], ax[0][1], ax[1][1]]\n\n# Show some qualitative results by predicting on test set images\nNUM_TEST_SAMPLES = 4\nsamples = np.random.choice(dataset_dicts_test, NUM_TEST_SAMPLES)\n\nfor i, sample in enumerate(samples):\n    img = cv2.imread(sample[\"file_name\"])\n    outputs = predictor(img)\n    visualizer = Visualizer(img, metadata=metadata_dicts_test, scale=0.5,)\n    visualizer = visualizer.draw_instance_predictions(\n        outputs[\"instances\"].to(\"cpu\")\n    )\n    display_img = visualizer.get_image()[:, :, ::-1]\n    indices[i].grid(False)\n    indices[i].imshow(display_img)","metadata":{"execution":{"iopub.status.busy":"2023-07-19T11:36:31.916165Z","iopub.execute_input":"2023-07-19T11:36:31.916526Z","iopub.status.idle":"2023-07-19T11:36:32.830901Z","shell.execute_reply.started":"2023-07-19T11:36:31.916496Z","shell.execute_reply":"2023-07-19T11:36:32.829389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rebuild_model():\n    model = build_model(inf_cfg)\n    _ = DetectionCheckpointer(model).load(inf_cfg.MODEL.WEIGHTS)\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-07-19T11:21:34.092901Z","iopub.status.idle":"2023-07-19T11:21:34.093775Z","shell.execute_reply.started":"2023-07-19T11:21:34.093524Z","shell.execute_reply":"2023-07-19T11:21:34.093546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = rebuild_model()","metadata":{"execution":{"iopub.status.busy":"2023-07-19T11:21:34.095157Z","iopub.status.idle":"2023-07-19T11:21:34.09596Z","shell.execute_reply.started":"2023-07-19T11:21:34.095716Z","shell.execute_reply":"2023-07-19T11:21:34.095739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!export LRU_CACHE_CAPACITY=1\n!export 'PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:512'","metadata":{"execution":{"iopub.status.busy":"2023-07-19T11:21:34.097478Z","iopub.status.idle":"2023-07-19T11:21:34.098253Z","shell.execute_reply.started":"2023-07-19T11:21:34.09801Z","shell.execute_reply":"2023-07-19T11:21:34.098032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vars_to_del = [\"trainer\", \"predictor\", \"outputs\"]\n\nfor v in vars_to_del:\n    if v in globals():\n        print(f\"Deleting {v}\")\n        del globals()[v]\n    elif v in locals():\n        print(f\"Deleting {v}\")\n        del locals()[v]","metadata":{"execution":{"iopub.status.busy":"2023-07-19T11:21:34.09962Z","iopub.status.idle":"2023-07-19T11:21:34.100371Z","shell.execute_reply.started":"2023-07-19T11:21:34.100129Z","shell.execute_reply":"2023-07-19T11:21:34.100152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rle_encode(mask):\n    pixels = mask.T.flatten()\n    use_padding = False\n    if pixels[0] or pixels[-1]:\n        use_padding = True\n        pixel_padded = np.zeros([len(pixels) + 2], dtype=pixels.dtype)\n        pixel_padded[1:-1] = pixels\n        pixels = pixel_padded\n    rle = np.where(pixels[1:] != pixels[:-1])[0] + 2\n    if use_padding:\n        rle = rle - 1\n    rle[1::2] = rle[1::2] - rle[:-1:2]\n    return ' '.join(str(x) for x in rle)","metadata":{"execution":{"iopub.status.busy":"2023-07-19T11:21:34.101803Z","iopub.status.idle":"2023-07-19T11:21:34.102628Z","shell.execute_reply.started":"2023-07-19T11:21:34.102347Z","shell.execute_reply":"2023-07-19T11:21:34.102371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@retry_if_cuda_oom\ndef get_masks(prediction):\n    # get masks for each category\n    take = prediction.scores >= ACCEPTANCE_THRESHOLD\n    pred_masks = (prediction.pred_masks[take] != 0)\n    pred_classes = prediction.pred_classes[take]\n  \n    rles = []\n    for cat in range(len(thing_classes)):\n        pred_mask = pred_masks[pred_classes == cat]\n        \n        # pred_mask = retry_if_cuda_oom(torch.any)(pred_mask, dim=0)\n        pred_mask = torch.any(pred_mask, dim=0)\n        rles.append(rle_encode(pred_mask.short().to(\"cpu\").numpy()))\n\n    return rles","metadata":{"execution":{"iopub.status.busy":"2023-07-19T11:21:34.10407Z","iopub.status.idle":"2023-07-19T11:21:34.104913Z","shell.execute_reply.started":"2023-07-19T11:21:34.104673Z","shell.execute_reply":"2023-07-19T11:21:34.104695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_inference(data):\n    results = []\n    with torch.no_grad():\n        outputs = model(data)\n        if torch.cuda.is_available():\n            torch.cuda.synchronize()\n\n        for idx, output in enumerate(outputs):\n            output = output[\"instances\"]\n\n            rles = get_masks(output)\n\n            result = [\n                f\"{data[idx]['image_id']}_{cat},{rles[cat]}\\n\"\n                for cat in range(len(thing_classes))\n            ]\n\n            results.extend(result)\n\n        del outputs, output\n\n    return results","metadata":{"execution":{"iopub.status.busy":"2023-07-19T11:21:34.106284Z","iopub.status.idle":"2023-07-19T11:21:34.107058Z","shell.execute_reply.started":"2023-07-19T11:21:34.106815Z","shell.execute_reply":"2023-07-19T11:21:34.106838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"#### RUNNING INFERENCE ON TEST DATA ####\")","metadata":{"execution":{"iopub.status.busy":"2023-07-19T11:21:34.108437Z","iopub.status.idle":"2023-07-19T11:21:34.109218Z","shell.execute_reply.started":"2023-07-19T11:21:34.108974Z","shell.execute_reply":"2023-07-19T11:21:34.108997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-07-19T11:37:35.452427Z","iopub.execute_input":"2023-07-19T11:37:35.45347Z","iopub.status.idle":"2023-07-19T11:37:36.944698Z","shell.execute_reply.started":"2023-07-19T11:37:35.453433Z","shell.execute_reply":"2023-07-19T11:37:36.943576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_inference:\n    model.eval()\n    submission_file = open(\"submission.csv\", \"w\")\n    submission_file.write(\"Id,Predicted\\n\")\n\n    results: list[str] = []\n    \n    for i, data in enumerate(tqdm(test_loader)):\n        res = run_inference(data)\n        results.extend(res)\n        \n        if i % (500 // BATCH) == 0:\n            print(f\"Inference on batch {i}/{len(test_loader)} done\")\n            submission_file.writelines(results)\n            results = []\n\n    submission_file.writelines(results)\n    submission_file.close()","metadata":{"execution":{"iopub.status.busy":"2023-07-19T11:21:34.112788Z","iopub.status.idle":"2023-07-19T11:21:34.113553Z","shell.execute_reply.started":"2023-07-19T11:21:34.113297Z","shell.execute_reply":"2023-07-19T11:21:34.113319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if Path(\"submission.csv\").exists:\n    display(FileLink(\"submission.csv\"))","metadata":{"execution":{"iopub.status.busy":"2023-07-19T11:21:34.114937Z","iopub.status.idle":"2023-07-19T11:21:34.115706Z","shell.execute_reply.started":"2023-07-19T11:21:34.115435Z","shell.execute_reply":"2023-07-19T11:21:34.115457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r detectron2/","metadata":{"execution":{"iopub.status.busy":"2023-07-19T11:21:34.117066Z","iopub.status.idle":"2023-07-19T11:21:34.117857Z","shell.execute_reply.started":"2023-07-19T11:21:34.117619Z","shell.execute_reply":"2023-07-19T11:21:34.117642Z"},"trusted":true},"execution_count":null,"outputs":[]}]}