{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from pathlib import Path\n\nTEST_IMG_DIR = Path(\"/kaggle/input/dlsprint2/badlad/images/test\")\n\nTEST_METADATA_PATH = Path(\"/kaggle/input/dlsprint2/badlad/badlad-test-metadata.json\")","metadata":{"id":"8dffdbd3-bc6c-48df-8830-cb0bc68f5bbc","execution":{"iopub.status.busy":"2023-08-05T07:10:24.891119Z","iopub.execute_input":"2023-08-05T07:10:24.894176Z","iopub.status.idle":"2023-08-05T07:10:24.902715Z","shell.execute_reply.started":"2023-08-05T07:10:24.894134Z","shell.execute_reply":"2023-08-05T07:10:24.901813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import datetime\nimport time\n\n# if True, run inference on test dataset\nis_inference = True\n\nSEED = int(datetime.now().timestamp())","metadata":{"execution":{"iopub.status.busy":"2023-08-05T07:10:25.686358Z","iopub.execute_input":"2023-08-05T07:10:25.687554Z","iopub.status.idle":"2023-08-05T07:10:25.692489Z","shell.execute_reply.started":"2023-08-05T07:10:25.687478Z","shell.execute_reply":"2023-08-05T07:10:25.691556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Detectron 2","metadata":{}},{"cell_type":"code","source":"%%capture\nimport sys, os, distutils.core\n# Note: This is a faster way to install detectron2 in Colab, but it does not include all functionalities (e.g. compiled operators).\n# See https://detectron2.readthedocs.io/tutorials/install.html for full installation instructions\n!git clone 'https://github.com/facebookresearch/detectron2'\ndist = distutils.core.run_setup(\"./detectron2/setup.py\")\n!python -m pip install {' '.join([f\"'{x}'\" for x in dist.install_requires])}\nsys.path.insert(0, os.path.abspath('./detectron2'))","metadata":{"id":"7b9d3b50-222c-4ef8-91f5-1b9b4b19c788","execution":{"iopub.status.busy":"2023-08-05T07:10:27.035328Z","iopub.execute_input":"2023-08-05T07:10:27.035697Z","iopub.status.idle":"2023-08-05T07:11:16.707077Z","shell.execute_reply.started":"2023-08-05T07:10:27.035666Z","shell.execute_reply":"2023-08-05T07:11:16.705617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!rm -rf /kaggle/working/unilm\n!pip install gdown\n!gdown 1CoGAoFqEK-4yUOkikxeM6LHS2MZEokDU\n!unzip /kaggle/working/unilm.zip","metadata":{"execution":{"iopub.status.busy":"2023-08-05T07:11:16.709971Z","iopub.execute_input":"2023-08-05T07:11:16.711186Z","iopub.status.idle":"2023-08-05T07:11:38.144505Z","shell.execute_reply.started":"2023-08-05T07:11:16.711144Z","shell.execute_reply":"2023-08-05T07:11:38.143022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# detectron2\nfrom detectron2.utils.memory import retry_if_cuda_oom\nfrom detectron2.utils.logger import setup_logger\nfrom detectron2.checkpoint import DetectionCheckpointer\nfrom detectron2.modeling import build_model\nfrom detectron2.evaluation import COCOEvaluator, inference_on_dataset\nimport detectron2.data.transforms as T\nfrom detectron2.data import detection_utils as utils\nfrom detectron2.data import DatasetCatalog, MetadataCatalog, build_detection_test_loader, build_detection_train_loader, DatasetMapper\nfrom detectron2.utils.visualizer import Visualizer\nfrom detectron2.structures import BoxMode\nfrom detectron2.engine import DefaultPredictor, DefaultTrainer\nfrom detectron2.config import get_cfg\nfrom detectron2 import model_zoo\n\nimport pandas as pd\nimport numpy as np\nfrom tqdm.notebook import tqdm  # progress bar\nimport matplotlib.pyplot as plt\nimport json\nimport cv2\nimport copy\nfrom typing import Optional\n\nfrom IPython.display import FileLink\n\n# torch\nimport torch\n\nimport gc\n\nimport warnings\n# Ignore \"future\" warnings and Data-Frame-Slicing warnings.\nwarnings.filterwarnings('ignore')\n\nsetup_logger()","metadata":{"id":"81837bde-6782-47df-b1d1-b306afeef73a","outputId":"84f63812-dd68-4d9c-bbe6-f88b88614b29","execution":{"iopub.status.busy":"2023-08-05T07:11:38.146332Z","iopub.execute_input":"2023-08-05T07:11:38.146741Z","iopub.status.idle":"2023-08-05T07:11:39.169140Z","shell.execute_reply.started":"2023-08-05T07:11:38.146706Z","shell.execute_reply":"2023-08-05T07:11:39.167987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with TEST_METADATA_PATH.open() as f:\n    test_dict = json.load(f)\n\nprint(\"#### LABELS AND METADATA LOADED ####\")","metadata":{"id":"d71784aa-ac16-4b8f-a9f6-7cde4e1c5559","outputId":"b6de20d9-853d-4fb9-e82a-75ec7eda4109","execution":{"iopub.status.busy":"2023-08-05T07:11:39.172282Z","iopub.execute_input":"2023-08-05T07:11:39.172861Z","iopub.status.idle":"2023-08-05T07:11:39.243625Z","shell.execute_reply.started":"2023-08-05T07:11:39.172818Z","shell.execute_reply":"2023-08-05T07:11:39.242596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def organize_coco_data(data_dict: dict) -> tuple[list[str], list[dict], list[dict]]:\n    thing_classes: list[str] = []\n\n    # Map Category Names to IDs\n    for cat in data_dict['categories']:\n        thing_classes.append(cat['name'])\n\n    # Images\n    images_metadata: list[dict] = data_dict['images']\n\n    # Convert COCO annotations to detectron2 annotations format\n    data_annotations = []\n    for ann in data_dict['annotations']:\n        # coco format -> detectron2 format\n        annot_obj = {\n            # Annotation ID\n            \"id\": ann['id'],\n\n            # Segmentation Polygon (x, y) coords\n            \"gt_masks\": ann['segmentation'],\n\n            # Image ID for this annotation (Which image does this annotation belong to?)\n            \"image_id\": ann['image_id'],\n\n            # Category Label (0: paragraph, 1: text box, 2: image, 3: table)\n            \"category_id\": ann['category_id'],\n\n            \"x_min\": ann['bbox'][0],  # left\n            \"y_min\": ann['bbox'][1],  # top\n            \"x_max\": ann['bbox'][0] + ann['bbox'][2],  # left+width\n            \"y_max\": ann['bbox'][1] + ann['bbox'][3]  # top+height\n        }\n        data_annotations.append(annot_obj)\n\n    return thing_classes, images_metadata, data_annotations","metadata":{"id":"a4443e88-8034-4319-bf18-9391626d1db3","execution":{"iopub.status.busy":"2023-08-05T07:11:39.245308Z","iopub.execute_input":"2023-08-05T07:11:39.245926Z","iopub.status.idle":"2023-08-05T07:11:39.255094Z","shell.execute_reply.started":"2023-08-05T07:11:39.245875Z","shell.execute_reply":"2023-08-05T07:11:39.253895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"thing_classes, images_metadata_test, _ = organize_coco_data(\n    test_dict\n)\n\n# del test_dict\n# gc.collect()\n","metadata":{"id":"cc314bee-23d7-4aa0-ba1d-a9089576856d","execution":{"iopub.status.busy":"2023-08-05T07:11:39.256588Z","iopub.execute_input":"2023-08-05T07:11:39.256965Z","iopub.status.idle":"2023-08-05T07:11:39.270273Z","shell.execute_reply.started":"2023-08-05T07:11:39.256885Z","shell.execute_reply":"2023-08-05T07:11:39.269403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_metadata = pd.DataFrame(images_metadata_test)\ntest_metadata = test_metadata[['id', 'file_name', 'width', 'height']]\ntest_metadata = test_metadata.rename(columns={\"id\": \"image_id\"})\nprint(\"test_metadata size=\", len(test_metadata))\ntest_metadata.head(5)","metadata":{"id":"2a1aca4e-55d1-4b15-a615-a622999a2946","outputId":"938edcae-48a2-45a6-fb59-b3d73cf7783a","execution":{"iopub.status.busy":"2023-08-05T07:11:39.271504Z","iopub.execute_input":"2023-08-05T07:11:39.271925Z","iopub.status.idle":"2023-08-05T07:11:39.345663Z","shell.execute_reply.started":"2023-08-05T07:11:39.271894Z","shell.execute_reply":"2023-08-05T07:11:39.344576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_coco_to_detectron2_format(\n    imgdir: Path,\n    metadata_df: pd.DataFrame,\n    annot_df: Optional[pd.DataFrame] = None,\n    target_indices: Optional[np.ndarray] = None,\n):\n\n    dataset_dicts = []\n    for _, train_meta_row in tqdm(metadata_df.iterrows(), total=len(metadata_df)):\n        # Iterate over each image\n        image_id, filename, width, height = train_meta_row.values\n\n        annotations = []\n\n        # If train/validation data, then there will be annotations\n        if annot_df is not None:\n            for _, ann in annot_df.query(\"image_id == @image_id\").iterrows():\n                # Get annotations of current iteration's image\n                class_id = ann[\"category_id\"]\n                gt_masks = ann[\"gt_masks\"]\n                bbox_resized = [\n                    float(ann[\"x_min\"]),\n                    float(ann[\"y_min\"]),\n                    float(ann[\"x_max\"]),\n                    float(ann[\"y_max\"]),\n                ]\n\n                annotation = {\n                    \"bbox\": bbox_resized,\n                    \"bbox_mode\": BoxMode.XYXY_ABS,\n                    \"segmentation\": gt_masks,\n                    \"category_id\": class_id,\n                }\n\n                annotations.append(annotation)\n\n        # coco format -> detectron2 format dict\n        record = {\n            \"file_name\": str(imgdir/filename),\n            \"image_id\": image_id,\n            \"width\": width,\n            \"height\": height,\n            \"annotations\": annotations\n        }\n\n        dataset_dicts.append(record)\n\n    if target_indices is not None:\n        dataset_dicts = [dataset_dicts[i] for i in target_indices]\n\n    return dataset_dicts","metadata":{"id":"82252771-a388-4f81-b6cc-0df1e5350045","execution":{"iopub.status.busy":"2023-08-05T07:11:39.347636Z","iopub.execute_input":"2023-08-05T07:11:39.348315Z","iopub.status.idle":"2023-08-05T07:11:39.360361Z","shell.execute_reply.started":"2023-08-05T07:11:39.348273Z","shell.execute_reply":"2023-08-05T07:11:39.359246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DatasetCatalog.clear()\nDATA_REGISTER_TEST     = \"badlad_test\"\n\n# Register Test Inference data\nDatasetCatalog.register(\n    DATA_REGISTER_TEST,\n    lambda: convert_coco_to_detectron2_format(\n        TEST_IMG_DIR,\n        test_metadata,\n    )\n)\n\n# Set Test data categories\nMetadataCatalog.get(DATA_REGISTER_TEST).set(\n    thing_classes=thing_classes\n)\n\ndataset_dicts_test = DatasetCatalog.get(DATA_REGISTER_TEST)\nmetadata_dicts_test = MetadataCatalog.get(DATA_REGISTER_TEST)\n\n# del test_metadata\n# gc.collect()\n# print(\"dicts valid size=\", len(dataset_dicts_test))\n# print(\"################\")","metadata":{"id":"fb9ce155-b2f9-4cd2-aa2c-bc24dbd8e3d4","outputId":"2c306ee9-56ec-4da2-aa3c-63d6ac649d51","execution":{"iopub.status.busy":"2023-08-05T07:11:39.362044Z","iopub.execute_input":"2023-08-05T07:11:39.362449Z","iopub.status.idle":"2023-08-05T07:11:40.408204Z","shell.execute_reply.started":"2023-08-05T07:11:39.362412Z","shell.execute_reply":"2023-08-05T07:11:40.407230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Get configs","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.insert(1, \"kaggle/working/unilm\")\n\n! sed -i 's/from collections import Iterable/from collections.abc import Iterable/' /kaggle/working/unilm/dit/object_detection/ditod/table_evaluation/data_structure.py","metadata":{"id":"817a8191-6fe8-40c3-9453-30a96420685d","execution":{"iopub.status.busy":"2023-08-05T07:11:40.412636Z","iopub.execute_input":"2023-08-05T07:11:40.413527Z","iopub.status.idle":"2023-08-05T07:11:41.415213Z","shell.execute_reply.started":"2023-08-05T07:11:40.413451Z","shell.execute_reply":"2023-08-05T07:11:41.413716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"code","source":"!gdown 1u3Qsh7951kR7S78vkmKE5cf8uj4i6PwC\nMODEL_PATH_DIT=Path(\"/kaggle/working/Dit_50kiter.pth\")","metadata":{"execution":{"iopub.status.busy":"2023-08-05T07:11:41.417347Z","iopub.execute_input":"2023-08-05T07:11:41.417774Z","iopub.status.idle":"2023-08-05T07:11:52.438862Z","shell.execute_reply.started":"2023-08-05T07:11:41.417735Z","shell.execute_reply":"2023-08-05T07:11:52.437672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from unilm.dit.object_detection.ditod import add_vit_config as add_dit\ndit_inf_cfg = get_cfg()\n\nadd_dit(dit_inf_cfg)\n# Load a config from file\ndit_inf_cfg.merge_from_file(\"./unilm/dit/object_detection/publaynet_configs/maskrcnn/maskrcnn_dit_base.yaml\")\ndit_inf_cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128\ndit_inf_cfg.MODEL.ROI_HEADS.NUM_CLASSES = 4\ndit_inf_cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5\ndit_inf_cfg.MODEL.DEVICE = \"cuda\"\ndit_inf_cfg.MODEL.IMAGE_ONLY = False\n\ndit_inf_cfg.DATALOADER.NUM_WORKERS = 1  # lower this if CUDA overflow occurs\ndit_inf_cfg.MODEL.WEIGHTS = str(MODEL_PATH_DIT)\nBATCH = 1 # lower this if CUDA overflow occurs\n# test_loader = build_detection_test_loader(dit_inf_cfg, DATA_REGISTER_TEST, batch_size=BATCH)","metadata":{"id":"8afe8959-3268-4717-9a63-5f22c8f0e31e","outputId":"dcdaefd3-0340-49b7-ef28-f2b05be47210","execution":{"iopub.status.busy":"2023-08-05T07:11:52.441143Z","iopub.execute_input":"2023-08-05T07:11:52.442188Z","iopub.status.idle":"2023-08-05T07:12:02.598081Z","shell.execute_reply.started":"2023-08-05T07:11:52.442147Z","shell.execute_reply":"2023-08-05T07:12:02.597086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ACCEPTANCE_THRESHOLD = 0.6  # for all categories","metadata":{"id":"c63cebda-2038-4070-8d61-8adfb88bea94","execution":{"iopub.status.busy":"2023-08-05T07:12:02.599606Z","iopub.execute_input":"2023-08-05T07:12:02.600331Z","iopub.status.idle":"2023-08-05T07:12:02.606105Z","shell.execute_reply.started":"2023-08-05T07:12:02.600297Z","shell.execute_reply":"2023-08-05T07:12:02.604051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(f\"#### MODEL: {dit_inf_cfg.MODEL.WEIGHTS} FOR INFERENCE ####\")\n# predictor_dit = DefaultPredictor(dit_inf_cfg)","metadata":{"id":"0dcb8109-8bff-4465-9ebd-3d6a2968b7a7","outputId":"0058d176-877c-4f4c-e95a-5e6b86be75fd","execution":{"iopub.status.busy":"2023-08-05T07:12:02.607570Z","iopub.execute_input":"2023-08-05T07:12:02.608449Z","iopub.status.idle":"2023-08-05T07:12:02.617105Z","shell.execute_reply.started":"2023-08-05T07:12:02.608416Z","shell.execute_reply":"2023-08-05T07:12:02.615898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import numpy as np\n# import cv2\n# import matplotlib.pyplot as plt\n# from detectron2.utils.visualizer import Visualizer\n\n# fig, ax = plt.subplots(4, 2, figsize=(20, 40))  # Updated subplot dimensions\n# indices = [ax[i // 2][i % 2] for i in range(8)]  # Adjusted indices for 8 plots\n\n# # Show some qualitative results by predicting on test set images\n# NUM_TEST_SAMPLES = 8  # Updated number of test samples\n# samples = np.random.choice(dataset_dicts_test, NUM_TEST_SAMPLES)\n\n# for i, sample in enumerate(samples):\n#     img = cv2.imread(sample[\"file_name\"])\n#     outputs = predictor_dit(img)\n#     visualizer = Visualizer(img, metadata=metadata_dicts_test, scale=0.5)\n#     visualizer = visualizer.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\n#     display_img = visualizer.get_image()[:, :, ::-1]\n#     indices[i].grid(False)\n#     indices[i].imshow(display_img)\n\n# plt.tight_layout()\n# plt.show()","metadata":{"id":"e76d9bf7-e289-41fb-b339-b1acc506e298","outputId":"74efab1e-f2cc-4b9c-bc5f-37955ea9cb34","execution":{"iopub.status.busy":"2023-08-05T07:12:02.620149Z","iopub.execute_input":"2023-08-05T07:12:02.621053Z","iopub.status.idle":"2023-08-05T07:12:02.629451Z","shell.execute_reply.started":"2023-08-05T07:12:02.621013Z","shell.execute_reply":"2023-08-05T07:12:02.628603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rebuild_model():\n    model = build_model(dit_inf_cfg)\n    _ = DetectionCheckpointer(model).load(dit_inf_cfg.MODEL.WEIGHTS)\n    return model\n# del model_dit\nmodel_dit = rebuild_model()\nmodel_dit.eval();","metadata":{"id":"0bebf2c6-6070-4fae-ad1b-ec56ebbd461e","execution":{"iopub.status.busy":"2023-08-05T07:12:02.632853Z","iopub.execute_input":"2023-08-05T07:12:02.633102Z","iopub.status.idle":"2023-08-05T07:12:10.550486Z","shell.execute_reply.started":"2023-08-05T07:12:02.633080Z","shell.execute_reply":"2023-08-05T07:12:10.549275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Layoutlm","metadata":{}},{"cell_type":"code","source":"# import sys\n# sys.path.insert(1, \"/kaggle/working/unilm/layoutlmv3\")\n! sed -i 's/from collections import Iterable/from collections.abc import Iterable/' /kaggle/working/unilm/layoutlmv3/examples/object_detection/ditod/table_evaluation/data_structure.py","metadata":{"execution":{"iopub.status.busy":"2023-08-05T07:12:10.551989Z","iopub.execute_input":"2023-08-05T07:12:10.552456Z","iopub.status.idle":"2023-08-05T07:12:11.565870Z","shell.execute_reply.started":"2023-08-05T07:12:10.552420Z","shell.execute_reply":"2023-08-05T07:12:11.564615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!gdown 1gmPsp0puIWib6QsIbq9wJQ98JbWwJNKH\n!gdown 1CwIgwAFY4s7Nz-ST7Al2KGL1qtrlIhFx","metadata":{"execution":{"iopub.status.busy":"2023-08-05T07:12:11.567785Z","iopub.execute_input":"2023-08-05T07:12:11.568200Z","iopub.status.idle":"2023-08-05T07:12:21.995252Z","shell.execute_reply.started":"2023-08-05T07:12:11.568160Z","shell.execute_reply":"2023-08-05T07:12:21.994052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from detectron2.config import get_cfg\nfrom unilm.layoutlmv3.examples.object_detection.ditod import add_vit_config as add_layout\n\nMODEL_PATH=Path(\"/kaggle/working/model_0023999.pth\")\n\nllm_inf_cfg = get_cfg()\n# config_name = \"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"\n\n# inf_cfg.merge_from_file(model_zoo.get_config_file(config_name))\nadd_layout(llm_inf_cfg)\n# Load a config from file\nllm_inf_cfg.merge_from_file(\"/kaggle/working/unilm/layoutlmv3/examples/object_detection/cascade_layoutlmv3.yaml\", allow_unsafe=True)\nllm_inf_cfg.MODEL.CONFIG_PATH=\"/kaggle/working/config.json\"\nllm_inf_cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128\nllm_inf_cfg.MODEL.ROI_HEADS.NUM_CLASSES = 4\nllm_inf_cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5\nllm_inf_cfg.MODEL.DEVICE = \"cuda\"\n\nllm_inf_cfg.DATALOADER.NUM_WORKERS = 1  # lower this if CUDA overflow occurs\nllm_inf_cfg.MODEL.WEIGHTS = str(MODEL_PATH)\nBATCH = 1 # lower this if CUDA overflow occurs\ntest_loader = build_detection_test_loader(llm_inf_cfg, DATA_REGISTER_TEST, batch_size=BATCH)","metadata":{"execution":{"iopub.status.busy":"2023-08-05T07:12:45.701440Z","iopub.execute_input":"2023-08-05T07:12:45.702376Z","iopub.status.idle":"2023-08-05T07:12:47.327395Z","shell.execute_reply.started":"2023-08-05T07:12:45.702330Z","shell.execute_reply":"2023-08-05T07:12:47.326493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# llm_predictor = DefaultPredictor(llm_inf_cfg)","metadata":{"execution":{"iopub.status.busy":"2023-08-05T07:12:23.344558Z","iopub.execute_input":"2023-08-05T07:12:23.344919Z","iopub.status.idle":"2023-08-05T07:12:23.349578Z","shell.execute_reply.started":"2023-08-05T07:12:23.344885Z","shell.execute_reply":"2023-08-05T07:12:23.348459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import numpy as np\n# import cv2\n# import matplotlib.pyplot as plt\n# from detectron2.utils.visualizer import Visualizer\n\n# fig, ax = plt.subplots(4, 2, figsize=(20, 40))  # Updated subplot dimensions\n# indices = [ax[i // 2][i % 2] for i in range(8)]  # Adjusted indices for 8 plots\n\n# # Show some qualitative results by predicting on test set images\n# NUM_TEST_SAMPLES = 8  # Updated number of test samples\n# samples = np.random.choice(dataset_dicts_test, NUM_TEST_SAMPLES)\n\n# for i, sample in enumerate(samples):\n#     img = cv2.imread(sample[\"file_name\"])\n#     outputs = llm_predictor(img)\n#     visualizer = Visualizer(img, metadata=metadata_dicts_test, scale=0.5)\n#     visualizer = visualizer.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\n#     display_img = visualizer.get_image()[:, :, ::-1]\n#     indices[i].grid(False)\n#     indices[i].imshow(display_img)\n\n# plt.tight_layout()\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-08-05T07:12:23.351097Z","iopub.execute_input":"2023-08-05T07:12:23.351761Z","iopub.status.idle":"2023-08-05T07:12:23.360288Z","shell.execute_reply.started":"2023-08-05T07:12:23.351726Z","shell.execute_reply":"2023-08-05T07:12:23.359580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rebuild_model():\n    model = build_model(llm_inf_cfg)\n    _ = DetectionCheckpointer(model).load(llm_inf_cfg.MODEL.WEIGHTS)\n    return model\n# del model_layout\nmodel_layout = rebuild_model()\nmodel_layout.eval();","metadata":{"execution":{"iopub.status.busy":"2023-08-05T07:13:10.187589Z","iopub.execute_input":"2023-08-05T07:13:10.188606Z","iopub.status.idle":"2023-08-05T07:13:12.880394Z","shell.execute_reply.started":"2023-08-05T07:13:10.188565Z","shell.execute_reply":"2023-08-05T07:13:12.879301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble","metadata":{}},{"cell_type":"code","source":"!export LRU_CACHE_CAPACITY=1\n!export 'PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:512'","metadata":{"id":"b250ce7e-c78b-42a5-b730-482c5cd36722","execution":{"iopub.status.busy":"2023-08-05T07:13:23.114416Z","iopub.execute_input":"2023-08-05T07:13:23.114994Z","iopub.status.idle":"2023-08-05T07:13:25.439808Z","shell.execute_reply.started":"2023-08-05T07:13:23.114955Z","shell.execute_reply":"2023-08-05T07:13:25.438408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vars_to_del = [\"trainer\", \"predictor\", \"outputs\"]\n\nfor v in vars_to_del:\n    if v in globals():\n        print(f\"Deleting {v}\")\n        del globals()[v]\n    elif v in locals():\n        print(f\"Deleting {v}\")\n        del locals()[v]","metadata":{"id":"c7d2bbb1-5aa8-4086-9757-c51992617b95","outputId":"d4ef01bb-1bc2-4b67-af6a-1a7fc3370368","execution":{"iopub.status.busy":"2023-08-05T07:13:25.442650Z","iopub.execute_input":"2023-08-05T07:13:25.443319Z","iopub.status.idle":"2023-08-05T07:13:25.450349Z","shell.execute_reply.started":"2023-08-05T07:13:25.443280Z","shell.execute_reply":"2023-08-05T07:13:25.449146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rle_encode(mask):\n    #print(f\"mask= {mask.shape}\")\n    pixels = mask.T.flatten()\n    use_padding = False\n    if pixels[0] or pixels[-1]:\n        use_padding = True\n        pixel_padded = np.zeros([len(pixels) + 2], dtype=pixels.dtype)\n        pixel_padded[1:-1] = pixels\n        pixels = pixel_padded\n    rle = np.where(pixels[1:] != pixels[:-1])[0] + 2\n    if use_padding:\n        rle = rle - 1\n    rle[1::2] = rle[1::2] - rle[:-1:2]\n    return ' '.join(str(x) for x in rle)","metadata":{"id":"8f0c95b5-0e4c-4829-bdb9-3d3150f5feb1","execution":{"iopub.status.busy":"2023-08-05T07:13:26.864014Z","iopub.execute_input":"2023-08-05T07:13:26.864778Z","iopub.status.idle":"2023-08-05T07:13:26.872030Z","shell.execute_reply.started":"2023-08-05T07:13:26.864741Z","shell.execute_reply":"2023-08-05T07:13:26.870990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# @retry_if_cuda_oom\n# def get_masks(prediction):\n#     # get masks for each category\n#     #print(f\"prediction: {prediction}\")\n#     take = prediction.scores >= ACCEPTANCE_THRESHOLD\n#     pred_masks = (prediction.pred_masks[take] != 0)\n#     pred_classes = prediction.pred_classes[take]\n#     #print(f\"pred_masks: {pred_masks.shape}\")\n#     #print(f\"pred_classes:{pred_classes}\")\n#     rles = []\n#     for cat in range(len(thing_classes)):\n#         pred_mask = pred_masks[pred_classes == cat]\n#         print(\"-->\", pred_mask.shape)\n#         # pred_mask = retry_if_cuda_oom(torch.any)(pred_mask, dim=0)\n#         pred_mask = torch.any(pred_mask, dim=0)\n#         #rle=len(rle_encode(pred_mask.short().to(\"cpu\").numpy()))\n#         #print(f\"rle {rle}\")\n#         print(pred_mask.shape)\n#         rles.append(rle_encode(pred_mask.short().to(\"cpu\").numpy()))\n#     #print(f\"rles: {rles}\")\n#     print(len(rles))\n#     return rles\n\n# def run_inference(model, data):\n#     print(data)\n#     print(data[0]['image'].shape)\n#     results = []\n#     with torch.no_grad():\n#         outputs = model(data)\n#         if torch.cuda.is_available():\n#             torch.cuda.synchronize()\n# #         print(outputs)\n#         for idx, output in enumerate(outputs):\n#             output = output[\"instances\"]\n\n#             rles = get_masks(output)\n\n#             result = [\n#                 f\"{data[idx]['image_id']}_{cat},{rles[cat]}\\n\"\n#                 for cat in range(len(thing_classes))\n#             ]\n\n#             results.extend(result)\n\n#         del outputs, output\n#     #print(f\"results: {results}\")\n#     return results\n\n# results: list[str] = []\n\n# for i, data in enumerate(tqdm(test_loader)):\n#     res = run_inference(model_dit, data)\n#     results.extend(res)\n#     break","metadata":{"id":"ce27602f-8305-47af-9e58-4c98b36d8868","execution":{"iopub.status.busy":"2023-08-01T07:52:55.750819Z","iopub.execute_input":"2023-08-01T07:52:55.751101Z","iopub.status.idle":"2023-08-01T07:52:55.759658Z","shell.execute_reply.started":"2023-08-01T07:52:55.751078Z","shell.execute_reply":"2023-08-01T07:52:55.758574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install ensemble-boxes\nfrom ensemble_boxes import weighted_boxes_fusion","metadata":{"execution":{"iopub.status.busy":"2023-08-05T07:14:41.995119Z","iopub.execute_input":"2023-08-05T07:14:41.995485Z","iopub.status.idle":"2023-08-05T07:14:54.277666Z","shell.execute_reply.started":"2023-08-05T07:14:41.995456Z","shell.execute_reply":"2023-08-05T07:14:54.276295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def nms_predictions(classes, scores, bboxes, masks, \n                    iou_th=.5, shape=(520, 704)):\n    he, wd = shape[0], shape[1]\n    boxes_list = [[x[0] / wd, x[1] / he, x[2] / wd, x[3] / he]\n                  for x in bboxes]\n    del bboxes\n    gc.collect()\n    scores_list = [x for x in scores]\n    labels_list = [x for x in classes]\n    nms_bboxes, nms_scores, nms_classes = weighted_boxes_fusion(\n        boxes_list=[boxes_list], \n        scores_list=[scores_list], \n        labels_list=[labels_list], \n        weights=None,\n        iou_thr=iou_th,\n        conf_type=\"max\"\n    )\n    nms_masks = []\n    for s in nms_scores:\n        nms_masks.append(masks[scores.index(s)])\n#     nms_scores, nms_classes, nms_masks = zip(*sorted(zip(nms_scores, nms_classes, nms_masks), reverse=True))\n#     nms_classes, nms_masks = zip(*sorted(zip(nms_classes, nms_masks), reverse=True))\n    \n#     result = []\n#     pred_class = max(set(nms_classes), key=nms_classes.count)\n#     used = np.zeros(shape, dtype=int) \n#     for i, mask in enumerate(nms_masks):\n#         mask = mask * (1 - used)\n#         if mask.sum() >= min_pixels[pred_class]:\n#             used += mask\n#             result.append(mask)\n#     result = np.stack(result)\n#     result = np.any(result, axis=0)\n    nms_masks = np.stack(nms_masks)\n    nms_masks = np.any(nms_masks, axis=0)\n#     return nms_classes, nms_masks #,nms_scores\n    return nms_masks\n\n# def ensemble_pred_masks(masks, classes, min_pixels, shape=(520, 704)):\n#     result = []\n#     pred_class = max(set(classes), key=classes.count)\n#     used = np.zeros(shape, dtype=int) \n#     for i, mask in enumerate(masks):\n#         mask = mask * (1 - used)\n#         if mask.sum() >= min_pixels[pred_class]:\n#             used += mask\n#             result.append(mask)\n#     print(\"res\", result)\n#     result = np.stack(result)\n#     result = np.any(result, axis=0)\n    \n#     return result","metadata":{"execution":{"iopub.status.busy":"2023-08-05T07:15:08.892350Z","iopub.execute_input":"2023-08-05T07:15:08.893109Z","iopub.status.idle":"2023-08-05T07:15:08.902503Z","shell.execute_reply.started":"2023-08-05T07:15:08.893073Z","shell.execute_reply":"2023-08-05T07:15:08.901586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def ensemble(img, models):\n#     encodings = []\n#     for cat in range(len(thing_classes)):\n#         classes = []\n#         scores = []\n#         bboxes = []\n#         masks = []\n        \n#         for i, model in enumerate(models):\n#             output = model(img)\n#             print(output)\n#             output = output['instances']\n            \n#             take = output.scores.cpu().numpy() >= ACCEPTANCE_THRESHOLD\n#             pred_classes = output.pred_classes.cpu().numpy()[take]\n            \n#             classes.extend(output.pred_classes[take][pred_classes == cat].cpu().numpy().tolist())\n#             scores.extend(output.scores[take][pred_classes == cat].cpu().numpy().tolist())\n#             bboxes.extend(output.pred_boxes[take][pred_classes == cat].tensor.cpu().numpy().tolist())\n#             masks.extend(output.pred_masks[take][pred_classes == cat].cpu().numpy())\n            \n#         assert len(classes) == len(masks) , 'ensemble lenght mismatch'\n#         if len(classes) > 1:\n#             try:\n# #                 print(\"BEFORE\")\n# #                 plt.figure()\n# #                 plt.imshow(img)\n# #                 plt.imshow(np.sum(masks, axis=0), alpha=0.3)\n# #                 plt.show()\n#                 classes, scores, masks = nms_predictions(\n#                     classes, \n#                     scores, \n#                     bboxes,\n#                     masks, shape=(masks[0].shape[0],masks[0].shape[1])\n#                 )\n# #                 print(\"BEFORE2\")\n# #                 plt.figure()\n# #                 plt.imshow(img)\n# #                 plt.imshow(np.sum(masks, axis=0), alpha=0.3)\n# #                 plt.show()\n#                 encoded_masks = ensemble_pred_masks(masks, classes, min_pixels=[75, 75, 75, 75], shape=(masks[0].shape[0],masks[0].shape[1]))\n# #                 print(encoded_masks.shape)\n# #                 print(\"AFTER\")\n# #                 plt.figure()\n# #                 plt.imshow(img)\n# #                 plt.imshow(encoded_masks, alpha=0.5)\n# #                 plt.show()\n# #                 plt.imsave(f\"{i}.jpg\", display_img)\n#                 encodings.append(rle_encode(encoded_masks))\n#             except:\n#                 print(\"Error\", len(masks))\n#                 encodings.append(encoded_masks)\n#         else:\n#             encodings.append([\" \"])\n# #     print(len(encodings))\n#     return encodings\n    ","metadata":{"execution":{"iopub.status.busy":"2023-07-30T18:26:35.230194Z","iopub.execute_input":"2023-07-30T18:26:35.230706Z","iopub.status.idle":"2023-07-30T18:26:35.239919Z","shell.execute_reply.started":"2023-07-30T18:26:35.23068Z","shell.execute_reply":"2023-07-30T18:26:35.238998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**a slight compression**","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport gc\n\ndef nms_predictions(classes, scores, bboxes, masks, \n                    iou_th=.5, shape=(520, 704)):\n    he, wd = shape[0], shape[1]\n    boxes_list = [[x[0] / wd, x[1] / he, x[2] / wd, x[3] / he]\n                  for x in bboxes]\n    del bboxes\n    gc.collect()\n    scores_list=[x for x in scores]\n    labels_list=[x for x in classes]\n\n    nms_bboxes, nms_scores, nms_classes = weighted_boxes_fusion(\n        boxes_list=[boxes_list], \n        scores_list=[scores_list], \n        labels_list=[labels_list], \n        weights=None,\n        iou_thr=iou_th\n    )\n\n    # Convert nms_scores to a list of indices\n    indices = [i for i, s in enumerate(scores) if s in nms_scores]\n    \n    # Extract masks corresponding to the indices\n    nms_masks = [masks[i] for i in indices]\n    \n    # Stack the masks into an array and perform OR operation along the first axis\n    nms_masks = np.stack(nms_masks)\n    nms_masks = np.any(nms_masks, axis=0)\n\n    return nms_masks\n\n","metadata":{"execution":{"iopub.status.busy":"2023-08-05T07:15:00.362111Z","iopub.execute_input":"2023-08-05T07:15:00.362822Z","iopub.status.idle":"2023-08-05T07:15:00.371912Z","shell.execute_reply.started":"2023-08-05T07:15:00.362787Z","shell.execute_reply":"2023-08-05T07:15:00.370742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# models = [predictor_dit, llm_predictor]\n\n# submission_file = open(\"submission.csv\", \"w\")\n# submission_file.write(\"Id,Predicted\\n\")\n# results = []\n\n# for i, sample in enumerate(tqdm(dataset_dicts_test)):\n#     img = cv2.imread(sample[\"file_name\"])\n#     rles = ensemble(img, models)\n    \n#     result = [\n#                 f\"{sample['image_id']}_{cat},{rles[cat]}\\n\"\n#                 for cat in range(len(thing_classes))\n#             ]\n\n#     results.extend(result)\n    \n#     if i % 500 == 0:\n#         print(f\"Inference on batch {i}/{len(test_loader)} done\")\n#         submission_file.writelines(results)\n#         results = []\n# #     break\n# submission_file.writelines(results)\n# submission_file.close()","metadata":{"execution":{"iopub.status.busy":"2023-08-03T20:14:10.954951Z","iopub.execute_input":"2023-08-03T20:14:10.955327Z","iopub.status.idle":"2023-08-03T20:14:10.960433Z","shell.execute_reply.started":"2023-08-03T20:14:10.955295Z","shell.execute_reply":"2023-08-03T20:14:10.959238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision.transforms.functional as F\n\ndef ensemble(data, models):\n    with torch.no_grad():\n        encodings = {}\n        cache = {}\n        for i in range(len(data)):\n            cache[i] = {\n#                 'image' : None,\n                'classes' : [[] for _ in range(len(thing_classes))],\n                'scores' : [[] for _ in range(len(thing_classes))],\n                'bboxes' : [[] for _ in range(len(thing_classes))],\n                'masks' : [[] for _ in range(len(thing_classes))]\n            }\n            encodings[i] = [[] for _ in range(len(thing_classes))]\n        for i, model in enumerate(models):\n            outputs = model(data)\n            \n            for idx, output in enumerate(outputs):\n#                 cache[idx]['image'] = F.resize(data[idx]['image'], (data[idx]['height'], data[idx]['width']))\n                \n                output = output['instances']\n                take = output.scores.cpu().numpy() >= 0.6\n                pred_classes = output.pred_classes.cpu().numpy()[take]\n\n                for cat in range(len(thing_classes)):\n                    cache[idx]['classes'][cat].extend(output.pred_classes[take][pred_classes == cat].cpu().numpy().tolist())\n                    cache[idx]['scores'][cat].extend(output.scores[take][pred_classes == cat].cpu().numpy().tolist())\n                    cache[idx]['bboxes'][cat].extend(output.pred_boxes[take][pred_classes == cat].tensor.cpu().numpy().tolist())\n                    cache[idx]['masks'][cat].extend(output.pred_masks[take][pred_classes == cat].cpu().numpy())\n                \n            del outputs\n            gc.collect()\n#             time.sleep(0.25)\n#         print(\"--Stored--\")\n        for item in cache:\n#             image = cache[item]['image']\n            for cat in range(len(thing_classes)):\n#                 classes = cache[item]['classes'][cat]\n#                 scores = cache[item]['scores'][cat]\n#                 bboxes = cache[item]['bboxes'][cat]\n#                 masks = cache[item]['masks'][cat]\n#                 assert len(classes) == len(masks) , 'ensemble lenght mismatch'\n                if len(cache[item]['classes'][cat]) > 1:\n#                     try:\n#                         print(\"BEFORE\")\n#                         print(image.shape, masks[0].shape)\n#                         plt.figure()\n#                         plt.imshow(image.permute((1,2,0)))\n#                         plt.imshow(np.sum(masks, axis=0), alpha=0.3)\n#                         plt.show()\n\n                        encoded_masks = nms_predictions(\n                            cache[item]['classes'][cat], \n                            cache[item]['scores'][cat], \n                            cache[item]['bboxes'][cat],\n                            cache[item]['masks'][cat], shape=(cache[item]['masks'][cat][0].shape[0],\n                                                              cache[item]['masks'][cat][0].shape[1])\n                        )\n#                         print(encoded_masks.shape)\n#                         print(\"BEFORE2\")\n#                         print(image.shape, masks[0].shape)\n#                         plt.figure()\n#                         plt.imshow(image.permute((1,2,0)))\n#                         plt.imshow(np.sum(masks, axis=0), alpha=0.3)\n#                         plt.show()\n\n#                         encoded_masks = ensemble_pred_masks(masks, classes, min_pixels=[75, 75, 75, 75], shape=(cache[item]['masks'][cat][0].shape[0],\n#                                                                                                                 cache[item]['masks'][cat][0].shape[1]))\n\n#                         print(encoded_masks.shape)\n#                         print(\"AFTER\")\n#                         plt.figure()\n#                         plt.imshow(image.permute((1,2,0)))\n#                         plt.imshow(encoded_masks, alpha=0.5)\n#                         plt.show()\n\n                        encodings[item][cat].append(rle_encode(encoded_masks))\n                        del encoded_masks\n                        gc.collect()\n#                         time.sleep(0.25)\n#                     except:\n#                         print(\"Error\", len(cache[item]['masks'][cat]))\n#                         encodings[item][cat].append(\" \")\n                else:\n                    encodings[item][cat].append(\" \")\n                    \n#                 del masks\n#                 del scores\n#                 del encoded_masks\n#                 gc.collect()\n                \n        del cache\n        gc.collect()\n#         time.sleep(0.25)\n    return encodings\n    ","metadata":{"execution":{"iopub.status.busy":"2023-08-05T07:15:14.783195Z","iopub.execute_input":"2023-08-05T07:15:14.783600Z","iopub.status.idle":"2023-08-05T07:15:14.802694Z","shell.execute_reply.started":"2023-08-05T07:15:14.783564Z","shell.execute_reply":"2023-08-05T07:15:14.801659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()\n!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2023-08-05T07:15:18.285868Z","iopub.execute_input":"2023-08-05T07:15:18.287091Z","iopub.status.idle":"2023-08-05T07:15:19.358023Z","shell.execute_reply.started":"2023-08-05T07:15:18.287041Z","shell.execute_reply":"2023-08-05T07:15:19.356849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = [model_layout]\n\nsubmission_file = open(\"submission.csv\", \"w\")\nsubmission_file.write(\"Id,Predicted\\n\")\n# results = []\n\nfor i, data in enumerate(tqdm(test_loader)):\n    torch.cuda.empty_cache()\n    \n    rles = ensemble(data, models)\n    \n    for j in rles:\n        rle = rles[j]\n        result = [\n                    f\"{data[j]['image_id']}_{cat},{rle[cat][0]}\\n\"\n                    for cat in range(len(thing_classes))\n                ]\n#         results.extend(result)\n        submission_file.writelines(result)\n#     if i % (500 // BATCH) == 0:\n#         print(f\"Inference on batch {i}/{len(test_loader)} done\")\n#         submission_file.writelines(results)\n#         results = []\n        \n    del(rles)\n    gc.collect()\n#     time.sleep(0.25)\n#     break\n# submission_file.writelines(results)\nsubmission_file.close()","metadata":{"execution":{"iopub.status.busy":"2023-08-05T07:15:22.484697Z","iopub.execute_input":"2023-08-05T07:15:22.485918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}