{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%capture\nimport sys, os, distutils.core\n# Note: This is a faster way to install detectron2 in Colab, but it does not include all functionalities (e.g. compiled operators).\n# See https://detectron2.readthedocs.io/tutorials/install.html for full installation instructions\n!git clone 'https://github.com/facebookresearch/detectron2'\ndist = distutils.core.run_setup(\"./detectron2/setup.py\")\n!python -m pip install {' '.join([f\"'{x}'\" for x in dist.install_requires])}\nsys.path.insert(0, os.path.abspath('./detectron2'))\n","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:02:51.917718Z","iopub.execute_input":"2023-07-14T09:02:51.918073Z","iopub.status.idle":"2023-07-14T09:03:40.141006Z","shell.execute_reply.started":"2023-07-14T09:02:51.918044Z","shell.execute_reply":"2023-07-14T09:03:40.139629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import datetime\n\n# if False, model is set to `PRETRAINED_PATH` model\nis_train = True\n\n# if True, evaluate on validation dataset\nis_evaluate = False\n\n# if True, run inference on test dataset\nis_inference = True\n\n# if True and `is_train` == True, `PRETRAINED_PATH` model is trained further\nis_resume_training = True\n\n# Perform augmentation\nis_augment = False\n\nSEED = int(datetime.now().timestamp())","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:03:40.143666Z","iopub.execute_input":"2023-07-14T09:03:40.144865Z","iopub.status.idle":"2023-07-14T09:03:40.154403Z","shell.execute_reply.started":"2023-07-14T09:03:40.144827Z","shell.execute_reply":"2023-07-14T09:03:40.151943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install gdown\n!gdown 1IbxaRd82hIrxPT4a1U61_g2vvE3zcRLO","metadata":{"execution":{"iopub.status.busy":"2023-07-14T10:20:06.034689Z","iopub.execute_input":"2023-07-14T10:20:06.035108Z","iopub.status.idle":"2023-07-14T10:20:24.840665Z","shell.execute_reply.started":"2023-07-14T10:20:06.035076Z","shell.execute_reply":"2023-07-14T10:20:24.839415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!unzip /kaggle/working/dit-base-224-p16-500k-62d53a.zip","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:59:06.168543Z","iopub.execute_input":"2023-07-14T09:59:06.168933Z","iopub.status.idle":"2023-07-14T09:59:14.835925Z","shell.execute_reply.started":"2023-07-14T09:59:06.168901Z","shell.execute_reply":"2023-07-14T09:59:14.834693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\n\nTRAIN_IMG_DIR = Path(\"/kaggle/input/dlsprint2/badlad/images/train\")\n\nTRAIN_COCO_PATH = Path(\"/kaggle/input/dlsprint2/badlad/labels/coco_format/train/badlad-train-coco.json\")\n\nTEST_IMG_DIR = Path(\"/kaggle/input/dlsprint2/badlad/images/test\")\n\nTEST_METADATA_PATH = Path(\"/kaggle/input/dlsprint2/badlad/badlad-test-metadata.json\")\n\n# Training output directory\nOUTPUT_DIR = Path(\"./output1\")\nOUTPUT_MODEL = OUTPUT_DIR/\"model_final.pth\"\n\n# Path to your pretrained model weights\nPRETRAINED_PATH = Path(\"\")","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:54:07.704291Z","iopub.execute_input":"2023-07-14T09:54:07.705007Z","iopub.status.idle":"2023-07-14T09:54:07.710634Z","shell.execute_reply.started":"2023-07-14T09:54:07.704973Z","shell.execute_reply":"2023-07-14T09:54:07.709460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# detectron2\nfrom detectron2.utils.memory import retry_if_cuda_oom\nfrom detectron2.utils.logger import setup_logger\nfrom detectron2.checkpoint import DetectionCheckpointer\nfrom detectron2.modeling import build_model\nfrom detectron2.evaluation import COCOEvaluator, inference_on_dataset\nimport detectron2.data.transforms as T\nfrom detectron2.data import detection_utils as utils\nfrom detectron2.data import DatasetCatalog, MetadataCatalog, build_detection_test_loader, build_detection_train_loader, DatasetMapper\nfrom detectron2.utils.visualizer import Visualizer\nfrom detectron2.structures import BoxMode\nfrom detectron2.engine import DefaultPredictor, DefaultTrainer\nfrom detectron2.config import get_cfg\nfrom detectron2 import model_zoo\n\nimport pandas as pd\nimport numpy as np\nfrom tqdm.notebook import tqdm  # progress bar\nimport matplotlib.pyplot as plt\nimport json\nimport cv2\nimport copy\nfrom typing import Optional\n\nfrom IPython.display import FileLink\n\n# torch\nimport torch\n\nimport gc\n\nimport warnings\n# Ignore \"future\" warnings and Data-Frame-Slicing warnings.\nwarnings.filterwarnings('ignore')\n\nsetup_logger()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:54:08.666544Z","iopub.execute_input":"2023-07-14T09:54:08.666924Z","iopub.status.idle":"2023-07-14T09:54:08.679829Z","shell.execute_reply.started":"2023-07-14T09:54:08.666894Z","shell.execute_reply":"2023-07-14T09:54:08.678738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with TRAIN_COCO_PATH.open() as f:\n    train_dict = json.load(f)\n\nwith TEST_METADATA_PATH.open() as f:\n    test_dict = json.load(f)\n\nprint(\"#### LABELS AND METADATA LOADED ####\")","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:54:09.877351Z","iopub.execute_input":"2023-07-14T09:54:09.877748Z","iopub.status.idle":"2023-07-14T09:54:14.861861Z","shell.execute_reply.started":"2023-07-14T09:54:09.877718Z","shell.execute_reply":"2023-07-14T09:54:14.860792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def organize_coco_data(data_dict: dict) -> tuple[list[str], list[dict], list[dict]]:\n    thing_classes: list[str] = []\n\n    # Map Category Names to IDs\n    for cat in data_dict['categories']:\n        thing_classes.append(cat['name'])\n\n    # Images\n    images_metadata: list[dict] = data_dict['images']\n\n    # Convert COCO annotations to detectron2 annotations format\n    data_annotations = []\n    for ann in data_dict['annotations']:\n        # coco format -> detectron2 format\n        annot_obj = {\n            # Annotation ID\n            \"id\": ann['id'],\n\n            # Segmentation Polygon (x, y) coords\n            \"gt_masks\": ann['segmentation'],\n\n            # Image ID for this annotation (Which image does this annotation belong to?)\n            \"image_id\": ann['image_id'],\n\n            # Category Label (0: paragraph, 1: text box, 2: image, 3: table)\n            \"category_id\": ann['category_id'],\n\n            \"x_min\": ann['bbox'][0],  # left\n            \"y_min\": ann['bbox'][1],  # top\n            \"x_max\": ann['bbox'][0] + ann['bbox'][2],  # left+width\n            \"y_max\": ann['bbox'][1] + ann['bbox'][3]  # top+height\n        }\n        data_annotations.append(annot_obj)\n\n    return thing_classes, images_metadata, data_annotations","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:54:14.863798Z","iopub.execute_input":"2023-07-14T09:54:14.864353Z","iopub.status.idle":"2023-07-14T09:54:14.873817Z","shell.execute_reply.started":"2023-07-14T09:54:14.864316Z","shell.execute_reply":"2023-07-14T09:54:14.872696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"thing_classes, images_metadata, data_annotations = organize_coco_data(\n    train_dict\n)\n\nthing_classes_test, images_metadata_test, _ = organize_coco_data(\n    test_dict\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:03:49.428676Z","iopub.execute_input":"2023-07-14T09:03:49.429280Z","iopub.status.idle":"2023-07-14T09:03:50.752285Z","shell.execute_reply.started":"2023-07-14T09:03:49.429243Z","shell.execute_reply":"2023-07-14T09:03:50.751321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata = pd.DataFrame(images_metadata)\ntrain_metadata = train_metadata[['id', 'file_name', 'width', 'height']]\ntrain_metadata = train_metadata.rename(columns={\"id\": \"image_id\"})\nprint(\"train_metadata size=\", len(train_metadata))\ntrain_metadata.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:03:50.753898Z","iopub.execute_input":"2023-07-14T09:03:50.754459Z","iopub.status.idle":"2023-07-14T09:03:50.839769Z","shell.execute_reply.started":"2023-07-14T09:03:50.754425Z","shell.execute_reply":"2023-07-14T09:03:50.838732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_annot_df = pd.DataFrame(data_annotations)\nprint(\"train_annot_df size=\", len(train_annot_df))\ntrain_annot_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:03:50.841315Z","iopub.execute_input":"2023-07-14T09:03:50.841683Z","iopub.status.idle":"2023-07-14T09:03:52.251490Z","shell.execute_reply.started":"2023-07-14T09:03:50.841651Z","shell.execute_reply":"2023-07-14T09:03:52.250443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_metadata = pd.DataFrame(images_metadata_test)\ntest_metadata = test_metadata[['id', 'file_name', 'width', 'height']]\ntest_metadata = test_metadata.rename(columns={\"id\": \"image_id\"})\nprint(\"test_metadata size=\", len(test_metadata))\ntest_metadata.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:03:52.255773Z","iopub.execute_input":"2023-07-14T09:03:52.256062Z","iopub.status.idle":"2023-07-14T09:03:52.309198Z","shell.execute_reply.started":"2023-07-14T09:03:52.256036Z","shell.execute_reply":"2023-07-14T09:03:52.308359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_SPLIT = 0.95","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:03:52.310552Z","iopub.execute_input":"2023-07-14T09:03:52.310898Z","iopub.status.idle":"2023-07-14T09:03:52.315512Z","shell.execute_reply.started":"2023-07-14T09:03:52.310873Z","shell.execute_reply":"2023-07-14T09:03:52.314509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_dataset = len(train_metadata)\nn_train = int(n_dataset * TRAIN_SPLIT)\nprint(\"n_dataset\", n_dataset, \"n_train\", n_train, \"n_val\", n_dataset-n_train)\n\nnp.random.seed(SEED)\n\ninds = np.random.permutation(n_dataset)\ntrain_inds, valid_inds = inds[:n_train], inds[n_train:]","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:03:52.317123Z","iopub.execute_input":"2023-07-14T09:03:52.317830Z","iopub.status.idle":"2023-07-14T09:03:52.327138Z","shell.execute_reply.started":"2023-07-14T09:03:52.317797Z","shell.execute_reply":"2023-07-14T09:03:52.326128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_coco_to_detectron2_format(\n    imgdir: Path,\n    metadata_df: pd.DataFrame,\n    annot_df: Optional[pd.DataFrame] = None,\n    target_indices: Optional[np.ndarray] = None,\n):\n\n    dataset_dicts = []\n    for _, train_meta_row in tqdm(metadata_df.iterrows(), total=len(metadata_df)):\n        # Iterate over each image\n        image_id, filename, width, height = train_meta_row.values\n\n        annotations = []\n\n        # If train/validation data, then there will be annotations\n        if annot_df is not None:\n            for _, ann in annot_df.query(\"image_id == @image_id\").iterrows():\n                # Get annotations of current iteration's image\n                class_id = ann[\"category_id\"]\n                gt_masks = ann[\"gt_masks\"]\n                bbox_resized = [\n                    float(ann[\"x_min\"]),\n                    float(ann[\"y_min\"]),\n                    float(ann[\"x_max\"]),\n                    float(ann[\"y_max\"]),\n                ]\n\n                annotation = {\n                    \"bbox\": bbox_resized,\n                    \"bbox_mode\": BoxMode.XYXY_ABS,\n                    \"segmentation\": gt_masks,\n                    \"category_id\": class_id,\n                }\n\n                annotations.append(annotation)\n\n        # coco format -> detectron2 format dict\n        record = {\n            \"file_name\": str(imgdir/filename),\n            \"image_id\": image_id,\n            \"width\": width,\n            \"height\": height,\n            \"annotations\": annotations\n        }\n\n        dataset_dicts.append(record)\n\n    if target_indices is not None:\n        dataset_dicts = [dataset_dicts[i] for i in target_indices]\n\n    return dataset_dicts","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:03:52.328695Z","iopub.execute_input":"2023-07-14T09:03:52.329329Z","iopub.status.idle":"2023-07-14T09:03:52.340112Z","shell.execute_reply.started":"2023-07-14T09:03:52.329297Z","shell.execute_reply":"2023-07-14T09:03:52.339079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DatasetCatalog.clear()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:03:52.341631Z","iopub.execute_input":"2023-07-14T09:03:52.342065Z","iopub.status.idle":"2023-07-14T09:03:52.353782Z","shell.execute_reply.started":"2023-07-14T09:03:52.342024Z","shell.execute_reply":"2023-07-14T09:03:52.352925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_REGISTER_TRAINING = \"my_dataset_train\"\nDATA_REGISTER_VALID    = \"my_dataset_val\"\nDATA_REGISTER_TEST     = \"badlad_test\"","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:03:52.355032Z","iopub.execute_input":"2023-07-14T09:03:52.355466Z","iopub.status.idle":"2023-07-14T09:03:52.363603Z","shell.execute_reply.started":"2023-07-14T09:03:52.355431Z","shell.execute_reply":"2023-07-14T09:03:52.362618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Register Training data\nDatasetCatalog.register(\n    DATA_REGISTER_TRAINING,\n    lambda: convert_coco_to_detectron2_format(\n        TRAIN_IMG_DIR,\n        train_metadata,\n        train_annot_df,\n        target_indices=train_inds,\n    ),\n)\n\n# Set Training data categories\nMetadataCatalog.get(DATA_REGISTER_TRAINING).set(thing_classes=thing_classes)\n\ndataset_dicts_train = DatasetCatalog.get(DATA_REGISTER_TRAINING)\ntrain_metadata1 = MetadataCatalog.get(DATA_REGISTER_TRAINING)\n\nprint(\"dicts training size=\", len(dataset_dicts_train))\nprint(\"################\")","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:03:52.365055Z","iopub.execute_input":"2023-07-14T09:03:52.365476Z","iopub.status.idle":"2023-07-14T09:05:28.772746Z","shell.execute_reply.started":"2023-07-14T09:03:52.365445Z","shell.execute_reply":"2023-07-14T09:05:28.771731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Register Test Inference data\nDatasetCatalog.register(\n    DATA_REGISTER_TEST,\n    lambda: convert_coco_to_detectron2_format(\n        TEST_IMG_DIR,\n        test_metadata,\n    )\n)\n\n# Set Test data categories\nMetadataCatalog.get(DATA_REGISTER_TEST).set(\n    thing_classes=thing_classes_test\n)\n\ndataset_dicts_test = DatasetCatalog.get(DATA_REGISTER_TEST)\nmetadata_dicts_test = MetadataCatalog.get(DATA_REGISTER_TEST)\n\n# print(\"dicts valid size=\", len(dataset_dicts_test))\n# print(\"################\")","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:05:28.774488Z","iopub.execute_input":"2023-07-14T09:05:28.775147Z","iopub.status.idle":"2023-07-14T09:05:29.569735Z","shell.execute_reply.started":"2023-07-14T09:05:28.775109Z","shell.execute_reply":"2023-07-14T09:05:29.568734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Register Validation data\nDatasetCatalog.register(\n    DATA_REGISTER_VALID,\n    lambda: convert_coco_to_detectron2_format(\n        TRAIN_IMG_DIR,\n        train_metadata,\n        train_annot_df,\n        target_indices=valid_inds,\n    ),\n)\n\n# Set Validation data categories\nMetadataCatalog.get(DATA_REGISTER_VALID).set(thing_classes=thing_classes)\n\ndataset_dicts_valid = DatasetCatalog.get(DATA_REGISTER_VALID)\nmetadata_dicts_valid = MetadataCatalog.get(DATA_REGISTER_VALID)\n\nprint(\"dicts valid size=\", len(dataset_dicts_valid))\nprint(\"################\")","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:05:29.571077Z","iopub.execute_input":"2023-07-14T09:05:29.572035Z","iopub.status.idle":"2023-07-14T09:07:05.956495Z","shell.execute_reply.started":"2023-07-14T09:05:29.571999Z","shell.execute_reply":"2023-07-14T09:07:05.955452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def custom_mapper(dataset_dict):\n    dataset_dict = copy.deepcopy(dataset_dict)\n    image = utils.read_image(dataset_dict[\"file_name\"], format=\"BGR\")\n\n    transform_list = [T.RandomBrightness(0.8, 1.2),\n                      T.RandomFlip(prob=0.5, horizontal=False, vertical=True),\n                      T.RandomFlip(prob=0.5, horizontal=True, vertical=False),\n                      CopyPaste(blend=True, sigma=1, pct_objects_paste=0.8, p=1.)\n                      ]\n    image, transforms = T.apply_transform_gens(transform_list, image)\n\n    dataset_dict[\"image\"] = torch.as_tensor(\n        image.transpose(2, 0, 1).astype(\"float32\"))\n\n    annos = [\n        utils.transform_instance_annotations(obj, transforms, image.shape[:2])\n        for obj in dataset_dict.pop(\"annotations\")\n        if obj.get(\"iscrowd\", 0) == 0\n    ]\n    instances = utils.annotations_to_instances(annos, image.shape[:2])\n\n    dataset_dict[\"instances\"] = utils.filter_empty_instances(instances)\n\n    return dataset_dict","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:07:05.958033Z","iopub.execute_input":"2023-07-14T09:07:05.958980Z","iopub.status.idle":"2023-07-14T09:07:05.968742Z","shell.execute_reply.started":"2023-07-14T09:07:05.958943Z","shell.execute_reply":"2023-07-14T09:07:05.967708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AugTrainer(DefaultTrainer):\n    @classmethod\n    def build_train_loader(cls, cfg):\n        return build_detection_train_loader(cfg, mapper=custom_mapper)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:07:05.969978Z","iopub.execute_input":"2023-07-14T09:07:05.970405Z","iopub.status.idle":"2023-07-14T09:07:05.985328Z","shell.execute_reply.started":"2023-07-14T09:07:05.970351Z","shell.execute_reply":"2023-07-14T09:07:05.984446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! git clone https://github.com/microsoft/unilm.git \n","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:07:05.987236Z","iopub.execute_input":"2023-07-14T09:07:05.988251Z","iopub.status.idle":"2023-07-14T09:07:11.158242Z","shell.execute_reply.started":"2023-07-14T09:07:05.988217Z","shell.execute_reply":"2023-07-14T09:07:11.157046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sys.path.insert(1, \"/kaggle/working/unilm/dit/object_detection\")\n","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:07:11.160794Z","iopub.execute_input":"2023-07-14T09:07:11.162010Z","iopub.status.idle":"2023-07-14T09:07:11.167363Z","shell.execute_reply.started":"2023-07-14T09:07:11.161971Z","shell.execute_reply":"2023-07-14T09:07:11.166355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! sed -i 's/from collections import Iterable/from collections.abc import Iterable/' /kaggle/working/unilm/dit/object_detection/ditod/table_evaluation/data_structure.py","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:07:11.168826Z","iopub.execute_input":"2023-07-14T09:07:11.169310Z","iopub.status.idle":"2023-07-14T09:07:12.152380Z","shell.execute_reply.started":"2023-07-14T09:07:11.169279Z","shell.execute_reply":"2023-07-14T09:07:12.150807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from ditod import add_vit_config\nfrom detectron2.config import CfgNode as CN","metadata":{"execution":{"iopub.status.busy":"2023-07-14T09:21:32.636344Z","iopub.execute_input":"2023-07-14T09:21:32.637334Z","iopub.status.idle":"2023-07-14T09:21:32.643050Z","shell.execute_reply.started":"2023-07-14T09:21:32.637286Z","shell.execute_reply":"2023-07-14T09:21:32.641672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"  if is_train:  \n    add_vit_config(cfg)\n    cfg.merge_from_file(\"/kaggle/working/unilm/dit/object_detection/publaynet_configs/maskrcnn/maskrcnn_dit_base.yaml\")\n    cfg.MODEL.DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    cfg.MODEL.WEIGHTS = \"/kaggle/working/model_final.pth\"\n    cfg.SOLVER.AMP.ENABLED = True\n    BATCH_SIZE=1\n    NOMINAL_BATCH_SIZE=2\n\n    cfg.SOLVER.WARMUP_ITERS = 5\n    \n    \n#     if (is_resume_training):\n#         print(\"#### SETTING PRETRAINED WEIGHTS TO RESUME TRAINING ####\")\n#         cfg.MODEL.WEIGHTS = str(PRETRAINED_PATH)\n#     else:\n#         print(\"#### TRAINING MODEL FROM SCRATCH ####\")\n\n    # Maximum number of iterations\n    cfg.SOLVER.MAX_ITER = 5000\n\n    # cfg.SOLVER.STEPS = (500, 1000) # must be less than MAX_ITER\n\n    cfg.SOLVER.GAMMA = 0.05\n\n    # Small value == Frequent save need a lot of storage.\n    #cfg.SOLVER.CHECKPOINT_PERIOD = 500\n    cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128\n    cfg.MODEL.ROI_HEADS.NUM_CLASSES = 4\n    cfg.MODEL.ROI_HEADS.NUM_CLASSES = 4\n    cfg.SOLVER.BASE_LR = 0.001 * BATCH_SIZE/NOMINAL_BATCH_SIZE\n    cfg.SOLVER.IMS_PER_BATCH = BATCH_SIZE\n    cfg.SOLVER.WARMUP_ITERS = 5\n    cfg.DATALOADER.NUM_WORKERS = 2\n    cfg.SOLVER.MAX_ITER = 20000 # Set higher value, like 10000 for better results\n    cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128\n    cfg.SOLVER.CHECKPOINT_PERIOD = 500\n    cfg.DATASETS.TRAIN = (DATA_REGISTER_TRAINING,)\n#     cfg.DATASETS.TEST = ()\n    # Create Output Directory\n    cfg.OUTPUT_DIR = str(OUTPUT_DIR)\n    print(\"creating cfg.OUTPUT_DIR -> \", cfg.OUTPUT_DIR)\n    OUTPUT_DIR.mkdir(exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T10:28:02.034475Z","iopub.execute_input":"2023-07-14T10:28:02.034849Z","iopub.status.idle":"2023-07-14T10:28:02.072213Z","shell.execute_reply.started":"2023-07-14T10:28:02.034820Z","shell.execute_reply":"2023-07-14T10:28:02.071293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from ditod import MyTrainer\n\n# os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)\ntrainer = MyTrainer(cfg) \ntrainer.resume_or_load(resume=True)\n# Uncomment to train\ntrainer.train()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T10:28:03.075939Z","iopub.execute_input":"2023-07-14T10:28:03.076323Z","iopub.status.idle":"2023-07-14T10:36:13.487029Z","shell.execute_reply.started":"2023-07-14T10:28:03.076290Z","shell.execute_reply":"2023-07-14T10:36:13.485275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metrics_df = pd.read_json(\n        \"/kaggle/working/output1/metrics.json\", orient=\"records\", lines=True)\nmdf = metrics_df.sort_values(\"iteration\")\nprint(mdf.head(10).T)\n\n# Plot loss\nfig, ax = plt.subplots()\n\nmdf1 = mdf[~mdf[\"total_loss\"].isna()]\nax.plot(mdf1[\"iteration\"], mdf1[\"total_loss\"], c=\"C0\", label=\"train\")\n\nif \"validation_loss\" in mdf.columns:\n    mdf2 = mdf[~mdf[\"validation_loss\"].isna()]\n    ax.plot(mdf2[\"iteration\"], mdf2[\"validation_loss\"],\n            c=\"C1\", label=\"validation\")\n\nax.legend()\nax.set_title(\"Loss curve\")\nplt.show()\n\n# Plot Accuracy\nfig, ax = plt.subplots()\n\nmdf1 = mdf[~mdf[\"fast_rcnn/cls_accuracy\"].isna()]\nax.plot(mdf1[\"iteration\"], mdf1[\"fast_rcnn/cls_accuracy\"],\n        c=\"C0\", label=\"train\")\n\nax.legend()\nax.set_title(\"Accuracy curve\")\nplt.show()\n\n# Plot Bounding Box regressor loss\nfig, ax = plt.subplots()\n\nmdf1 = mdf[~mdf[\"loss_box_reg\"].isna()]\nax.plot(mdf1[\"iteration\"], mdf1[\"loss_box_reg\"], c=\"C0\", label=\"train\")\n\nax.legend()\nax.set_title(\"loss_box_reg\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T10:45:51.343215Z","iopub.execute_input":"2023-07-14T10:45:51.344186Z","iopub.status.idle":"2023-07-14T10:45:52.168905Z","shell.execute_reply.started":"2023-07-14T10:45:51.344150Z","shell.execute_reply":"2023-07-14T10:45:52.167929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_PATH=Path(\"/kaggle/working/output1/model_final.pth\")","metadata":{"execution":{"iopub.status.busy":"2023-07-14T10:48:50.126835Z","iopub.execute_input":"2023-07-14T10:48:50.127215Z","iopub.status.idle":"2023-07-14T10:48:50.134696Z","shell.execute_reply.started":"2023-07-14T10:48:50.127182Z","shell.execute_reply":"2023-07-14T10:48:50.133728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inf_cfg = get_cfg()\n\n# config_name = \"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"\n\n# inf_cfg.merge_from_file(model_zoo.get_config_file(config_name))\nadd_vit_config(inf_cfg)\n    # Load a config from file\ninf_cfg.merge_from_file(\"/kaggle/working/unilm/dit/object_detection/publaynet_configs/maskrcnn/maskrcnn_dit_base.yaml\")\ninf_cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128\ninf_cfg.MODEL.ROI_HEADS.NUM_CLASSES = 4\ninf_cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5\ninf_cfg.MODEL.DEVICE = \"cuda\"\n\ninf_cfg.DATALOADER.NUM_WORKERS = 1  # lower this if CUDA overflow occurs\ninf_cfg.MODEL.WEIGHTS = str(MODEL_PATH)\nBATCH = 1 # lower this if CUDA overflow occurs\ntest_loader = build_detection_test_loader(inf_cfg, DATA_REGISTER_TEST, batch_size=BATCH)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T10:48:51.555228Z","iopub.execute_input":"2023-07-14T10:48:51.555614Z","iopub.status.idle":"2023-07-14T10:48:52.769441Z","shell.execute_reply.started":"2023-07-14T10:48:51.555583Z","shell.execute_reply":"2023-07-14T10:48:52.768357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ACCEPTANCE_THRESHOLD = 0.6  # for all categories","metadata":{"execution":{"iopub.status.busy":"2023-07-14T10:48:53.545194Z","iopub.execute_input":"2023-07-14T10:48:53.545923Z","iopub.status.idle":"2023-07-14T10:48:53.550440Z","shell.execute_reply.started":"2023-07-14T10:48:53.545887Z","shell.execute_reply":"2023-07-14T10:48:53.549504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"#### MODEL: {inf_cfg.MODEL.WEIGHTS} FOR INFERENCE ####\")","metadata":{"execution":{"iopub.status.busy":"2023-07-14T10:48:54.737005Z","iopub.execute_input":"2023-07-14T10:48:54.737655Z","iopub.status.idle":"2023-07-14T10:48:54.743459Z","shell.execute_reply.started":"2023-07-14T10:48:54.737621Z","shell.execute_reply":"2023-07-14T10:48:54.742265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictor = DefaultPredictor(inf_cfg)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-14T10:48:56.471006Z","iopub.execute_input":"2023-07-14T10:48:56.471396Z","iopub.status.idle":"2023-07-14T10:48:59.011033Z","shell.execute_reply.started":"2023-07-14T10:48:56.471352Z","shell.execute_reply":"2023-07-14T10:48:59.010027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport cv2\nimport matplotlib.pyplot as plt\nfrom detectron2.utils.visualizer import Visualizer\n\nfig, ax = plt.subplots(4, 2, figsize=(20, 40))  # Updated subplot dimensions\nindices = [ax[i // 2][i % 2] for i in range(8)]  # Adjusted indices for 8 plots\n\n# Show some qualitative results by predicting on test set images\nNUM_TEST_SAMPLES = 8  # Updated number of test samples\nsamples = np.random.choice(dataset_dicts_test, NUM_TEST_SAMPLES)\n\nfor i, sample in enumerate(samples):\n    img = cv2.imread(sample[\"file_name\"])\n    outputs = predictor(img)\n    visualizer = Visualizer(img, metadata=metadata_dicts_test, scale=0.5)\n    visualizer = visualizer.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\n    display_img = visualizer.get_image()[:, :, ::-1]\n    indices[i].grid(False)\n    indices[i].imshow(display_img)\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T10:49:49.073861Z","iopub.execute_input":"2023-07-14T10:49:49.074233Z","iopub.status.idle":"2023-07-14T10:50:07.979796Z","shell.execute_reply.started":"2023-07-14T10:49:49.074201Z","shell.execute_reply":"2023-07-14T10:50:07.978483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}