{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1 Install detectron2","metadata":{}},{"cell_type":"code","source":"!python -m pip install 'git+https://github.com/facebookresearch/detectron2.git'","metadata":{"_kg_hide-input":false,"_kg_hide-output":false,"execution":{"iopub.status.busy":"2023-07-30T08:45:50.595298Z","iopub.execute_input":"2023-07-30T08:45:50.595738Z","iopub.status.idle":"2023-07-30T08:49:30.313530Z","shell.execute_reply.started":"2023-07-30T08:45:50.595695Z","shell.execute_reply":"2023-07-30T08:49:30.312035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2 Imports","metadata":{}},{"cell_type":"code","source":"# detectron2\nfrom detectron2.utils.memory import retry_if_cuda_oom\nfrom detectron2.utils.logger import setup_logger\nfrom detectron2.checkpoint import DetectionCheckpointer\nfrom detectron2.modeling import build_model\nfrom detectron2.evaluation import COCOEvaluator, inference_on_dataset\nimport detectron2.data.transforms as T\nfrom detectron2.data import detection_utils as utils\nfrom detectron2.data import DatasetCatalog, MetadataCatalog, build_detection_test_loader, build_detection_train_loader, DatasetMapper\nfrom detectron2.utils.visualizer import Visualizer\nfrom detectron2.structures import BoxMode\nfrom detectron2.engine import DefaultPredictor, DefaultTrainer\nfrom detectron2.config import get_cfg\nfrom detectron2 import model_zoo\n\nimport pandas as pd\nimport numpy as np\nfrom tqdm.notebook import tqdm  # progress bar\nimport matplotlib.pyplot as plt\nimport json\nimport cv2\nimport copy\nfrom typing import Optional\n\nfrom IPython.display import FileLink\n\n# torch\nimport torch\n\nimport gc\n\nimport warnings\n# Ignore \"future\" warnings and Data-Frame-Slicing warnings.\nwarnings.filterwarnings('ignore')\n\nsetup_logger()","metadata":{"execution":{"iopub.status.busy":"2023-07-30T08:49:30.316013Z","iopub.execute_input":"2023-07-30T08:49:30.316587Z","iopub.status.idle":"2023-07-30T08:49:32.722904Z","shell.execute_reply.started":"2023-07-30T08:49:30.316545Z","shell.execute_reply":"2023-07-30T08:49:32.721073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3 Paths","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\n\nTRAIN_IMG_DIR = Path(\"/kaggle/input/dlsprint2/badlad/images/train\")\n\nTRAIN_COCO_PATH = Path(\"/kaggle/input/dlsprint2/badlad/labels/coco_format/train/badlad-train-coco.json\")\n\n# Training output directory\nOUTPUT_DIR = Path(\"./output\")\nOUTPUT_MODEL = OUTPUT_DIR/\"model_final.pth\"\n\n# Path to your pretrained model weights\nPRETRAINED_PATH = Path(\"/kaggle/input/badlad-m-rcnn-5k/output/model_final.pth\")","metadata":{"execution":{"iopub.status.busy":"2023-07-30T08:49:32.724454Z","iopub.execute_input":"2023-07-30T08:49:32.725194Z","iopub.status.idle":"2023-07-30T08:49:32.731797Z","shell.execute_reply.started":"2023-07-30T08:49:32.725158Z","shell.execute_reply":"2023-07-30T08:49:32.730407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4 COCO Annotations Data","metadata":{}},{"cell_type":"code","source":"with TRAIN_COCO_PATH.open() as f:\n    train_dict = json.load(f)\n\nprint(\"#### LABELS AND METADATA LOADED ####\")","metadata":{"execution":{"iopub.status.busy":"2023-07-30T08:49:32.736033Z","iopub.execute_input":"2023-07-30T08:49:32.736694Z","iopub.status.idle":"2023-07-30T08:49:39.181056Z","shell.execute_reply.started":"2023-07-30T08:49:32.736661Z","shell.execute_reply":"2023-07-30T08:49:39.179873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def organize_coco_data(data_dict: dict) -> tuple[list[str], list[dict], list[dict]]:\n    thing_classes: list[str] = []\n\n    # Map Category Names to IDs\n    for cat in data_dict['categories']:\n        thing_classes.append(cat['name'])\n\n    # Images\n    images_metadata: list[dict] = data_dict['images']\n\n    # Convert COCO annotations to detectron2 annotations format\n    data_annotations = []\n    for ann in data_dict['annotations']:\n        # coco format -> detectron2 format\n        annot_obj = {\n            # Annotation ID\n            \"id\": ann['id'],\n\n            # Segmentation Polygon (x, y) coords\n            \"gt_masks\": ann['segmentation'],\n\n            # Image ID for this annotation (Which image does this annotation belong to?)\n            \"image_id\": ann['image_id'],\n\n            # Category Label (0: paragraph, 1: text box, 2: image, 3: table)\n            \"category_id\": ann['category_id'],\n\n            \"x_min\": ann['bbox'][0],  # left\n            \"y_min\": ann['bbox'][1],  # top\n            \"x_max\": ann['bbox'][0] + ann['bbox'][2],  # left+width\n            \"y_max\": ann['bbox'][1] + ann['bbox'][3]  # top+height\n        }\n        data_annotations.append(annot_obj)\n\n    return thing_classes, images_metadata, data_annotations","metadata":{"execution":{"iopub.status.busy":"2023-07-30T08:49:39.182794Z","iopub.execute_input":"2023-07-30T08:49:39.183189Z","iopub.status.idle":"2023-07-30T08:49:39.194246Z","shell.execute_reply.started":"2023-07-30T08:49:39.183154Z","shell.execute_reply":"2023-07-30T08:49:39.191500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"thing_classes, images_metadata, data_annotations = organize_coco_data(\n    train_dict\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T08:49:39.195558Z","iopub.execute_input":"2023-07-30T08:49:39.196374Z","iopub.status.idle":"2023-07-30T08:49:40.474385Z","shell.execute_reply.started":"2023-07-30T08:49:39.196341Z","shell.execute_reply":"2023-07-30T08:49:40.473276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata = pd.DataFrame(images_metadata)\ntrain_metadata = train_metadata[['id', 'file_name', 'width', 'height']]\ntrain_metadata = train_metadata.rename(columns={\"id\": \"image_id\"})\nprint(\"train_metadata size=\", len(train_metadata))\ntrain_metadata.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T08:49:40.475767Z","iopub.execute_input":"2023-07-30T08:49:40.478031Z","iopub.status.idle":"2023-07-30T08:49:40.568564Z","shell.execute_reply.started":"2023-07-30T08:49:40.477984Z","shell.execute_reply":"2023-07-30T08:49:40.567628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_annot_df = pd.DataFrame(data_annotations)\nprint(\"train_annot_df size=\", len(train_annot_df))\ntrain_annot_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T08:49:40.570184Z","iopub.execute_input":"2023-07-30T08:49:40.570818Z","iopub.status.idle":"2023-07-30T08:49:42.042165Z","shell.execute_reply.started":"2023-07-30T08:49:40.570782Z","shell.execute_reply":"2023-07-30T08:49:42.041247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here `gt_masks` are the sequence of `(x, y)` coordinates of vertices of the polygon surrounding the target object. ","metadata":{}},{"cell_type":"markdown","source":"# 5 Split","metadata":{}},{"cell_type":"code","source":"TRAIN_SPLIT = 0.0","metadata":{"execution":{"iopub.status.busy":"2023-07-30T08:49:42.045142Z","iopub.execute_input":"2023-07-30T08:49:42.045826Z","iopub.status.idle":"2023-07-30T08:49:42.050821Z","shell.execute_reply.started":"2023-07-30T08:49:42.045792Z","shell.execute_reply":"2023-07-30T08:49:42.049802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 42\n\nn_dataset = len(train_metadata)\nn_train = int(n_dataset * TRAIN_SPLIT)\nprint(\"n_dataset\", n_dataset, \"n_train\", n_train, \"n_val\", n_dataset-n_train)\n\nnp.random.seed(SEED)\n\ninds = np.random.permutation(n_dataset)\ntrain_inds, valid_inds = inds[:n_train], inds[n_train:]","metadata":{"execution":{"iopub.status.busy":"2023-07-30T08:49:42.055443Z","iopub.execute_input":"2023-07-30T08:49:42.055766Z","iopub.status.idle":"2023-07-30T08:49:42.064843Z","shell.execute_reply.started":"2023-07-30T08:49:42.055740Z","shell.execute_reply":"2023-07-30T08:49:42.063573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6 Preparing Data","metadata":{}},{"cell_type":"code","source":"def convert_coco_to_detectron2_format(\n    imgdir: Path,\n    metadata_df: pd.DataFrame,\n    annot_df: Optional[pd.DataFrame] = None,\n    target_indices: Optional[np.ndarray] = None,\n):\n\n    dataset_dicts = []\n    for _, train_meta_row in tqdm(metadata_df.iterrows(), total=len(metadata_df)):\n        # Iterate over each image\n        image_id, filename, width, height = train_meta_row.values\n\n        annotations = []\n\n        # If train/validation data, then there will be annotations\n        if annot_df is not None:\n            for _, ann in annot_df.query(\"image_id == @image_id\").iterrows():\n                # Get annotations of current iteration's image\n                class_id = ann[\"category_id\"]\n                gt_masks = ann[\"gt_masks\"]\n                bbox_resized = [\n                    float(ann[\"x_min\"]),\n                    float(ann[\"y_min\"]),\n                    float(ann[\"x_max\"]),\n                    float(ann[\"y_max\"]),\n                ]\n\n                annotation = {\n                    \"bbox\": bbox_resized,\n                    \"bbox_mode\": BoxMode.XYXY_ABS,\n                    \"segmentation\": gt_masks,\n                    \"category_id\": class_id,\n                }\n\n                annotations.append(annotation)\n\n        # coco format -> detectron2 format dict\n        record = {\n            \"file_name\": str(imgdir/filename),\n            \"image_id\": image_id,\n            \"width\": width,\n            \"height\": height,\n            \"annotations\": annotations\n        }\n\n        dataset_dicts.append(record)\n\n    if target_indices is not None:\n        dataset_dicts = [dataset_dicts[i] for i in target_indices]\n\n    return dataset_dicts","metadata":{"execution":{"iopub.status.busy":"2023-07-30T08:49:42.066390Z","iopub.execute_input":"2023-07-30T08:49:42.066880Z","iopub.status.idle":"2023-07-30T08:49:42.080577Z","shell.execute_reply.started":"2023-07-30T08:49:42.066843Z","shell.execute_reply":"2023-07-30T08:49:42.079495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_REGISTER_VALID    = \"badlad_valid\"","metadata":{"execution":{"iopub.status.busy":"2023-07-30T08:49:42.082041Z","iopub.execute_input":"2023-07-30T08:49:42.082547Z","iopub.status.idle":"2023-07-30T08:49:42.092125Z","shell.execute_reply.started":"2023-07-30T08:49:42.082511Z","shell.execute_reply":"2023-07-30T08:49:42.090835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Register Validation data\nDatasetCatalog.register(\n    DATA_REGISTER_VALID,\n    lambda: convert_coco_to_detectron2_format(\n        TRAIN_IMG_DIR,\n        train_metadata,\n        train_annot_df,\n        target_indices=valid_inds,\n    ),\n)\n\n# Set Validation data categories\nMetadataCatalog.get(DATA_REGISTER_VALID).set(thing_classes=thing_classes)\n\ndataset_dicts_valid = DatasetCatalog.get(DATA_REGISTER_VALID)\nmetadata_dicts_valid = MetadataCatalog.get(DATA_REGISTER_VALID)\n\nprint(\"dicts valid size=\", len(dataset_dicts_valid))\nprint(\"################\")","metadata":{"execution":{"iopub.status.busy":"2023-07-30T08:49:42.093715Z","iopub.execute_input":"2023-07-30T08:49:42.094230Z","iopub.status.idle":"2023-07-30T08:51:26.708505Z","shell.execute_reply.started":"2023-07-30T08:49:42.094189Z","shell.execute_reply":"2023-07-30T08:51:26.707465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"#### DATA REGISTERED ####\")","metadata":{"execution":{"iopub.status.busy":"2023-07-30T08:51:26.710017Z","iopub.execute_input":"2023-07-30T08:51:26.711086Z","iopub.status.idle":"2023-07-30T08:51:26.716202Z","shell.execute_reply.started":"2023-07-30T08:51:26.711046Z","shell.execute_reply":"2023-07-30T08:51:26.715280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 7 Hyperparameters","metadata":{}},{"cell_type":"markdown","source":"Detectron2 models need a config file to build the model. This config file has the associated hyperparameters.\n\nYou can play with this: https://detectron2.readthedocs.io/en/latest/modules/config.html#yaml-config-references","metadata":{}},{"cell_type":"code","source":"cfg = get_cfg()\n\n# config_name = \"COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml\"\nconfig_name = \"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"\n\ncfg.merge_from_file(model_zoo.get_config_file(config_name))\n\ncfg.DATASETS.TEST = (DATA_REGISTER_VALID,)\n\ncfg.DATALOADER.NUM_WORKERS = 2\n\n# cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(config_name)\ncfg.MODEL.WEIGHTS = str(PRETRAINED_PATH)\n\ncfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128\ncfg.MODEL.ROI_HEADS.NUM_CLASSES = 4\n\ncfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5   # set a custom testing threshold\ncfg.SOLVER.IMS_PER_BATCH = 64\n\n# Create Output Directory\ncfg.OUTPUT_DIR = str(OUTPUT_DIR)\nprint(\"creating cfg.OUTPUT_DIR -> \", cfg.OUTPUT_DIR)\nOUTPUT_DIR.mkdir(exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T08:51:26.717441Z","iopub.execute_input":"2023-07-30T08:51:26.718426Z","iopub.status.idle":"2023-07-30T08:51:26.750136Z","shell.execute_reply.started":"2023-07-30T08:51:26.718391Z","shell.execute_reply":"2023-07-30T08:51:26.748888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 8 Evaluation","metadata":{}},{"cell_type":"code","source":"def build_model_from_cfg(cfg):\n    model = build_model(cfg)\n    _ = DetectionCheckpointer(model).load(cfg.MODEL.WEIGHTS)\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-07-30T08:51:26.751838Z","iopub.execute_input":"2023-07-30T08:51:26.752204Z","iopub.status.idle":"2023-07-30T08:51:26.760647Z","shell.execute_reply.started":"2023-07-30T08:51:26.752171Z","shell.execute_reply":"2023-07-30T08:51:26.759717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"### EVALUATING ON VALIDATION DATA ####\")\n\n# trained model weights\nevaluation_model = build_model_from_cfg(cfg)\n\nevaluator = COCOEvaluator(\n    DATA_REGISTER_VALID, cfg, False, output_dir=cfg.OUTPUT_DIR, use_fast_impl=True\n)\n\nval_loader = build_detection_test_loader(cfg, DATA_REGISTER_VALID)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T08:51:26.762089Z","iopub.execute_input":"2023-07-30T08:51:26.762487Z","iopub.status.idle":"2023-07-30T08:53:21.391436Z","shell.execute_reply.started":"2023-07-30T08:51:26.762456Z","shell.execute_reply":"2023-07-30T08:53:21.390107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = inference_on_dataset(\n    evaluation_model, val_loader, evaluator=evaluator\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T08:53:21.393231Z","iopub.execute_input":"2023-07-30T08:53:21.394012Z","iopub.status.idle":"2023-07-30T09:01:32.980270Z","shell.execute_reply.started":"2023-07-30T08:53:21.393954Z","shell.execute_reply":"2023-07-30T09:01:32.979134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(results)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T09:01:57.875877Z","iopub.execute_input":"2023-07-30T09:01:57.876287Z","iopub.status.idle":"2023-07-30T09:01:57.882216Z","shell.execute_reply.started":"2023-07-30T09:01:57.876252Z","shell.execute_reply":"2023-07-30T09:01:57.880814Z"},"trusted":true},"execution_count":null,"outputs":[]}]}