{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2021-07-28T09:16:23.029969Z","iopub.execute_input":"2021-07-28T09:16:23.030337Z","iopub.status.idle":"2021-07-28T09:16:23.703829Z","shell.execute_reply.started":"2021-07-28T09:16:23.030299Z","shell.execute_reply":"2021-07-28T09:16:23.702838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvcc --version","metadata":{"execution":{"iopub.status.busy":"2021-07-28T09:16:23.706362Z","iopub.execute_input":"2021-07-28T09:16:23.706716Z","iopub.status.idle":"2021-07-28T09:16:24.347115Z","shell.execute_reply.started":"2021-07-28T09:16:23.706679Z","shell.execute_reply":"2021-07-28T09:16:24.346212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install detectron2 -f https://dl.fbaipublicfiles.com/detectron2/wheels/cu102/torch1.9/index.html","metadata":{"execution":{"iopub.status.busy":"2021-07-28T09:16:24.350461Z","iopub.execute_input":"2021-07-28T09:16:24.350735Z","iopub.status.idle":"2021-07-28T09:16:30.912489Z","shell.execute_reply.started":"2021-07-28T09:16:24.350706Z","shell.execute_reply":"2021-07-28T09:16:30.911521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check pytorch installation: \n!pip install torch==1.9.0\n!pip install torchvision==0.10.0\nimport torch, torchvision","metadata":{"execution":{"iopub.status.busy":"2021-07-28T09:16:30.915776Z","iopub.execute_input":"2021-07-28T09:16:30.916046Z","iopub.status.idle":"2021-07-28T09:16:44.202709Z","shell.execute_reply.started":"2021-07-28T09:16:30.916017Z","shell.execute_reply":"2021-07-28T09:16:44.201772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(torch.__version__, torchvision.__version__, torch.cuda.is_available())","metadata":{"execution":{"iopub.status.busy":"2021-07-28T09:17:57.500651Z","iopub.execute_input":"2021-07-28T09:17:57.500997Z","iopub.status.idle":"2021-07-28T09:17:57.529921Z","shell.execute_reply.started":"2021-07-28T09:17:57.500964Z","shell.execute_reply":"2021-07-28T09:17:57.528712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert torch.__version__.startswith(\"1.9\")   ","metadata":{"execution":{"iopub.status.busy":"2021-07-28T09:18:04.834168Z","iopub.execute_input":"2021-07-28T09:18:04.834639Z","iopub.status.idle":"2021-07-28T09:18:04.843794Z","shell.execute_reply.started":"2021-07-28T09:18:04.834596Z","shell.execute_reply":"2021-07-28T09:18:04.842532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Some basic setup:\n# Setup detectron2 logger\nimport detectron2\nfrom detectron2.utils.logger import setup_logger\nsetup_logger()\n\n# import some common libraries\nimport numpy as np\nimport os, json, cv2, random\n\n# import some common detectron2 utilities\nfrom detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor\nfrom detectron2.config import get_cfg\nfrom detectron2.utils.visualizer import Visualizer\nfrom detectron2.data import MetadataCatalog, DatasetCatalog\nfrom matplotlib import pyplot as plt\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2021-07-28T09:18:05.139504Z","iopub.execute_input":"2021-07-28T09:18:05.139872Z","iopub.status.idle":"2021-07-28T09:18:05.503124Z","shell.execute_reply.started":"2021-07-28T09:18:05.139843Z","shell.execute_reply":"2021-07-28T09:18:05.502256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!wget http://images.cocodataset.org/val2017/000000439715.jpg -q -O input.jpg\nim = cv2.imread(\"./input.jpg\")\nplt.imshow(im)","metadata":{"execution":{"iopub.status.busy":"2021-07-28T09:18:05.504686Z","iopub.execute_input":"2021-07-28T09:18:05.504995Z","iopub.status.idle":"2021-07-28T09:18:06.970466Z","shell.execute_reply.started":"2021-07-28T09:18:05.504962Z","shell.execute_reply":"2021-07-28T09:18:06.969471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg = get_cfg()\ncfg.merge_from_file(model_zoo.get_config_file(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"))\ncfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5  # set threshold for this model\ncfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\")\npredictor = DefaultPredictor(cfg)\noutputs = predictor(im)","metadata":{"execution":{"iopub.status.busy":"2021-07-28T09:18:56.771116Z","iopub.execute_input":"2021-07-28T09:18:56.771511Z","iopub.status.idle":"2021-07-28T09:18:57.793462Z","shell.execute_reply.started":"2021-07-28T09:18:56.771475Z","shell.execute_reply":"2021-07-28T09:18:57.792634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(outputs[\"instances\"].pred_classes)\nprint(outputs[\"instances\"].pred_boxes)","metadata":{"execution":{"iopub.status.busy":"2021-07-28T09:19:40.995782Z","iopub.execute_input":"2021-07-28T09:19:40.996109Z","iopub.status.idle":"2021-07-28T09:19:41.005763Z","shell.execute_reply.started":"2021-07-28T09:19:40.996070Z","shell.execute_reply":"2021-07-28T09:19:41.004718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"v = Visualizer(im[:, :, ::-1], MetadataCatalog.get(cfg.DATASETS.TRAIN[0]), scale=1.2)\nout = v.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\nplt.imshow(out.get_image()[:, :, ::-1])","metadata":{"execution":{"iopub.status.busy":"2021-07-28T09:20:08.340000Z","iopub.execute_input":"2021-07-28T09:20:08.340387Z","iopub.status.idle":"2021-07-28T09:20:08.740660Z","shell.execute_reply.started":"2021-07-28T09:20:08.340331Z","shell.execute_reply":"2021-07-28T09:20:08.739852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!wget https://github.com/matterport/Mask_RCNN/releases/download/v2.1/balloon_dataset.zip\n!unzip balloon_dataset.zip > /dev/null","metadata":{"execution":{"iopub.status.busy":"2021-07-28T09:20:45.418752Z","iopub.execute_input":"2021-07-28T09:20:45.419082Z","iopub.status.idle":"2021-07-28T09:20:48.439392Z","shell.execute_reply.started":"2021-07-28T09:20:45.419051Z","shell.execute_reply":"2021-07-28T09:20:48.438340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if your dataset is in COCO format, this cell can be replaced by the following three lines:\n# from detectron2.data.datasets import register_coco_instances\n# register_coco_instances(\"my_dataset_train\", {}, \"json_annotation_train.json\", \"path/to/image/dir\")\n# register_coco_instances(\"my_dataset_val\", {}, \"json_annotation_val.json\", \"path/to/image/dir\")\n\nfrom detectron2.structures import BoxMode\n\ndef get_balloon_dicts(img_dir):\n    json_file = os.path.join(img_dir, \"via_region_data.json\")\n    with open(json_file) as f:\n        imgs_anns = json.load(f)\n\n    dataset_dicts = []\n    for idx, v in enumerate(imgs_anns.values()):\n        record = {}\n        \n        filename = os.path.join(img_dir, v[\"filename\"])\n        height, width = cv2.imread(filename).shape[:2]\n        \n        record[\"file_name\"] = filename\n        record[\"image_id\"] = idx\n        record[\"height\"] = height\n        record[\"width\"] = width\n      \n        annos = v[\"regions\"]\n        objs = []\n        for _, anno in annos.items():\n            assert not anno[\"region_attributes\"]\n            anno = anno[\"shape_attributes\"]\n            px = anno[\"all_points_x\"]\n            py = anno[\"all_points_y\"]\n            poly = [(x + 0.5, y + 0.5) for x, y in zip(px, py)]\n            poly = [p for x in poly for p in x]\n\n            obj = {\n                \"bbox\": [np.min(px), np.min(py), np.max(px), np.max(py)],\n                \"bbox_mode\": BoxMode.XYXY_ABS,\n                \"segmentation\": [poly],\n                \"category_id\": 0,\n            }\n            objs.append(obj)\n        record[\"annotations\"] = objs\n        dataset_dicts.append(record)\n    return dataset_dicts\n\nfor d in [\"train\", \"val\"]:\n    DatasetCatalog.register(\"balloon_\" + d, lambda d=d: get_balloon_dicts(\"balloon/\" + d))\n    MetadataCatalog.get(\"balloon_\" + d).set(thing_classes=[\"balloon\"])\nballoon_metadata = MetadataCatalog.get(\"balloon_train\")","metadata":{"execution":{"iopub.status.busy":"2021-07-28T09:21:53.251323Z","iopub.execute_input":"2021-07-28T09:21:53.251711Z","iopub.status.idle":"2021-07-28T09:21:53.265958Z","shell.execute_reply.started":"2021-07-28T09:21:53.251674Z","shell.execute_reply":"2021-07-28T09:21:53.264583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_dicts = get_balloon_dicts(\"balloon/train\")\nfor d in random.sample(dataset_dicts, 3):\n    img = cv2.imread(d[\"file_name\"])\n    visualizer = Visualizer(img[:, :, ::-1], metadata=balloon_metadata, scale=0.5)\n    out = visualizer.draw_dataset_dict(d)\n    plt.imshow(out.get_image()[:, :, ::-1])","metadata":{"execution":{"iopub.status.busy":"2021-07-28T09:22:22.283472Z","iopub.execute_input":"2021-07-28T09:22:22.283807Z","iopub.status.idle":"2021-07-28T09:22:24.573998Z","shell.execute_reply.started":"2021-07-28T09:22:22.283777Z","shell.execute_reply":"2021-07-28T09:22:24.573218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from detectron2.engine import DefaultTrainer\n\ncfg = get_cfg()\ncfg.merge_from_file(model_zoo.get_config_file(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"))\ncfg.DATASETS.TRAIN = (\"balloon_train\",)\ncfg.DATASETS.TEST = ()\ncfg.DATALOADER.NUM_WORKERS = 2\ncfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\")  # Let training initialize from model zoo\ncfg.SOLVER.IMS_PER_BATCH = 2\ncfg.SOLVER.BASE_LR = 0.00025  # pick a good LR\ncfg.SOLVER.MAX_ITER = 300    # 300 iterations seems good enough for this toy dataset; you will need to train longer for a practical dataset\ncfg.SOLVER.STEPS = []        # do not decay learning rate\ncfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128   # faster, and good enough for this toy dataset (default: 512)\ncfg.MODEL.ROI_HEADS.NUM_CLASSES = 1  # only has one class (ballon). (see https://detectron2.readthedocs.io/tutorials/datasets.html#update-the-config-for-new-datasets)\n# NOTE: this config means the number of classes, but a few popular unofficial tutorials incorrect uses num_classes+1 here.\n\nos.makedirs(cfg.OUTPUT_DIR, exist_ok=True)\ntrainer = DefaultTrainer(cfg) \ntrainer.resume_or_load(resume=False)\ntrainer.train()","metadata":{"execution":{"iopub.status.busy":"2021-07-28T09:22:54.029548Z","iopub.execute_input":"2021-07-28T09:22:54.029879Z","iopub.status.idle":"2021-07-28T09:24:38.958973Z","shell.execute_reply.started":"2021-07-28T09:22:54.029850Z","shell.execute_reply":"2021-07-28T09:24:38.957943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Look at training curves in tensorboard:\n%load_ext tensorboard\n%tensorboard --logdir output","metadata":{"execution":{"iopub.status.busy":"2021-07-28T09:24:55.099183Z","iopub.execute_input":"2021-07-28T09:24:55.099570Z","iopub.status.idle":"2021-07-28T09:24:55.120855Z","shell.execute_reply.started":"2021-07-28T09:24:55.099531Z","shell.execute_reply":"2021-07-28T09:24:55.120038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Inference should use the config with parameters that are used in training\n# cfg now already contains everything we've set previously. We changed it a little bit for inference:\ncfg.MODEL.WEIGHTS = os.path.join(cfg.OUTPUT_DIR, \"model_final.pth\")  # path to the model we just trained\ncfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.7   # set a custom testing threshold\npredictor = DefaultPredictor(cfg)","metadata":{"execution":{"iopub.status.busy":"2021-07-28T09:25:49.384496Z","iopub.execute_input":"2021-07-28T09:25:49.384870Z","iopub.status.idle":"2021-07-28T09:25:50.207588Z","shell.execute_reply.started":"2021-07-28T09:25:49.384837Z","shell.execute_reply":"2021-07-28T09:25:50.206692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from detectron2.utils.visualizer import ColorMode\ndataset_dicts = get_balloon_dicts(\"balloon/val\")\nfor d in random.sample(dataset_dicts, 3):    \n    im = cv2.imread(d[\"file_name\"])\n    outputs = predictor(im)  # format is documented at https://detectron2.readthedocs.io/tutorials/models.html#model-output-format\n    v = Visualizer(im[:, :, ::-1],\n                   metadata=balloon_metadata, \n                   scale=0.5, \n                   instance_mode=ColorMode.IMAGE_BW   # remove the colors of unsegmented pixels. This option is only available for segmentation models\n    )\n    out = v.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\n    plt.imshow(out.get_image()[:, :, ::-1])","metadata":{"execution":{"iopub.status.busy":"2021-07-28T09:26:22.086058Z","iopub.execute_input":"2021-07-28T09:26:22.086421Z","iopub.status.idle":"2021-07-28T09:26:24.130542Z","shell.execute_reply.started":"2021-07-28T09:26:22.086389Z","shell.execute_reply":"2021-07-28T09:26:24.129336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from detectron2.evaluation import COCOEvaluator, inference_on_dataset\nfrom detectron2.data import build_detection_test_loader\nevaluator = COCOEvaluator(\"balloon_val\", cfg, False, output_dir=\"./output/\")\nval_loader = build_detection_test_loader(cfg, \"balloon_val\")\nprint(inference_on_dataset(trainer.model, val_loader, evaluator))\n# another equivalent way to evaluate the model is to use `trainer.test`","metadata":{"execution":{"iopub.status.busy":"2021-07-28T09:40:10.228100Z","iopub.execute_input":"2021-07-28T09:40:10.228476Z","iopub.status.idle":"2021-07-28T09:40:14.793304Z","shell.execute_reply.started":"2021-07-28T09:40:10.228435Z","shell.execute_reply":"2021-07-28T09:40:14.792243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Inference with a keypoint detection model\ncfg = get_cfg()   # get a fresh new config\ncfg.merge_from_file(model_zoo.get_config_file(\"COCO-Keypoints/keypoint_rcnn_R_50_FPN_3x.yaml\"))\ncfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.7  # set threshold for this model\ncfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-Keypoints/keypoint_rcnn_R_50_FPN_3x.yaml\")\npredictor = DefaultPredictor(cfg)\noutputs = predictor(im)\nv = Visualizer(im[:,:,::-1], MetadataCatalog.get(cfg.DATASETS.TRAIN[0]), scale=1.2)\nout = v.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\nplt.imshow(out.get_image()[:, :, ::-1])","metadata":{"execution":{"iopub.status.busy":"2021-07-28T09:41:06.368703Z","iopub.execute_input":"2021-07-28T09:41:06.369066Z","iopub.status.idle":"2021-07-28T09:41:08.620273Z","shell.execute_reply.started":"2021-07-28T09:41:06.369034Z","shell.execute_reply":"2021-07-28T09:41:08.619246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Inference with a panoptic segmentation model\ncfg = get_cfg()\ncfg.merge_from_file(model_zoo.get_config_file(\"COCO-PanopticSegmentation/panoptic_fpn_R_101_3x.yaml\"))\ncfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-PanopticSegmentation/panoptic_fpn_R_101_3x.yaml\")\npredictor = DefaultPredictor(cfg)\npanoptic_seg, segments_info = predictor(im)[\"panoptic_seg\"]\nv = Visualizer(im[:, :, ::-1], MetadataCatalog.get(cfg.DATASETS.TRAIN[0]), scale=1.2)\nout = v.draw_panoptic_seg_predictions(panoptic_seg.to(\"cpu\"), segments_info)\nplt.imshow(out.get_image()[:, :, ::-1])","metadata":{"execution":{"iopub.status.busy":"2021-07-28T09:41:45.769538Z","iopub.execute_input":"2021-07-28T09:41:45.769891Z","iopub.status.idle":"2021-07-28T09:41:48.447798Z","shell.execute_reply.started":"2021-07-28T09:41:45.769860Z","shell.execute_reply":"2021-07-28T09:41:48.447000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}