{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## fork from https://www.kaggle.com/code/shinya7y/great-barrier-reef-cascade-r-cnn-private-0-694\n\n## Reference\n\n* https://www.kaggle.com/mlneo07/mmdetection-swin-transfomer-frcnn-inference-0-443\n* https://www.kaggle.com/c/tensorflow-great-barrier-reef/overview/evaluation\n* https://www.kaggle.com/kocha1/only-yolov5-tracking-lb-642/notebook\n* https://www.kaggle.com/parapapapam/yolox-inference-tracking-on-cots-lb-0-539","metadata":{"_uuid":"6096cbd2-2cbe-447e-9963-29db740511e0","_cell_guid":"28f9e49b-4d3b-4147-bfc9-d461270c4b73","trusted":true}},{"cell_type":"markdown","source":"## Installation","metadata":{}},{"cell_type":"code","source":"# !ls /kaggle/input/\n# !ls /kaggle/input/universenet-for-offline/","metadata":{"_uuid":"088e3838-eec5-472c-b3aa-9e6936cadb9c","_cell_guid":"df139d5c-8bd0-4218-adcc-93552ea618a0","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-02-19T13:38:16.715248Z","iopub.execute_input":"2022-02-19T13:38:16.71581Z","iopub.status.idle":"2022-02-19T13:38:16.73379Z","shell.execute_reply.started":"2022-02-19T13:38:16.715732Z","shell.execute_reply":"2022-02-19T13:38:16.7332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# install norfair and its dependencies\n!pip install '/kaggle/input/norfair031py3/commonmark-0.9.1-py2.py3-none-any.whl' -f ./ --no-index\n!pip install '/kaggle/input/norfair031py3/rich-9.13.0-py3-none-any.whl' --no-deps\n!cp -r /kaggle/input/norfair031py3/filterpy-1.4.5/filterpy-1.4.5 /kaggle/working/\n!pip install '/kaggle/working/filterpy-1.4.5'\n!rm -rf '/kaggle/working/filterpy-1.4.5'\n!pip install '/kaggle/input/norfair031py3/norfair-0.3.1-py3-none-any.whl' -f ./ --no-index --no-deps","metadata":{"execution":{"iopub.status.busy":"2022-03-10T12:04:37.731667Z","iopub.execute_input":"2022-03-10T12:04:37.732139Z","iopub.status.idle":"2022-03-10T12:05:40.57906Z","shell.execute_reply.started":"2022-03-10T12:04:37.732058Z","shell.execute_reply":"2022-03-10T12:05:40.578212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# downgrade pytorch to avoid compatibility issue with mmcv whl\n!pip install '/kaggle/input/pytorch-190/torch-1.9.0+cu111-cp37-cp37m-linux_x86_64.whl'","metadata":{"execution":{"iopub.status.busy":"2022-03-10T12:05:40.580992Z","iopub.execute_input":"2022-03-10T12:05:40.581395Z","iopub.status.idle":"2022-03-10T12:07:19.78554Z","shell.execute_reply.started":"2022-03-10T12:05:40.581341Z","shell.execute_reply":"2022-03-10T12:07:19.784532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# install mmdet requirements\n!pip install '/kaggle/input/universenet-for-offline/addict-2.4.0-py3-none-any.whl' --no-deps\n!pip install '/kaggle/input/universenet-for-offline/yapf-0.32.0-py2.py3-none-any.whl' --no-deps\n!pip install '/kaggle/input/universenet-for-offline/terminaltables-3.1.10-py2.py3-none-any.whl' --no-deps\n!pip install '/kaggle/input/universenet-for-offline/mmcv_full-1.4.4-cp37-cp37m-manylinux1_x86_64.whl' --no-deps\n!cp -r '/kaggle/input/universenet-for-offline/pycocotools-2.0.2/pycocotools-2.0.2' /kaggle/working/\n!pip install '/kaggle/working/pycocotools-2.0.2' --no-deps\n!rm -rf '/kaggle/working/pycocotools-2.0.2'","metadata":{"_uuid":"edffbf60-b8b3-4ce2-9367-a9dc1197bffe","_cell_guid":"1784bc72-420e-446c-b24a-7b67f3cb7fd1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-03-10T12:07:19.787289Z","iopub.execute_input":"2022-03-10T12:07:19.787601Z","iopub.status.idle":"2022-03-10T12:09:21.669945Z","shell.execute_reply.started":"2022-03-10T12:07:19.787559Z","shell.execute_reply":"2022-03-10T12:09:21.66881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# install UniverseNet\n!cp -r '/kaggle/input/universenet-for-offline/UniverseNet-master' /kaggle/working/UniverseNet\n!pip install -e '/kaggle/working/UniverseNet'","metadata":{"_uuid":"6e03f4b1-3d9a-4d2b-933a-30058b38187c","_cell_guid":"f98d1ad2-a550-492c-8c2f-01b5084aae74","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-03-10T12:09:21.673653Z","iopub.execute_input":"2022-03-10T12:09:21.673875Z","iopub.status.idle":"2022-03-10T12:10:03.097232Z","shell.execute_reply.started":"2022-03-10T12:09:21.673847Z","shell.execute_reply":"2022-03-10T12:10:03.096424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Helper","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom norfair import Detection, Tracker\n\ndef to_norfair(detects, frame_id):\n    \"\"\"Convert bbox format from xyxy+score to norfair.Detection class.\"\"\"\n    result = []\n    for x_min, y_min, x_max, y_max, score in detects:\n        xc, yc = (x_min + x_max) / 2, (y_min + y_max) / 2\n        w, h = x_max - x_min, y_max - y_min\n        result.append(\n            Detection(\n                points=np.array([xc, yc]),\n                scores=np.array([score]),\n                data=np.array([w, h, frame_id])))\n    return result\n\ndef euclidean_distance(detection, tracked_object):\n    \"\"\"Euclidean distance function for norfair tracker.\"\"\"\n    # match detections on this frame with tracked_objects from previous frames\n    return np.linalg.norm(detection.points - tracked_object.estimate)","metadata":{"execution":{"iopub.status.busy":"2022-03-10T12:10:03.099881Z","iopub.execute_input":"2022-03-10T12:10:03.100166Z","iopub.status.idle":"2022-03-10T12:10:04.197564Z","shell.execute_reply.started":"2022-03-10T12:10:03.100125Z","shell.execute_reply":"2022-03-10T12:10:04.196827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def result_to_gbr_str(result):\n    \"\"\"Convert mmdet result to GBR competition format.\"\"\"\n    result = result[0].copy()  # cots class only\n    bboxes = result[:, :4]\n    scores = result[:, 4]\n    # xyxy2xywh\n    bboxes[:, 2] = bboxes[:, 2] - bboxes[:, 0]  # width\n    bboxes[:, 3] = bboxes[:, 3] - bboxes[:, 1]  # height\n\n    pred_strings = []\n    for box, score in zip(bboxes, scores):\n        x_min, y_min, bbox_width, bbox_height = box\n        pred_strings.append(f'{score:.4f} {x_min} {y_min} {bbox_width} {bbox_height}')\n    gbr_str = ' '.join(pred_strings)\n    return gbr_str\n\ndef result_to_gbr_str_with_tracker(result, tracker, frame_id):\n    \"\"\"Convert mmdet result and tracked result to GBR competition format.\n\n    TODO: refactor to split tracking\n    \"\"\"\n    result = result[0].copy()  # cots class only\n    bboxes = result[:, :4]\n    scores = result[:, 4]\n    # xyxy2xywh\n    bboxes[:, 2] = bboxes[:, 2] - bboxes[:, 0]  # width\n    bboxes[:, 3] = bboxes[:, 3] - bboxes[:, 1]  # height\n\n    detects = []\n    pred_strings = []\n\n    for box, score in zip(bboxes, scores):\n        x_min, y_min, bbox_width, bbox_height = box\n        detects.append([x_min, y_min, x_min+bbox_width, y_min+bbox_height, score])\n        pred_strings.append(f'{score:.4f} {x_min} {y_min} {bbox_width} {bbox_height}')\n    # Update tracks using detects from current frame\n    tracked_objects = tracker.update(detections=to_norfair(detects, frame_id))\n\n    for tobj in tracked_objects:\n        bbox_width, bbox_height, last_detected_frame_id = tobj.last_detection.data\n        if last_detected_frame_id == frame_id:  # Skip objects that were detected on current frame\n            continue\n        # Add objects that have no detections on current frame to predictions\n        xc, yc = tobj.estimate[0]\n        x_min = xc - bbox_width / 2\n        y_min = yc - bbox_height / 2\n        score = tobj.last_detection.scores[0]\n        pred_strings.append(f'{score:.4f} {x_min} {y_min} {bbox_width} {bbox_height}')\n\n    gbr_str = ' '.join(pred_strings)\n    return gbr_str","metadata":{"execution":{"iopub.status.busy":"2022-03-10T12:10:04.200608Z","iopub.execute_input":"2022-03-10T12:10:04.200797Z","iopub.status.idle":"2022-03-10T12:10:04.214133Z","shell.execute_reply.started":"2022-03-10T12:10:04.200773Z","shell.execute_reply":"2022-03-10T12:10:04.213142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.append('./UniverseNet')\nsys.path.append('../input/tensorflow-great-barrier-reef/greatbarrierreef')\nimport cv2\nimport torch\nfrom mmcv import Config\nfrom mmdet.apis import init_detector, inference_detector, show_result_pyplot\nimport greatbarrierreef","metadata":{"_uuid":"53709ac7-2f71-49c5-be1f-c422b478b12d","_cell_guid":"887cc1dc-056b-44c7-91e0-22353270ee71","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-03-10T12:10:04.215375Z","iopub.execute_input":"2022-03-10T12:10:04.216499Z","iopub.status.idle":"2022-03-10T12:10:23.435133Z","shell.execute_reply.started":"2022-03-10T12:10:04.216462Z","shell.execute_reply":"2022-03-10T12:10:23.434375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# call env.iter_test() before loading model to reduce risk of out of memory\nenv = greatbarrierreef.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2022-03-10T12:10:23.436607Z","iopub.execute_input":"2022-03-10T12:10:23.436854Z","iopub.status.idle":"2022-03-10T12:10:23.440919Z","shell.execute_reply.started":"2022-03-10T12:10:23.436821Z","shell.execute_reply":"2022-03-10T12:10:23.440162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#######################################################################################################\ndef prepare_mmdet_cots_model():\n    device = 'cuda:0' if torch.cuda.is_available() else 'cpu'\n    checkpoint = '../input/gbr-starfish/cascade_rcnn_r2_101_fpn_fp16_4x2_mixup_affine_hsv_1440_lr002_7e_cots_alltrain_1600pafpn.pth'\n    config_path = '../input/gbr-starfish/cascade_rcnn_r2_101_fpn_fp16_4x2_mixup_affine_hsv_1440_lr002_7e_cots_alltrain_1600pafpn.py'\n    config = Config.fromfile(config_path)\n\n    # TODO check settings\n    config.model.test_cfg.rcnn.score_thr = 0.3\n    print(config.model.test_cfg.rcnn)\n    config.data.test.pipeline[1].img_scale = (2560, 1440)\n    config.data.test.pipeline[1].flip = True\n    print(config.data.test.pipeline[1])\n    # TODO check speed and accuracy with fp16 on P100\n    # config.fp16 = dict(loss_scale=dict(init_scale=512))\n    config.fp16 = None\n\n    model = init_detector(config, checkpoint, device=device)\n    return model\n\nmodel = prepare_mmdet_cots_model()\n#######################################################################################################","metadata":{"_uuid":"e421beaf-94ff-4c2c-9938-52209b0c1154","_cell_guid":"5523eae4-0a7b-41e5-829b-0b64882b223c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-03-10T12:10:23.442373Z","iopub.execute_input":"2022-03-10T12:10:23.442786Z","iopub.status.idle":"2022-03-10T12:10:34.333544Z","shell.execute_reply.started":"2022-03-10T12:10:23.442749Z","shell.execute_reply":"2022-03-10T12:10:34.332594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def infer_gbr_train(model):\n    \"\"\"Infer GBR train data.\"\"\"\n    tracker = Tracker(\n        distance_function=euclidean_distance, \n        distance_threshold=30,\n        hit_inertia_min=2,\n        hit_inertia_max=4,\n        initialization_delay=1)\n\n    image_paths = ['/kaggle/input/tensorflow-great-barrier-reef/train_images/video_1/9114.jpg']\n    image_paths += ['/kaggle/input/tensorflow-great-barrier-reef/train_images/video_2/5766.jpg']\n#     image_paths += ['/kaggle/working/UniverseNet/demo/demo.jpg']\n    for image_index, image_path in enumerate(image_paths):\n        frame_id = image_index\n        image_bgr = cv2.imread(image_path)\n        result = inference_detector(model, image_bgr)\n        gbr_str = result_to_gbr_str_with_tracker(result, tracker, frame_id)\n        print(gbr_str)\n        show_result_pyplot(model, image_bgr, result, score_thr=0., palette=(255, 111, 0))\n\ninfer_gbr_train(model)","metadata":{"_uuid":"0e0d3dc6-83db-4149-9119-a54387bbac39","_cell_guid":"fcfcdc39-be51-4875-a199-51b6985e1a3a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-03-10T12:10:34.334656Z","iopub.execute_input":"2022-03-10T12:10:34.334896Z","iopub.status.idle":"2022-03-10T12:10:38.612618Z","shell.execute_reply.started":"2022-03-10T12:10:34.334863Z","shell.execute_reply":"2022-03-10T12:10:38.609436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def infer_gbr_test(model):\n    \"\"\"Infer GBR test data using time-series API.\"\"\"\n    tracker = Tracker(\n        distance_function=euclidean_distance, \n        distance_threshold=30,\n        hit_inertia_min=2,\n        hit_inertia_max=4,\n        initialization_delay=1)\n\n    for image_index, (image_rgb, pred_df) in enumerate(iter_test):\n        frame_id = image_index\n        image_bgr = image_rgb[:, :, ::-1]\n        result = inference_detector(model, image_bgr)\n        gbr_str = result_to_gbr_str_with_tracker(result, tracker, frame_id)\n        pred_df['annotations'] = gbr_str\n        env.predict(pred_df)\n        if image_index < 3:\n            print(gbr_str)\n            show_result_pyplot(model, image_bgr, result, score_thr=0., palette=(255, 111, 0))\n\ninfer_gbr_test(model)","metadata":{"_uuid":"cab8db51-eb20-4dcd-a663-924e6de3c833","_cell_guid":"71e7e19b-f15a-4ea8-a1f1-9e20d7f7167b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-03-10T12:10:38.613831Z","iopub.execute_input":"2022-03-10T12:10:38.615517Z","iopub.status.idle":"2022-03-10T12:10:44.968763Z","shell.execute_reply.started":"2022-03-10T12:10:38.615475Z","shell.execute_reply":"2022-03-10T12:10:44.968134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf '/kaggle/working/UniverseNet'","metadata":{"_uuid":"19576c61-5a02-415d-ac9d-5a1fe4dae360","_cell_guid":"ed2465c9-5759-49b2-b76a-2eb34fa289b8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-03-10T12:10:44.970054Z","iopub.execute_input":"2022-03-10T12:10:44.970481Z","iopub.status.idle":"2022-03-10T12:10:45.755847Z","shell.execute_reply.started":"2022-03-10T12:10:44.970443Z","shell.execute_reply":"2022-03-10T12:10:45.754889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}