{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install ../input/timm-0-4-12-py3-none-any-whl/timm-0.4.12-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2022-02-14T06:49:47.33204Z","iopub.execute_input":"2022-02-14T06:49:47.332343Z","iopub.status.idle":"2022-02-14T06:50:16.094324Z","shell.execute_reply.started":"2022-02-14T06:49:47.332255Z","shell.execute_reply":"2022-02-14T06:50:16.093503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport albumentations as A\nimport torch\nimport torch.nn.functional as F\nfrom tqdm import tqdm\nimport sys\nimport math\nimport cv2\nimport time\nimport timm\nfrom albumentations.augmentations.geometric.functional import bbox_rot90\nimport matplotlib.pyplot as plt\nfrom torchvision.ops import box_iou\n\n%matplotlib inline\n\nsys.path.append('../input/tensorflow-great-barrier-reef')\nsys.path.append('../input/yolov5-lib-ds')\nsys.path.append('/kaggle/input/weightedboxesfusion/')\n\nfrom ensemble_boxes import weighted_boxes_fusion","metadata":{"execution":{"iopub.status.busy":"2022-02-14T06:50:16.097541Z","iopub.execute_input":"2022-02-14T06:50:16.097805Z","iopub.status.idle":"2022-02-14T06:50:21.119879Z","shell.execute_reply.started":"2022-02-14T06:50:16.097761Z","shell.execute_reply":"2022-02-14T06:50:21.119097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /root/.config/Ultralytics\n!cp /kaggle/input/yolov5-font/Arial.ttf /root/.config/Ultralytics/","metadata":{"execution":{"iopub.status.busy":"2022-02-14T06:50:21.121519Z","iopub.execute_input":"2022-02-14T06:50:21.121806Z","iopub.status.idle":"2022-02-14T06:50:22.452255Z","shell.execute_reply.started":"2022-02-14T06:50:21.121756Z","shell.execute_reply":"2022-02-14T06:50:22.451261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import greatbarrierreef\nenv = greatbarrierreef.make_env()# initialize the environment\niter_test = env.iter_test()      # an iterator which loops over the test set and sample submission","metadata":{"execution":{"iopub.status.busy":"2022-02-14T06:50:22.45458Z","iopub.execute_input":"2022-02-14T06:50:22.45483Z","iopub.status.idle":"2022-02-14T06:50:22.478588Z","shell.execute_reply.started":"2022-02-14T06:50:22.454795Z","shell.execute_reply":"2022-02-14T06:50:22.477904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# hyp\n\n# attention\nATT_ENABLED = True\nATT_ANCHOR_SCORE = 0.15\nATT_BOOST_SCORE = 0.1\nATT_SPLITOR_THRESHOLD = 4\n\n# predict\nFINAL_THRESHOLD = 0.08","metadata":{"execution":{"iopub.status.busy":"2022-02-14T06:50:22.481414Z","iopub.execute_input":"2022-02-14T06:50:22.481605Z","iopub.status.idle":"2022-02-14T06:50:22.486449Z","shell.execute_reply.started":"2022-02-14T06:50:22.481581Z","shell.execute_reply":"2022-02-14T06:50:22.48574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"splitor = timm.create_model(\"efficientnet_b0\", num_classes=1)\nsplitor.load_state_dict(torch.load(\"../input/patric-seq-splitor/splitor.pt\"))\nsplitor.eval()\nsplitor.cuda()\nprint(\"fine\")","metadata":{"execution":{"iopub.status.busy":"2022-02-14T06:50:22.487858Z","iopub.execute_input":"2022-02-14T06:50:22.488098Z","iopub.status.idle":"2022-02-14T06:50:25.897635Z","shell.execute_reply.started":"2022-02-14T06:50:22.488065Z","shell.execute_reply":"2022-02-14T06:50:25.896872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_scaler(pivot, mode, strenth=2.5, max_weight=None):\n    if max_weight is None:\n        max_weight = strenth\n\n    def _func(boxes):\n        areas = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])\n        for i, (area, box) in enumerate(zip(areas, boxes)):\n            # boxes[i, 4] *= np.exp(1. - thr / area)\n            if mode == \"desc\":\n                boxes[i, 4] *= min(strenth**(1-area/pivot), strenth)\n            elif mode == \"asc\":\n                boxes[i, 4] *= min(strenth**(-1+area/pivot), strenth)\n            else:\n                raise Exception(\"No such mode\")\n        return boxes\n\n    return _func","metadata":{"execution":{"iopub.status.busy":"2022-02-14T06:50:25.898763Z","iopub.execute_input":"2022-02-14T06:50:25.899483Z","iopub.status.idle":"2022-02-14T06:50:25.907516Z","shell.execute_reply.started":"2022-02-14T06:50:25.899445Z","shell.execute_reply":"2022-02-14T06:50:25.906676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def filter_by_area(boxes, operator, threshold):\n    areas = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])\n\n    if operator == \">\":\n        return boxes[areas > threshold]\n    elif operator == \"<\":\n        return boxes[areas < threshold]\n    else:\n        raise Exception(\"No such operator\")","metadata":{"execution":{"iopub.status.busy":"2022-02-14T06:50:25.908843Z","iopub.execute_input":"2022-02-14T06:50:25.909616Z","iopub.status.idle":"2022-02-14T06:50:25.921169Z","shell.execute_reply.started":"2022-02-14T06:50:25.909579Z","shell.execute_reply":"2022-02-14T06:50:25.920276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # visualize\n# pivot = 2000.0\n# strenth = 2.7\n# mode = \"asc\"\n# x = np.linspace(1, 10000, num=10000)\n# if mode == \"asc\":\n#     y = np.clip(strenth**(-1+x/pivot), 0, strenth)\n# else:\n#     y = np.clip(strenth**(1-x/pivot), 0, strenth)\n\n# plt.plot(x, y)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T06:50:25.922689Z","iopub.execute_input":"2022-02-14T06:50:25.922977Z","iopub.status.idle":"2022-02-14T06:50:25.929443Z","shell.execute_reply.started":"2022-02-14T06:50:25.922939Z","shell.execute_reply":"2022-02-14T06:50:25.92876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def wbf(mmdet_list, weights, thres=0.0):\n\n    for mmdet_res in mmdet_list:\n        mask = np.ones_like(mmdet_res[:, :4])\n        mask[:, 0], mask[:, 1], mask[:, 2], mask[:, 3] = 1280.0, 720.0, 1280.0, 720.0\n        mmdet_res[:, :4] /= mask\n\n    boxes_list = [ mmdet_res[:, :4] for mmdet_res in mmdet_list ]\n    score_list = [ mmdet_res[:, 4] for mmdet_res in mmdet_list ]\n    label_list = [ [0] * len(boxes) for boxes in boxes_list ]\n\n    res = weighted_boxes_fusion(\n        boxes_list,\n        score_list,\n        label_list,\n        weights=weights,\n        conf_type=\"avg\",\n        skip_box_thr=thres\n    )\n\n    boxes = res[0]\n    scores = res[1]\n\n    mask = np.ones_like(boxes)\n    mask[:, 0], mask[:, 1], mask[:, 2], mask[:, 3] = 1280.0, 720.0, 1280.0, 720.0\n    boxes *= mask\n\n    return np.concatenate([boxes, scores[:, np.newaxis]], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T06:50:25.931497Z","iopub.execute_input":"2022-02-14T06:50:25.931878Z","iopub.status.idle":"2022-02-14T06:50:25.942181Z","shell.execute_reply.started":"2022-02-14T06:50:25.931842Z","shell.execute_reply":"2022-02-14T06:50:25.94126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chk1280 = \"../input/patric-final-models/r1280-e13.pt\"\nchk1800 = \"../input/patric-final-models/r1800-e13.pt\"\nchk2400 = \"../input/patric-yolov5l6-multiscale-full-e11/best.pt\"\nchk2400a = \"../input/2400-seeds-2022/best.pt\"\nchk2400b = \"../input/patric-2400-seeds-888/best.pt\"\nchk2400c = \"../input/patric-2400-seeds-777/best.pt\"\nchk3200 = \"../input/3200-full/best.pt\"\n\nmodels = [\n#     {\n#         \"chk\": chk2400,\n#         \"size\": 3600,\n#         \"conf\": 0.10,\n#         \"w\": 2\n#     },\n#     {\n#         \"chk\": chk2400a,\n#         \"size\": 4000,\n#         \"conf\": 0.10,\n#         \"w\": 2,\n#         \"rotate\": \"90deg\"\n#     },\n#     {\n#         \"chk\": chk3200,\n#         \"size\": 3200,\n#         \"conf\": 0.10,\n#         \"w\": 2,\n#         \"rotate\": \"90deg\",\n#     },\n#     {\n#         \"chk\": chk2400b,\n#         \"size\": 2400,\n#         \"conf\": 0.10,\n#         \"rotate\": \"270deg\",\n#         \"w\": 2\n#     },\n#     {\n#         \"chk\": chk3200,\n#         \"size\": 3200,\n#         \"conf\": 0.10,\n#         \"w\": 2,\n#         \"rotate\": \"90deg\",\n#     },\n#     {\n#         \"chk\": chk3200,\n#         \"size\": 4000,\n#         \"conf\": 0.10,\n#         \"w\": 2,\n#     },\n#     {\n#         \"chk\": chk2400,\n#         \"size\": 4800,\n#         \"conf\": 0.10,\n#         \"rotate\": \"90deg\",\n#         \"w\": 2\n#     },\n\n    \n    \n    # adapt CV part\n#     {\n#         \"chk\": chk1280,\n#         \"size\": 1800,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#     },\n\n#     {\n#         \"chk\": chk1800,\n#         \"size\": 1800,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#     },\n#     {\n#         \"chk\": chk1800,\n#         \"size\": 2000,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#         \"rotate\": \"90deg\",\n#     },\n\n    {\n        \"chk\": chk2400,\n        \"size\": 2400,\n        \"conf\": 0.01,\n        \"w\": 1,\n    },\n    {\n        \"chk\": chk2400,\n        \"size\": 2400,\n        \"conf\": 0.01,\n        \"w\": 1,\n        \"rotate\": \"90deg\",\n    },\n#     {\n#         \"chk\": chk2400a,\n#         \"size\": 2800,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#         \"rotate\": \"90deg\",\n#     },\n\n\n#     {\n#         \"chk\": chk3200,\n#         \"size\": 3200,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#     },\n\n#     {\n#         \"chk\": chk3200,\n#         \"size\": 3200,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#         \"rotate\": \"90deg\",\n#     },\n#     {\n#         \"chk\": chk3200,\n#         \"size\": 3400,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#     },\n#     {\n#         \"chk\": chk3200,\n#         \"size\": 3600,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#         \"rotate\": \"180deg\",\n#     },\n\n#     {\n#         \"chk\": chk3200,\n#         \"size\": 4000,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#         \"rotate\": \"90deg\",\n#     },\n\n\n#     # adapt LB part\n#     {\n#         \"chk\": chk2400,\n#         \"size\": 4800,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#         \"rotate\": \"180deg\",\n#     },\n\n#     {\n#         \"chk\": chk2400b,\n#         \"size\": 4000,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#     },\n\n#     {\n#         \"chk\": chk2400c,\n#         \"size\": 3200,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#         \"rotate\": \"90deg\",\n#     },\n]\n","metadata":{"execution":{"iopub.status.busy":"2022-02-14T06:58:49.957253Z","iopub.execute_input":"2022-02-14T06:58:49.957509Z","iopub.status.idle":"2022-02-14T06:58:49.972074Z","shell.execute_reply.started":"2022-02-14T06:58:49.957479Z","shell.execute_reply":"2022-02-14T06:58:49.969928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for m in models:\n    m[\"model\"] = torch.hub.load('../input/yolov5-lib-ds',\n                                'custom',\n                                path=m[\"chk\"],\n                                source='local',\n                                force_reload=True)\n    m[\"model\"].conf = 0.01\n    m[\"__attention_area__\"] = np.ndarray(shape=[0, 5], dtype=float)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T06:58:50.347457Z","iopub.execute_input":"2022-02-14T06:58:50.34812Z","iopub.status.idle":"2022-02-14T06:59:03.381712Z","shell.execute_reply.started":"2022-02-14T06:58:50.348078Z","shell.execute_reply":"2022-02-14T06:59:03.380949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_rotate_back_fn(deg):\n    \n    h, w = {\n        \"90deg\": (1280, 720),\n        \"180deg\": (720, 1280),\n        \"270deg\": (1280, 720)\n    }[deg]\n\n    bbox_rot90_count = {\n        \"90deg\": 1,\n        \"180deg\": 2,\n        \"270deg\": 3\n    }[deg]\n\n    norm_back = np.array([1280, 720, 1280, 720, 1])\n\n    def _func(xyxyc):\n        xyxyc /= np.array([w, h, w, h, 1])\n        xyxyc = np.array([*bbox_rot90(xyxyc, bbox_rot90_count, None, None), xyxyc[4]])\n        return xyxyc * norm_back\n\n    return _func\n\n\nrotates_funcs = {\n    \"90deg\": (cv2.ROTATE_90_CLOCKWISE, create_rotate_back_fn(\"90deg\")),\n    \"180deg\": (cv2.ROTATE_180, create_rotate_back_fn(\"180deg\")),\n    \"270deg\": (cv2.ROTATE_90_COUNTERCLOCKWISE, create_rotate_back_fn(\"270deg\"))\n}\n\natt_area = np.ndarray(shape=[0, 5], dtype=float)\nprevious_image = np.zeros(shape=(720, 1280, 3))\n\ndef infer(image):\n    global previous_image\n\n    cat_image = np.concatenate([previous_image, image], axis=0)\n    previous_image = image.copy()\n    res = splitor(torch.tensor(cat_image / 255.0).permute(2, 0, 1).unsqueeze(0).float().cuda()).item()\n    is_continouse = res < ATT_SPLITOR_THRESHOLD\n\n    all_boxes = []\n    ws = []\n    for m in models:\n        augment = m[\"augment\"] if \"augment\" in m else False\n        if 'rotate' in m:\n            rc = m['rotate']\n            cv_flag, rotate_back_fn = rotates_funcs[rc]\n            image_rotated = cv2.rotate(image, cv_flag)\n            preds_rotated = m[\"model\"](image_rotated, size=m[\"size\"], augment=augment).pandas().xyxy[0]\n            preds_rotated = preds_rotated[['xmin', 'ymin', 'xmax', 'ymax', 'confidence']].to_numpy()\n            for i, box in enumerate(preds_rotated):\n                preds_rotated[i] = rotate_back_fn(box)\n            boxes = preds_rotated\n        else:\n            r = m[\"model\"](image, size=m[\"size\"], augment=augment).pandas().xyxy[0]\n            boxes = r[['xmin', 'ymin', 'xmax', 'ymax', 'confidence']].to_numpy()\n        if \"area_scaler\" in m:\n            boxes = m[\"area_scaler\"](boxes)\n        \n        if \"area\" in m:\n            operator, threshold = m[\"area\"]\n            boxes = filter_by_area(boxes, operator, threshold)\n\n        if ATT_ENABLED:\n            # if not continouse, clear the previous attention area\n            if not is_continouse:\n                m[\"__attention_area__\"] = np.ndarray(shape=[0, 5], dtype=float)\n            boxes = boxes.astype(float)\n            _att_area = m[\"__attention_area__\"].copy()\n            _boxes = boxes.copy()\n            ious = box_iou(torch.tensor(_boxes[:, :4]), torch.tensor(_att_area[:, :4])).numpy()\n            if len(ious):\n                target_index = ious.argmax(axis=0)\n                if len(target_index) > 0:\n                    boxes[target_index, 4] += ATT_BOOST_SCORE\n\n            att_area = _boxes[_boxes[:, 4] > ATT_ANCHOR_SCORE]\n            m[\"__attention_area__\"] = att_area\n\n        boxes = boxes[boxes[:, 4] > m[\"conf\"]]\n        all_boxes.append(boxes)\n        ws.append(m[\"w\"])\n    preds = wbf(all_boxes, ws)\n\n    # filtering out area <= 500\n    # preds = preds[((preds[:, 2] - preds[:, 0]) * (preds[:, 3] - preds[:, 1])) > 500]\n    return preds","metadata":{"execution":{"iopub.status.busy":"2022-02-14T06:59:03.383957Z","iopub.execute_input":"2022-02-14T06:59:03.38438Z","iopub.status.idle":"2022-02-14T06:59:03.408758Z","shell.execute_reply.started":"2022-02-14T06:59:03.384337Z","shell.execute_reply":"2022-02-14T06:59:03.407983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"../input/tensorflow-great-barrier-reef/train.csv\")\ndf = df[df.annotations != '[]']\nitem = df.sample(1).iloc[0]\nimage = cv2.imread(f\"../input/tensorflow-great-barrier-reef/train_images/video_{item.video_id}/{item.video_frame}.jpg\")\nimage = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T06:59:03.41078Z","iopub.execute_input":"2022-02-14T06:59:03.411145Z","iopub.status.idle":"2022-02-14T06:59:03.485627Z","shell.execute_reply.started":"2022-02-14T06:59:03.411081Z","shell.execute_reply":"2022-02-14T06:59:03.484823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res = infer(image)\nprint(res)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T06:59:03.487757Z","iopub.execute_input":"2022-02-14T06:59:03.488059Z","iopub.status.idle":"2022-02-14T06:59:05.960578Z","shell.execute_reply.started":"2022-02-14T06:59:03.48802Z","shell.execute_reply":"2022-02-14T06:59:05.959764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 22s ~= 9 hours infering\nstart = time.time()\nfor i in range(10):\n    infer(image)\nend = time.time()\nhours = round((end - start) / 22 * 9, 1)\nprint(f\"About {hours} hours\")","metadata":{"execution":{"iopub.status.busy":"2022-02-14T06:59:05.962044Z","iopub.execute_input":"2022-02-14T06:59:05.96253Z","iopub.status.idle":"2022-02-14T06:59:30.655876Z","shell.execute_reply.started":"2022-02-14T06:59:05.96247Z","shell.execute_reply":"2022-02-14T06:59:30.655043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx, (img, pred_df) in enumerate(tqdm(iter_test)):\n    anno = ''\n    r = infer(img)\n    if r.shape[0] == 0:\n        anno = ''\n    else:\n        for b in r:\n            if b[4] > FINAL_THRESHOLD:\n                anno += '{} {} {} {} {} '.format(b[4], int(b[0]), int(b[1]), int(b[2]-b[0]), int(b[3]-b[1]))\n\n    pred_df['annotations'] = anno.strip(' ')\n    env.predict(pred_df)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T06:58:14.955575Z","iopub.execute_input":"2022-02-14T06:58:14.955847Z","iopub.status.idle":"2022-02-14T06:58:14.968058Z","shell.execute_reply.started":"2022-02-14T06:58:14.955809Z","shell.execute_reply":"2022-02-14T06:58:14.967259Z"},"trusted":true},"execution_count":null,"outputs":[]}]}