{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install ../input/segmentation-models-pytorch-021/wheels/timm-0.4.12-py3-none-any.whl\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport albumentations as A\nimport torch\nimport torch.nn.functional as F\nfrom tqdm import tqdm\nimport sys\nimport math\nimport cv2\nimport time\nimport timm\nfrom albumentations.augmentations.geometric.functional import bbox_rot90\nimport matplotlib.pyplot as plt\nfrom torchvision.ops import box_iou\n\n%matplotlib inline\n\nsys.path.append('../input/tensorflow-great-barrier-reef')\nsys.path.append('../input/yolov5-lib-ds')\nsys.path.append('/kaggle/input/weightedboxesfusion/')\n\nfrom ensemble_boxes import weighted_boxes_fusion","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /root/.config/Ultralytics\n!cp /kaggle/input/yolov5-font/Arial.ttf /root/.config/Ultralytics/","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import greatbarrierreef\nenv = greatbarrierreef.make_env()# initialize the environment\niter_test = env.iter_test()      # an iterator which loops over the test set and sample submission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# hyp\n\n# attention\nATT_ENABLED = True\nATT_ANCHOR_SCORE = 0.15\nATT_BOOST_SCORE = 0.1\nATT_SPLITOR_THRESHOLD = 4\n\n# predict\nFINAL_THRESHOLD = 0.08","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"splitor = timm.create_model(\"efficientnet_b0\", num_classes=1)\nsplitor.load_state_dict(torch.load(\"../input/patric-seq-splitor/splitor.pt\"))\nsplitor.eval()\nsplitor.cuda()\nprint(\"fine\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_scaler(pivot, mode, strenth=2.5, max_weight=None):\n    if max_weight is None:\n        max_weight = strenth\n\n    def _func(boxes):\n        areas = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])\n        for i, (area, box) in enumerate(zip(areas, boxes)):\n            # boxes[i, 4] *= np.exp(1. - thr / area)\n            if mode == \"desc\":\n                boxes[i, 4] *= min(strenth**(1-area/pivot), strenth)\n            elif mode == \"asc\":\n                boxes[i, 4] *= min(strenth**(-1+area/pivot), strenth)\n            else:\n                raise Exception(\"No such mode\")\n        return boxes\n\n    return _func","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def wbf(mmdet_list, weights, thres=0.0):\n\n    for mmdet_res in mmdet_list:\n        mask = np.ones_like(mmdet_res[:, :4])\n        mask[:, 0], mask[:, 1], mask[:, 2], mask[:, 3] = 1280.0, 720.0, 1280.0, 720.0\n        mmdet_res[:, :4] /= mask\n\n    boxes_list = [ mmdet_res[:, :4] for mmdet_res in mmdet_list ]\n    score_list = [ mmdet_res[:, 4] for mmdet_res in mmdet_list ]\n    label_list = [ [0] * len(boxes) for boxes in boxes_list ]\n\n    res = weighted_boxes_fusion(\n        boxes_list,\n        score_list,\n        label_list,\n        weights=weights,\n        conf_type=\"avg\",\n        skip_box_thr=thres\n    )\n\n    boxes = res[0]\n    scores = res[1]\n\n    mask = np.ones_like(boxes)\n    mask[:, 0], mask[:, 1], mask[:, 2], mask[:, 3] = 1280.0, 720.0, 1280.0, 720.0\n    boxes *= mask\n\n    return np.concatenate([boxes, scores[:, np.newaxis]], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chk1280 = \"../input/yolov5/last1280.pt\"\nchk1800 = \"../input/yolov5/last1800.pt\"\nchk1280a = \"../input/yolov5/best.pt\"\nchk1800a = \"../input/yolov5/best1800.pt\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# chk1280 = \"../input/yolov5/last1280.pt\"\n# chk1800 = \"../input/yolov5/last1800.pt\"\n# chk1280a = \"../input/yolov5/best.pt\"\n# chk1800a = \"../input/yolov5/best1800.pt\"\n\n\n# # models = [\n# #     {\n# #         \"chk\": chk1280,\n# #         \"size\": 1280,\n# #         \"conf\": 0.01,\n# #         \"w\": 1,\n# #     },\n# #     {\n# #         \"chk\": chk1280a,\n# #         \"size\": 1280,\n# #         \"conf\": 0.01,\n# #         \"w\": 1,\n# #     },\n# #     {\n# #         \"chk\": chk1280,\n# #         \"size\": 1800,\n# #         \"conf\": 0.01,\n# #         \"w\": 1,\n# #         \"rotate\": \"90deg\",\n# #     },\n# #     {\n# #         \"chk\": chk1280a,\n# #         \"size\": 1800,\n# #         \"conf\": 0.01,\n# #         \"w\": 1,\n# #         \"rotate\": \"180deg\",\n# #     },\n    \n\n# #     {\n# #         \"chk\": chk1800,\n# #         \"size\": 1800,\n# #         \"conf\": 0.01,\n# #         \"w\": 1,\n# #     },\n# #     {\n# #         \"chk\": chk1800a,\n# #         \"size\": 1800,\n# #         \"conf\": 0.01,\n# #         \"w\": 1,\n# #     },\n# #     {\n# #         \"chk\": chk1800,\n# #         \"size\": 2000,\n# #         \"conf\": 0.01,\n# #         \"w\": 1,\n# #         \"rotate\": \"90deg\",\n# #         },\n# #     {\n# #         \"chk\": chk1800a,\n# #         \"size\": 2000,\n# #         \"conf\": 0.01,\n# #         \"w\": 1,\n# #         \"rotate\": \"180deg\",\n# #     },\n    \n# # ]\n\n# # models = [\n# #     {\n# #         \"chk\": chk1280,\n# #         \"size\": 1280,\n# #         \"conf\": 0.01,\n# #         \"w\": 1,\n# #     },\n# #     {\n# #         \"chk\": chk1280a,\n# #         \"size\": 1280,\n# #         \"conf\": 0.01,\n# #         \"w\": 2,\n# #     },\n# #     {\n# #         \"chk\": chk1280,\n# #         \"size\": 1800,\n# #         \"conf\": 0.01,\n# #         \"w\": 1,\n# #         \"rotate\": \"90deg\",\n# #     },\n# #     {\n# #         \"chk\": chk1280a,\n# #         \"size\": 1800,\n# #         \"conf\": 0.01,\n# #         \"w\": 2,\n# #         \"rotate\": \"180deg\",\n# #     },\n    \n# #     {\n# #         \"chk\": chk1800,\n# #         \"size\": 1800,\n# #         \"conf\": 0.01,\n# #         \"w\": 1,\n# #     },\n# #     {\n# #         \"chk\": chk1800a,\n# #         \"size\": 1800,\n# #         \"conf\": 0.01,\n# #         \"w\": 2,\n# #     },\n# #     {\n# #         \"chk\": chk1800,\n# #         \"size\": 2000,\n# #         \"conf\": 0.01,\n# #         \"w\": 1,\n# #         \"rotate\": \"90deg\",\n# #         },\n# #     {\n# #         \"chk\": chk1800a,\n# #         \"size\": 2000,\n# #         \"conf\": 0.01,\n# #         \"w\": 2,\n# #         \"rotate\": \"180deg\",\n# #     },\n    \n# # ]\n# models = [\n#     {\n#         \"chk\": chk1280,\n#         \"size\": 1800,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#         \"rotate\": \"90deg\",\n#     },\n#     {\n#         \"chk\": chk1280,\n#         \"size\": 1800,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#         \"rotate\": \"180deg\",\n#     },\n#     {\n#         \"chk\": chk1280a,\n#         \"size\": 1800,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#         \"rotate\": \"180deg\",\n#     },\n#     {\n#         \"chk\": chk1280a,\n#         \"size\": 1800,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#         \"rotate\": \"90deg\",\n#     },\n#      {\n#         \"chk\": chk1800,\n#         \"size\": 1280,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#         \"rotate\": \"90deg\",\n#     },\n#     {\n#         \"chk\": chk1800,\n#         \"size\": 1280,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#         \"rotate\": \"180deg\",\n#     },\n#     {\n#         \"chk\": chk1800a,\n#         \"size\": 1280,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#         \"rotate\": \"180deg\",\n#     },\n#     {\n#         \"chk\": chk1800a,\n#         \"size\": 1280,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#         \"rotate\": \"90deg\",\n#     },\n#     {\n#         \"chk\": chk1800,\n#         \"size\": 2000,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#         \"rotate\": \"90deg\",\n#         },\n#     {\n#         \"chk\": chk1800,\n#         \"size\": 2000,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#         \"rotate\": \"180deg\",\n#         },\n#     {\n#         \"chk\": chk1800a,\n#         \"size\": 2000,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#         \"rotate\": \"180deg\",\n#     },\n#     {\n#         \"chk\": chk1800a,\n#         \"size\": 2000,\n#         \"conf\": 0.01,\n#         \"w\": 1,\n#         \"rotate\": \"90deg\",\n#     },\n    \n    \n# ]\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = [\n    {\n        \"chk\": chk1280,\n        \"size\": 1800,\n        \"conf\": 0.01,\n        \"w\": 1,\n        \"rotate\": \"90deg\",\n    },\n    {\n        \"chk\": chk1280,\n        \"size\": 1800,\n        \"conf\": 0.01,\n        \"w\": 1,\n        \"rotate\": \"180deg\",\n    },\n    {\n        \"chk\": chk1280a,\n        \"size\": 1800,\n        \"conf\": 0.01,\n        \"w\": 1,\n        \"rotate\": \"180deg\",\n    },\n    {\n        \"chk\": chk1280a,\n        \"size\": 1800,\n        \"conf\": 0.01,\n        \"w\": 1,\n        \"rotate\": \"90deg\",\n    },\n    \n    {\n        \"chk\": chk1800,\n        \"size\": 2000,\n        \"conf\": 0.01,\n        \"w\": 1,\n        \"rotate\": \"90deg\",\n        },\n    {\n        \"chk\": chk1800,\n        \"size\": 2000,\n        \"conf\": 0.01,\n        \"w\": 1,\n        \"rotate\": \"180deg\",\n        },\n    {\n        \"chk\": chk1800a,\n        \"size\": 2000,\n        \"conf\": 0.01,\n        \"w\": 1,\n        \"rotate\": \"180deg\",\n    },\n    {\n        \"chk\": chk1800a,\n        \"size\": 2000,\n        \"conf\": 0.01,\n        \"w\": 1,\n        \"rotate\": \"90deg\",\n    },\n]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for m in models:\n    m[\"model\"] = torch.hub.load('../input/yolov5-lib-ds',\n                                'custom',\n                                path=m[\"chk\"],\n                                source='local',\n                                force_reload=True)\n    m[\"model\"].conf = 0.01\n    m[\"__attention_area__\"] = np.ndarray(shape=[0, 5], dtype=float)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_rotate_back_fn(deg):\n    \n    h, w = {\n        \"90deg\": (1280, 720),\n        \"180deg\": (720, 1280),\n        \"270deg\": (1280, 720)\n    }[deg]\n\n    bbox_rot90_count = {\n        \"90deg\": 1,\n        \"180deg\": 2,\n        \"270deg\": 3\n    }[deg]\n\n    norm_back = np.array([1280, 720, 1280, 720, 1])\n\n    def _func(xyxyc):\n        xyxyc /= np.array([w, h, w, h, 1])\n        xyxyc = np.array([*bbox_rot90(xyxyc, bbox_rot90_count, None, None), xyxyc[4]])\n        return xyxyc * norm_back\n\n    return _func\n\n\nrotates_funcs = {\n    \"90deg\": (cv2.ROTATE_90_CLOCKWISE, create_rotate_back_fn(\"90deg\")),\n    \"180deg\": (cv2.ROTATE_180, create_rotate_back_fn(\"180deg\")),\n    \"270deg\": (cv2.ROTATE_90_COUNTERCLOCKWISE, create_rotate_back_fn(\"270deg\"))\n}\n\natt_area = np.ndarray(shape=[0, 5], dtype=float)\nprevious_image = np.zeros(shape=(720, 1280, 3))\n\ndef infer(image):\n    global previous_image\n\n    cat_image = np.concatenate([previous_image, image], axis=0)\n    previous_image = image.copy()\n    res = splitor(torch.tensor(cat_image / 255.0).permute(2, 0, 1).unsqueeze(0).float().cuda()).item()\n    is_continouse = res < ATT_SPLITOR_THRESHOLD\n\n    all_boxes = []\n    ws = []\n    for m in models:\n        augment = m[\"augment\"] if \"augment\" in m else False\n        if 'rotate' in m:\n            rc = m['rotate']\n            cv_flag, rotate_back_fn = rotates_funcs[rc]\n            image_rotated = cv2.rotate(image, cv_flag)\n            preds_rotated = m[\"model\"](image_rotated, size=m[\"size\"], augment=augment).pandas().xyxy[0]\n            preds_rotated = preds_rotated[['xmin', 'ymin', 'xmax', 'ymax', 'confidence']].to_numpy()\n            for i, box in enumerate(preds_rotated):\n                preds_rotated[i] = rotate_back_fn(box)\n            boxes = preds_rotated\n        else:\n            r = m[\"model\"](image, size=m[\"size\"], augment=augment).pandas().xyxy[0]\n            boxes = r[['xmin', 'ymin', 'xmax', 'ymax', 'confidence']].to_numpy()\n\n        if ATT_ENABLED:\n            # if not continouse, clear the previous attention area\n            if not is_continouse:\n                m[\"__attention_area__\"] = np.ndarray(shape=[0, 5], dtype=float)\n            boxes = boxes.astype(float)\n            _att_area = m[\"__attention_area__\"].copy()\n            _boxes = boxes.copy()\n            ious = box_iou(torch.tensor(_boxes[:, :4]), torch.tensor(_att_area[:, :4])).numpy()\n            if len(ious):\n                target_index = ious.argmax(axis=0)\n                if len(target_index) > 0:\n                    boxes[target_index, 4] += ATT_BOOST_SCORE\n\n            att_area = _boxes[_boxes[:, 4] > ATT_ANCHOR_SCORE]\n            m[\"__attention_area__\"] = att_area\n\n        boxes = boxes[boxes[:, 4] > m[\"conf\"]]\n        all_boxes.append(boxes)\n        ws.append(m[\"w\"])\n    preds = wbf(all_boxes, ws)\n\n   \n    return preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"../input/tensorflow-great-barrier-reef/train.csv\")\ndf = df[df.annotations != '[]']\nitem = df.sample(1).iloc[0]\nprint(item)\nimage = cv2.imread(f\"../input/tensorflow-great-barrier-reef/train_images/video_{item.video_id}/{item.video_frame}.jpg\")\nimage = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res = infer(image)\nprint(res)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 22s ~= 9 hours infering\nstart = time.time()\nfor i in range(10):\n    infer(image)\nend = time.time()\nhours = round((end - start) / 22 * 9, 1)\nprint(f\"About {hours} hours\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx, (img, pred_df) in enumerate(tqdm(iter_test)):\n    anno = ''\n    r = infer(img)\n    if r.shape[0] == 0:\n        anno = ''\n    else:\n        for b in r:\n            if b[4] > FINAL_THRESHOLD:\n                anno += '{} {} {} {} {} '.format(b[4], int(b[0]), int(b[1]), int(b[2]-b[0]), int(b[3]-b[1]))\n\n    pred_df['annotations'] = anno.strip(' ')\n    env.predict(pred_df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n# 定义视频对象的参数，包括编码格式，帧率，分辨率等\nfourcc = cv2.VideoWriter_fourcc(*'XVID')\nfps = 20\nsize = (1280, 720)\nvideo = cv2.VideoWriter('output.avi', fourcc, fps, size)\n\n\ncnt=0\ndir='/kaggle/input/tensorflow-great-barrier-reef/train_images/video_0/'\npath=os.listdir(dir)\npath.sort(key=lambda x:int(x[:-4])) #将'.jpg'左边的字符转换成整数型进行排序)\nfor file in path:\n    print(file)\n    ## 读取并反转图片颜色通道\n    img = cv2.imread(dir+file)\n    img=cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n    pred=infer(img)\n    if pred.shape[0]!= 0:\n        for b in pred:\n            if b[4] > FINAL_THRESHOLD:\n                cv2.rectangle(img, (int(b[0]), int(b[1])), (int(b[2]), int(b[3])), (0, 255, 0), 2)\n                cv2.putText(img, str(round(b[4],2)), (int(b[0]), int(b[1]) - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255))\n    img=cv2.cvtColor(img, cv2.COLOR_BGR2RGB)  #恢复通道顺序\n    video.write(img)  \n    cnt+=1\n    \n        \nvideo.release()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.chdir('/kaggle/working')\n\nfrom IPython.display import FileLink\nFileLink('output.avi')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}