{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\nimport pandas as pd\nimport os\nimport cv2\nimport matplotlib.pyplot as plt\nimport glob\nimport shutil\nimport sys\nsys.path.append('../input/tensorflow-great-barrier-reef')\nimport torch\nfrom PIL import Image\nimport ast\nimport copy\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n\nfrom shutil import copyfile\nfrom IPython.display import display\n\nsys.path.append(\"../input/ensemble-boxes-gbr\")\nfrom ensemble_boxes import nms, weighted_boxes_fusion\n\n\nsys.path.append(\"../input/filterpy-gbr\")\n\nsys.path.append(\"../input/sort-lib\")\nfrom sort import Sort","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-02-14T12:29:18.225084Z","iopub.execute_input":"2022-02-14T12:29:18.225475Z","iopub.status.idle":"2022-02-14T12:29:21.411065Z","shell.execute_reply.started":"2022-02-14T12:29:18.225393Z","shell.execute_reply":"2022-02-14T12:29:21.410354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /root/.config/Ultralytics\n!cp /kaggle/input/yolov5-font/Arial.ttf /root/.config/Ultralytics/","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:29:21.412735Z","iopub.execute_input":"2022-02-14T12:29:21.412982Z","iopub.status.idle":"2022-02-14T12:29:22.808259Z","shell.execute_reply.started":"2022-02-14T12:29:21.412953Z","shell.execute_reply":"2022-02-14T12:29:22.807331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# inference train data or test data\n# debug = True\ndebug = False\n\nUSE_TRACKING = True\n# USE_TRACKING = False\n\nOUTPUT_IMAGE = False\n\n# kaggle notebook or local jupyter notebook\nKAGGLE_NOTEBOOK = False\n\nSPLIT_TYPE = \"video\"\n# SPLIT_TYPE = \"group\"\n\nFOLD = 0\nCONF_THRE = 0.05\n# IOU_THRE = 0.3\nIOU_THRE = 0.45\nMIN_DETECT_LENGTH = 5\n\nMIN_TIME_SINCE_UPDATE = 2\nINCLUDE_FIRST_BOX = True\n# INCLUDE_FIRST_BOX = False\n\n# wbf_iou = 0.3\nwbf_iou = 0.45\n# wbf_conf_thresh = 0.2\nwbf_conf_thresh = 0.2\n\nROOT_DIR  = os.path.abspath('../input/tensorflow-great-barrier-reef/')\n","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:29:22.810333Z","iopub.execute_input":"2022-02-14T12:29:22.810866Z","iopub.status.idle":"2022-02-14T12:29:22.817638Z","shell.execute_reply.started":"2022-02-14T12:29:22.810823Z","shell.execute_reply":"2022-02-14T12:29:22.81699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_bbox(annots):\n    bboxes = [list(annot.values()) for annot in annots]\n    return bboxes\n\ndef get_path(row):\n    row['image_path'] = f'{ROOT_DIR}/train_images/video_{row.video_id}/{row.video_frame}.jpg'\n    return row","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:29:22.821587Z","iopub.execute_input":"2022-02-14T12:29:22.821815Z","iopub.status.idle":"2022-02-14T12:29:22.828827Z","shell.execute_reply.started":"2022-02-14T12:29:22.821773Z","shell.execute_reply":"2022-02-14T12:29:22.82794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def voc2yolo(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    voc  => [x1, y1, x2, y1]\n    yolo => [xmid, ymid, w, h] (normalized)\n    \"\"\"\n    \n    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    \n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]]/ image_width\n    bboxes[..., [1, 3]] = bboxes[..., [1, 3]]/ image_height\n    \n    w = bboxes[..., 2] - bboxes[..., 0]\n    h = bboxes[..., 3] - bboxes[..., 1]\n    \n    bboxes[..., 0] = bboxes[..., 0] + w/2\n    bboxes[..., 1] = bboxes[..., 1] + h/2\n    bboxes[..., 2] = w\n    bboxes[..., 3] = h\n    \n    return bboxes\n\ndef yolo2voc(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    yolo => [xmid, ymid, w, h] (normalized)\n    voc  => [x1, y1, x2, y1]\n    \n    \"\"\" \n    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    \n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]]* image_width\n    bboxes[..., [1, 3]] = bboxes[..., [1, 3]]* image_height\n    \n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]]/2\n    bboxes[..., [2, 3]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]\n    \n    return bboxes\n\ndef coco2yolo(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    coco => [xmin, ymin, w, h]\n    yolo => [xmid, ymid, w, h] (normalized)\n    \"\"\"\n    \n    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    \n    # normolizinig\n    bboxes[..., [0, 2]]= bboxes[..., [0, 2]]/ image_width\n    bboxes[..., [1, 3]]= bboxes[..., [1, 3]]/ image_height\n    \n    # converstion (xmin, ymin) => (xmid, ymid)\n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]/2\n    \n    return bboxes\n\ndef yolo2coco(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    yolo => [xmid, ymid, w, h] (normalized)\n    coco => [xmin, ymin, w, h]\n    \n    \"\"\" \n    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    \n    # denormalizing\n    bboxes[..., [0, 2]]= bboxes[..., [0, 2]]* image_width\n    bboxes[..., [1, 3]]= bboxes[..., [1, 3]]* image_height\n    \n    # converstion (xmid, ymid) => (xmin, ymin) \n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]]/2\n    \n    return bboxes\n\ndef voc2coco(bboxes, image_height=720, image_width=1280):\n    bboxes  = voc2yolo(bboxes, image_height, image_width)\n    bboxes  = yolo2coco(bboxes, image_height, image_width)\n    return bboxes\n\n\ndef load_image(image_path):\n    return cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)\n","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:29:22.830225Z","iopub.execute_input":"2022-02-14T12:29:22.83074Z","iopub.status.idle":"2022-02-14T12:29:22.849295Z","shell.execute_reply.started":"2022-02-14T12:29:22.830704Z","shell.execute_reply":"2022-02-14T12:29:22.848593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from ast import literal_eval\n\ndef load_image(video_id, video_frame, image_dir):\n    img_path = f'{image_dir}/video_{video_id}/{video_frame}.jpg'\n    assert os.path.exists(img_path), f'{img_path} does not exist.'\n    img = cv2.imread(img_path)\n    return img\n\n\ndef decode_annotations(annotaitons_str):\n    \"\"\"decode annotations in string to list of dict\"\"\"\n    return literal_eval(annotaitons_str)\n\n\ndef generate_gt(annotations):\n    gt_bboxes = []\n\n    for ann in annotations:\n        gt_bboxes.append(np.array([ann['x'], ann['y'], ann['width'], ann['height']]))\n\n    gt_bboxes = np.array(gt_bboxes)\n    return gt_bboxes\n\n\nimport torch\nfrom torchvision.ops import box_iou\n\ndef calc_f2_score(gt_bboxes_list, pred_bboxes_list, iou_th):    \n    num_tp = 0\n    num_fp = 0\n    num_fn = 0\n    for p, gt in zip(pred_bboxes_list, gt_bboxes_list):\n        if len(p) > 0 and len(gt) > 0:\n            # print(\"p:\",p)\n            # print(\"gt:\",gt)\n            \n            p = p[:,1:].copy()\n            gt = gt.copy()\n            p[:,2:] += p[:,:2]\n            gt[:,2:] += gt[:,:2]\n            p = torch.from_numpy(p.astype(np.float32)).clone()\n            gt = torch.from_numpy(gt.astype(np.float32)).clone()\n            \n            iou_matrix = box_iou(p, gt)\n            tp = len(torch.where(iou_matrix.max(0)[0] >= iou_th)[0])\n            fp = len(p) - tp\n            fn = len(torch.where(iou_matrix.max(0)[0] < iou_th)[0])\n            \n            # print(f'num_gt:{len(gt):<3} num_pred:{len(p):<3} tp:{tp:<3} fp:{fp:<3} fn:{fn:<3}')\n            # print()\n\n            num_tp += tp\n            num_fp += fp\n            num_fn += fn\n        elif len(p) == 0 and len(gt):\n            num_fn += len(gt)\n        elif len(p) and len(gt) == 0:\n            num_fp += len(p)\n        \n    score = 5 * num_tp / (5 * num_tp + 4 * num_fn + num_fp)\n    return score, num_tp, num_fp\n","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:29:22.851223Z","iopub.execute_input":"2022-02-14T12:29:22.852358Z","iopub.status.idle":"2022-02-14T12:29:23.242462Z","shell.execute_reply.started":"2022-02-14T12:29:22.85232Z","shell.execute_reply":"2022-02-14T12:29:23.241637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_model(ckpt_path, conf, iou):\n    model = torch.hub.load('../input/yolov5-lib',\n                           'custom',\n                           path=ckpt_path,\n                           source='local',\n                           force_reload=True)  # local repo\n    model.conf = conf  # NMS confidence threshold\n    model.iou  = iou  # NMS IoU threshold\n    model.classes = None   # (optional list) filter by class, i.e. = [0, 15, 16] for persons, cats and dogs\n    model.multi_label = False  # NMS multiple labels per box\n    model.max_det = 1000  # maximum number of detections per image\n    return model\n\ndef predict(model, img, size, augment):\n    height, width = img.shape[:2]\n    results = model(img, size=size, augment=augment)\n    preds   = results.pandas().xyxy[0]\n    bboxes  = preds[['xmin','ymin','xmax','ymax']].values\n    if len(bboxes):\n        bboxes  = voc2coco(bboxes,height,width)\n        confs   = preds.confidence.values\n        return bboxes, confs\n    else:\n        return [],[]","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:29:23.245148Z","iopub.execute_input":"2022-02-14T12:29:23.245628Z","iopub.status.idle":"2022-02-14T12:29:23.25297Z","shell.execute_reply.started":"2022-02-14T12:29:23.245588Z","shell.execute_reply":"2022-02-14T12:29:23.252289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_yolov5_single_image(model, img, hflip, vflip, image_size):\n    if hflip:\n        img = np.fliplr(img)\n    if vflip:\n        img = np.flipud(img)\n\n    boxes, scores = predict(model, img, size=image_size, augment=False)\n    \n    if hflip:\n        if len(boxes) > 0:\n            boxes[:, 0] = 1280 - (boxes[:, 0] + boxes[:, 2])\n    if vflip:\n        if len(boxes) > 0:\n            boxes[:, 1] = 720 - (boxes[:, 1] + boxes[:, 3])\n\n    concat_box = np.array([])\n    if len(scores) > 0:\n        scores = np.expand_dims(scores, 1)\n        # print(scores.shape, boxes.shape)\n        concat_box = np.concatenate([scores, boxes], 1)\n\n    return concat_box","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:29:23.254337Z","iopub.execute_input":"2022-02-14T12:29:23.254795Z","iopub.status.idle":"2022-02-14T12:29:23.266192Z","shell.execute_reply.started":"2022-02-14T12:29:23.254755Z","shell.execute_reply":"2022-02-14T12:29:23.265441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_yolov5(hflip, vflip):\n    pred_bboxes_list = []\n    for image_id, ann_str in tqdm(valid_df[['image_id','annotations']].values):\n        \n        # pred_bboxes = []\n        video_id, image_id = image_id.split(\"-\")\n        image_path = f\"../input/tensorflow-great-barrier-reef/train_images/video_{video_id}/{image_id}.jpg\"\n        # print(image_path)\n        img = cv2.imread(image_path)[...,::-1]\n        \n#         concat_box = predict_yolov5_single_image(model, img, hflip, vflip)\n        concat_box = predict_yolov5_single_image(model, img, hflip, vflip, 2304)\n        pred_bboxes_list.append(concat_box)\n\n    return pred_bboxes_list","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:29:23.267753Z","iopub.execute_input":"2022-02-14T12:29:23.268677Z","iopub.status.idle":"2022-02-14T12:29:23.276788Z","shell.execute_reply.started":"2022-02-14T12:29:23.268637Z","shell.execute_reply":"2022-02-14T12:29:23.275998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if debug:\n    df = pd.read_csv(\"../input/tensorflow-great-barrier-reef/train.csv\")\n\n    # Taken only annotated photos\n    df[\"num_bbox\"] = df['annotations'].apply(lambda x: str.count(x, 'x'))\n    # df_train = df[df[\"num_bbox\"]>0]\n\n    #Annotations \n    df['annotations'] = df['annotations'].progress_apply(lambda x: ast.literal_eval(x))\n    df['bboxes'] = df.annotations.progress_apply(get_bbox)\n\n    #Images resolution\n    df[\"width\"] = 1280\n    df[\"height\"] = 720\n\n    #Path of images\n    df = df.progress_apply(get_path, axis=1)\n\n    if SPLIT_TYPE == \"group\":\n        sequence_list = [\n            [8503, 37114],\n            [45518, 35305, 53708, 26651],\n            [59337, 18048, 17665, 60754],\n            [996, 60510, 22643, 29859],\n            [40258, 8399, 45015, 15827, 29424, 44160]\n        ]\n\n        valid_df = df[df[\"sequence\"].isin(sequence_list[FOLD])]\n\n        train_sequence_list = []\n        for fold, each_sequence_list in enumerate(sequence_list):\n            if fold == FOLD:\n                continue\n            train_sequence_list += each_sequence_list\n\n        train_df = df[df[\"sequence\"].isin(train_sequence_list)]\n        train_df = train_df[train_df[\"num_bbox\"] > 0]\n        print(train_df.shape, valid_df.shape)\n    \n    elif SPLIT_TYPE == \"video\":\n        train_df = df[df[\"video_id\"]!=FOLD]\n        valid_df = df[df[\"video_id\"]==FOLD]\n        print(train_df.shape, valid_df.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:29:23.279731Z","iopub.execute_input":"2022-02-14T12:29:23.280122Z","iopub.status.idle":"2022-02-14T12:29:23.291212Z","shell.execute_reply.started":"2022-02-14T12:29:23.280085Z","shell.execute_reply":"2022-02-14T12:29:23.290448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if debug:\n    yolov5_path = \"../input/yolov5-alldata-m6x6/x6_alldata.pt\"\n    model = load_model(yolov5_path, conf=CONF_THRE, iou=IOU_THRE)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:29:23.292343Z","iopub.execute_input":"2022-02-14T12:29:23.292727Z","iopub.status.idle":"2022-02-14T12:29:23.301757Z","shell.execute_reply.started":"2022-02-14T12:29:23.292687Z","shell.execute_reply":"2022-02-14T12:29:23.30115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if debug:\n    gt_bboxes_list = []\n    for image_id, ann_str in tqdm(valid_df[['image_id','annotations']].values):\n        gt_bboxes = generate_gt(ann_str)\n        gt_bboxes_list.append(gt_bboxes)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:29:23.30416Z","iopub.execute_input":"2022-02-14T12:29:23.305031Z","iopub.status.idle":"2022-02-14T12:29:23.311083Z","shell.execute_reply.started":"2022-02-14T12:29:23.30499Z","shell.execute_reply":"2022-02-14T12:29:23.310477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if debug:\n    yolov5_pred_bboxes_lists = []\n    yolov5_pred_bboxes_lists.append(predict_yolov5(hflip=False, vflip=False))","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-02-14T12:29:23.312381Z","iopub.execute_input":"2022-02-14T12:29:23.312866Z","iopub.status.idle":"2022-02-14T12:29:23.318972Z","shell.execute_reply.started":"2022-02-14T12:29:23.312834Z","shell.execute_reply":"2022-02-14T12:29:23.318193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if debug:\n    for pred_bboxes_list in yolov5_pred_bboxes_lists:\n        f2_score_list = []\n        for iou_th in np.arange(0.3, 0.85, 0.05):\n            f2_score, _, _ = calc_f2_score(gt_bboxes_list, pred_bboxes_list, iou_th)\n            f2_score_list.append(f2_score)\n            # print(iou_th, f2_score)\n        print(f2_score_list)\n        print(sum(f2_score_list)/len(f2_score_list))\n        print()","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:29:23.320341Z","iopub.execute_input":"2022-02-14T12:29:23.320826Z","iopub.status.idle":"2022-02-14T12:29:23.327702Z","shell.execute_reply.started":"2022-02-14T12:29:23.320791Z","shell.execute_reply":"2022-02-14T12:29:23.326899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gbf_to_wbf(boxes):\n    boxes = boxes.copy()\n    if len(boxes) == 0:\n        return boxes\n    boxes[:, 3:] += boxes[:, 1:3]\n    boxes[:, 1] /= 1280\n    boxes[:, 3] /= 1280\n    boxes[:, 2] /= 720\n    boxes[:, 4] /= 720\n    return boxes\n\ndef wbf_to_gbf(boxes):\n    boxes = boxes.copy()\n    if len(boxes) == 0:\n        return boxes\n    boxes[:, 1] *= 1280\n    boxes[:, 3] *= 1280\n    boxes[:, 2] *= 720\n    boxes[:, 4] *= 720\n    boxes[:, 3:] -= boxes[:, 1:3]\n    return boxes","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:29:23.329711Z","iopub.execute_input":"2022-02-14T12:29:23.330534Z","iopub.status.idle":"2022-02-14T12:29:23.338877Z","shell.execute_reply.started":"2022-02-14T12:29:23.330497Z","shell.execute_reply":"2022-02-14T12:29:23.338116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def wbf_single_image(single_image_pred_boxes_list):\n#     print(\"single_image_pred_boxes_list\",single_image_pred_boxes_list)\n    boxes_list = []\n    scores_list = []\n    labels_list = []\n    \n    for boxes in single_image_pred_boxes_list:\n        boxes = gbf_to_wbf(boxes)\n        \n        pos_list = []\n        score_list = []\n        for box in boxes:\n            pos_list.append(box[1:].tolist())\n            score_list.append(box[0])\n\n        boxes_list.append(pos_list)\n        scores_list.append(score_list)\n        labels_list.append([0]*len(boxes))\n    \n    boxes, scores, labels = weighted_boxes_fusion(boxes_list, scores_list, labels_list, weights=wbf_weights, iou_thr=wbf_iou)\n    \n    concat_box = np.array([])\n    if len(boxes) > 0:\n        scores = np.expand_dims(scores, 1)\n        concat_box = np.concatenate([scores, boxes], 1)\n        concat_box = concat_box[concat_box[:,0] > wbf_conf_thresh]\n        if len(concat_box) == 0:\n            concat_box = np.array([])\n        else:\n            concat_box = np.array(wbf_to_gbf(concat_box))\n\n    return concat_box","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:29:23.340099Z","iopub.execute_input":"2022-02-14T12:29:23.34058Z","iopub.status.idle":"2022-02-14T12:29:23.350794Z","shell.execute_reply.started":"2022-02-14T12:29:23.340543Z","shell.execute_reply":"2022-02-14T12:29:23.350008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if debug:\n    wbf_pred_list = [\n        # pred_bboxes_lists_5120,\n    ]\n    wbf_weights = [\n        # 1\n    ]\n\n    wbf_pred_list += yolov5_pred_bboxes_lists\n    wbf_weights += [1]*len(yolov5_pred_bboxes_lists)\n    # wbf_pred_list = [yolov5_pred_bboxes_lists[0]]\n    # wbf_weights = [1]\n\n    wbf_conf_thresh = 0.2","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:29:23.352769Z","iopub.execute_input":"2022-02-14T12:29:23.353464Z","iopub.status.idle":"2022-02-14T12:29:23.359407Z","shell.execute_reply.started":"2022-02-14T12:29:23.353421Z","shell.execute_reply":"2022-02-14T12:29:23.358695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_TRACKING:\n    if debug:\n        sort = Sort(\n            min_time_since_update = MIN_TIME_SINCE_UPDATE,\n            include_first_box = INCLUDE_FIRST_BOX\n        )\n\n        wbf_bboxes_list = []\n        interpolated_bboxes_list = []\n\n        for i in range(len(gt_bboxes_list)):\n            single_image_pred_boxes_list = []\n            for each_pred_boxes_list in wbf_pred_list:\n                single_image_pred_boxes_list.append(each_pred_boxes_list[i])\n\n            wbf_box = wbf_single_image(single_image_pred_boxes_list)\n            wbf_bboxes_list.append(wbf_box.copy())\n\n            if len(wbf_box) > 0:\n                wbf_box[:, 3:] += wbf_box[:, 1:3]\n                wbf_box = np.roll(wbf_box, 4, axis=1)\n            else:\n                wbf_box = np.empty((0, 5))\n\n            tracked_box = sort.update(wbf_box)\n            if len(tracked_box) > 0:\n                tracked_box = np.roll(tracked_box, 1, axis=1)\n                tracked_box[:, 3:] -= tracked_box[:, 1:3]\n            interpolated_bboxes_list.append(tracked_box)\n\nelse:\n    if debug:\n        wbf_bboxes_list = []\n\n        for i in range(len(gt_bboxes_list)):\n            single_image_pred_boxes_list = []\n            for each_pred_boxes_list in wbf_pred_list:\n                single_image_pred_boxes_list.append(each_pred_boxes_list[i])\n\n            wbf_box = wbf_single_image(single_image_pred_boxes_list)\n\n            wbf_bboxes_list.append(wbf_box)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:29:23.360808Z","iopub.execute_input":"2022-02-14T12:29:23.361356Z","iopub.status.idle":"2022-02-14T12:29:23.372044Z","shell.execute_reply.started":"2022-02-14T12:29:23.361319Z","shell.execute_reply":"2022-02-14T12:29:23.371392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if debug:\n    f2_score_list = []\n    for iou_th in np.arange(0.3, 0.85, 0.05):\n        f2_score, num_tp, num_fp = calc_f2_score(gt_bboxes_list, wbf_bboxes_list, iou_th)\n        f2_score_list.append(f2_score)\n        print(iou_th, f2_score, num_tp, num_fp)\n    print(f2_score_list)\n    print(sum(f2_score_list)/len(f2_score_list))","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:29:23.373631Z","iopub.execute_input":"2022-02-14T12:29:23.374198Z","iopub.status.idle":"2022-02-14T12:29:23.380886Z","shell.execute_reply.started":"2022-02-14T12:29:23.374158Z","shell.execute_reply":"2022-02-14T12:29:23.380209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if debug:\n    f2_score_list = []\n    for iou_th in np.arange(0.3, 0.85, 0.05):\n        f2_score, num_tp, num_fp = calc_f2_score(gt_bboxes_list, interpolated_bboxes_list, iou_th)\n        f2_score_list.append(f2_score)\n        print(iou_th, f2_score, num_tp, num_fp)\n    print(f2_score_list)\n    print(sum(f2_score_list)/len(f2_score_list))","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:29:23.382131Z","iopub.execute_input":"2022-02-14T12:29:23.383029Z","iopub.status.idle":"2022-02-14T12:29:23.391003Z","shell.execute_reply.started":"2022-02-14T12:29:23.382973Z","shell.execute_reply":"2022-02-14T12:29:23.390352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if debug:\n    if OUTPUT_IMAGE:\n        os.makedirs(\"../output/result_image\", exist_ok=True)\n\n        for index, image_id in enumerate(tqdm(valid_df['image_id'].values)):\n            pred_boxes = wbf_bboxes_list[index]\n            interpolated_boxes = interpolated_bboxes_list[index]\n            gt_boxes = gt_bboxes_list[index]\n            # print(\"pred_boxes:\",pred_boxes)\n            # print(\"interpolated_boxes:\",interpolated_boxes)\n            # raise\n\n            # pred_bboxes = []\n            video_id, image_id = image_id.split(\"-\")\n            image_path = f\"../input/tensorflow-great-barrier-reef/train_images/video_{video_id}/{image_id}.jpg\"\n            # print(image_path)\n            image = cv2.imread(image_path)\n\n            for gt_box in gt_boxes:\n                gt_box = map(int, gt_box)\n                left, top, width, height = gt_box\n                cv2.rectangle(image, (left, top), (left+width, top+height), (0, 255, 0), 4)\n\n            # if len(pred_boxes)>0:\n            #     pred_boxes = pred_boxes[pred_boxes[:,0]>0.2]\n            for pred_box in pred_boxes:\n                score, left, top, width, height = pred_box\n                left, top, width, height = map(int, [left, top, width, height])\n                cv2.rectangle(image, (left, top), (left+width, top+height), (0, 0, 255), 3)\n                # cv2.putText(image, str(score)[:5], (left, top+width), cv2.FONT_HERSHEY_PLAIN, 2, (0, 0, 0), 1, cv2.LINE_AA)\n\n            # if len(interpolated_boxes)>0:\n            #     interpolated_boxes = interpolated_boxes[interpolated_boxes[:,0]>0.2]\n            for pred_box in interpolated_boxes:\n                score, left, top, width, height = pred_box\n                left, top, width, height = map(int, [left, top, width, height])\n                cv2.rectangle(image, (left, top), (left+width, top+height), (0, 255, 255), 2)\n                cv2.putText(image, str(score)[:5], (left, top+width), cv2.FONT_HERSHEY_PLAIN, 2, (0, 0, 0), 1, cv2.LINE_AA)\n\n            cv2.imwrite(f\"../output/result_image/{video_id}_{image_id}.jpg\", image)\n\n            # raise","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:29:23.392197Z","iopub.execute_input":"2022-02-14T12:29:23.393077Z","iopub.status.idle":"2022-02-14T12:29:23.406357Z","shell.execute_reply.started":"2022-02-14T12:29:23.39304Z","shell.execute_reply":"2022-02-14T12:29:23.405645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not debug:\n    import greatbarrierreef\n    env = greatbarrierreef.make_env()# initialize the environment\n    iter_test = env.iter_test()      # an iterator which loops over the test set and sample submission","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-14T12:29:23.407583Z","iopub.execute_input":"2022-02-14T12:29:23.408325Z","iopub.status.idle":"2022-02-14T12:29:23.432682Z","shell.execute_reply.started":"2022-02-14T12:29:23.408288Z","shell.execute_reply":"2022-02-14T12:29:23.43198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not debug:\n    if USE_TRACKING:\n        sort = Sort(\n            min_time_since_update=MIN_TIME_SINCE_UPDATE,\n            include_first_box = INCLUDE_FIRST_BOX\n        )\n    \n    yolov5_weights_path_list = [\n#         \"../input/yolov5l-videofold-3008/best_fold0.pt\",\n#         \"../input/yolov5l-videofold-3008/best_fold1.pt\",\n#         \"../input/yolov5l-videofold-3008/best_fold2.pt\",\n        \n#         \"../input/yolov5l6-2944/best_fold0.pt\",\n#         \"../input/yolov5l6-2944/best_fold1.pt\",\n#         \"../input/yolov5l6-2944/best_fold2.pt\",\n        \n#         \"../input/yolov5-l6-defaultanchor/best_fold0.pt\",\n#         \"../input/yolov5-l6-defaultanchor/best_fold1.pt\",\n#         \"../input/yolov5l6-2944/best_fold2.pt\",\n        \n        \"../input/yolov5-l6-alldata/l6_alldata.pt\",\n        \"../input/yolov5-l6-alldata/l6_alldata.pt\",\n        \n#         \"../input/yolov5m6-3712-videofold/best_fold0.pt\",\n#         \"../input/yolov5m6-3712-videofold/best_fold1.pt\",\n#         \"../input/yolov5m6-3712-videofold/best_fold2.pt\",\n        \n#         \"../input/yolov5-m6-defaultanchor/best_fold0.pt\",\n#         \"../input/yolov5-m6-defaultanchor/best_fold1.pt\",\n#         \"../input/yolov5-m6-defaultanchor/best_fold2.pt\",\n        \n        \"../input/yolov5-alldata-m6x6/m6_alldata.pt\",\n        \"../input/yolov5-alldata-m6x6/m6_alldata.pt\",\n        \n#         \"../input/yolov5x6-2304-videofold/best_fold0.pt\",\n#         \"../input/yolov5x6-2304-videofold/best_fold1.pt\",\n#         \"../input/yolov5x6-2304-videofold/best_fold2.pt\",\n        \n        \"../input/yolov5-alldata-m6x6/x6_alldata.pt\",\n        \"../input/yolov5-alldata-m6x6/x6_alldata.pt\",\n    ]\n    yolov5_models = []\n    for yolov5_weights in yolov5_weights_path_list:\n        yolov5_models.append(load_model(yolov5_weights, conf=CONF_THRE, iou=IOU_THRE))\n        \n    image_size_list = [\n#         3008,\n        2944,\n        2944*1.35,\n#         2944,\n#         6016,\n        3712,\n        3712*1.35,\n#         3712,\n#         3712,\n#         3712,\n#         7424,\n        2304,\n        2304*1.35,\n#         2304,\n#         2304,\n#         2304,\n#         4608\n    ]\n    \n    wbf_weights = [\n#         1,\n#         1,1,\n#         1,1,1,1,\n        1,1,1,1,1,1,\n#         1,1,1,1,1,1,1,1\n#         1,1,1,1,1,1,1,1,1\n    ]\n    \n    tta_list = [\n        [[False,False]],\n        [[False,True]],\n        [[True,False]],\n        [[True,True]],\n        [[True,True]],\n        [[False,False]],\n#         [[False,False]],\n#         [[False,False]],\n#         [[False,False]],\n        \n#         [[False,False]],\n#         [[False,True]],\n#         [[True,False]],\n#         [[True,True]],\n#         [[False,False]],\n#         [[False,True]],\n#         [[True,False]],\n#         [[True,True]],\n#         [[False,False]],\n\n#         [[False,False],[True,False]],\n#         [[False,False],[False,True]],\n#         [[False,False],[False,False]],\n        \n#         [[False,False],[True,False],[False,True]],\n#         [[False,False],[True,False],[True,True]],\n#         [[False,False],[False,True],[True,True]],\n        \n#         [[False,False],[False,True]],\n#         [[False,False],[True,True]],\n    ]\n    \n    for img, pred_df in iter_test:\n        boxes_list = []\n        scores_list = []\n        labels_list = []\n        \n        wbf_pred_list = []\n        for yolov5_model, image_size, tta in zip(yolov5_models, image_size_list, tta_list):\n#             for hflip, vflip in [[False,False],[True,False],[False,True],[True,True]]:\n#             for hflip, vflip in [[False,False],[True,True]]:\n            for hflip, vflip in tta:\n                concat_box = predict_yolov5_single_image(yolov5_model, img, hflip, vflip, image_size)\n                wbf_pred_list.append(concat_box)\n        wbf_box = wbf_single_image(wbf_pred_list)\n    \n        if USE_TRACKING:\n            if len(wbf_box) > 0:\n                wbf_box[:, 3:] += wbf_box[:, 1:3]\n                wbf_box = np.roll(wbf_box, 4, axis=1)\n            else:\n                wbf_box = np.empty((0, 5))\n\n            tracked_box = sort.update(wbf_box)\n            if len(tracked_box) > 0:\n                tracked_box = np.roll(tracked_box, 1, axis=1)\n                tracked_box[:, 3:] -= tracked_box[:, 1:3]\n            wbf_box = tracked_box\n        \n        predictions = []\n        for conf, xmin, ymin, width, height in wbf_box:\n            predictions.append('{} {} {} {} {}'.format(conf, xmin, ymin, width, height))\n        prediction_str = ' '.join(predictions)\n        pred_df['annotations'] = prediction_str\n        env.predict(pred_df)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-14T12:29:23.433902Z","iopub.execute_input":"2022-02-14T12:29:23.434484Z","iopub.status.idle":"2022-02-14T12:30:01.018712Z","shell.execute_reply.started":"2022-02-14T12:29:23.434446Z","shell.execute_reply":"2022-02-14T12:30:01.017846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not debug:\n    sub_df = pd.read_csv('submission.csv')\n    display(sub_df.head())","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:30:01.020485Z","iopub.execute_input":"2022-02-14T12:30:01.02075Z","iopub.status.idle":"2022-02-14T12:30:01.037966Z","shell.execute_reply.started":"2022-02-14T12:30:01.020714Z","shell.execute_reply":"2022-02-14T12:30:01.037148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}