{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\nimport uuid\nimport os\nimport scipy\nimport cv2\nfrom tqdm import tqdm\nimport math\nimport ast\nsns.set()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-27T22:20:56.020560Z","iopub.execute_input":"2021-12-27T22:20:56.021298Z","iopub.status.idle":"2021-12-27T22:20:57.443895Z","shell.execute_reply.started":"2021-12-27T22:20:56.021153Z","shell.execute_reply":"2021-12-27T22:20:57.442873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv('../input/tensorflow-great-barrier-reef/train.csv')\ndf_train","metadata":{"execution":{"iopub.status.busy":"2021-12-27T22:20:57.445921Z","iopub.execute_input":"2021-12-27T22:20:57.446235Z","iopub.status.idle":"2021-12-27T22:20:57.534345Z","shell.execute_reply.started":"2021-12-27T22:20:57.446197Z","shell.execute_reply":"2021-12-27T22:20:57.533465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def bbox_inv_iou(boxA, boxB):\n    \"\"\"Copied from: https://gist.github.com/meyerjo/dd3533edc97c81258898f60d8978eddc\n    \"\"\"\n    xA, yA = max(boxA[0], boxB[0]), max(boxA[1], boxB[1])\n    xB, yB = min(boxA[2], boxB[2]), min(boxA[3], boxB[3])\n    interArea = abs(max((xB - xA, 0)) * max((yB - yA), 0))\n    if interArea == 0:\n        return 0\n    boxAArea = abs((boxA[2] - boxA[0]) * (boxA[3] - boxA[1]))\n    boxBArea = abs((boxB[2] - boxB[0]) * (boxB[3] - boxB[1]))\n    return 1. - (interArea / float(boxAArea + boxBArea - interArea))\n\n\ndef bbox_center_distance(boxA, boxB):\n    cAx, cAy = (boxA[2] + boxA[0]) / 2., (boxA[3] + boxA[1]) / 2.\n    cBx, cBy = (boxB[2] + boxB[0]) / 2., (boxB[3] + boxB[1]) / 2.\n    return np.sqrt((cBx - cAx) ** 2 + (cBy - cAy) ** 2)\n\n\ndef bbox_center_abs_difference(boxA, boxB):\n    cAx, cAy = (boxA[2] - boxA[0]) / 2., (boxA[3] - boxA[1]) / 2.\n    cBx, cBy = (boxB[2] - boxB[0]) / 2., (boxB[3] - boxB[1]) / 2.\n    return np.sqrt((cBx - cAx) ** 2 + (cBy - cAy) ** 2)\n\n\ndef bbox_center_rel_difference(boxA, boxB, multiplier=10.):\n    cAx, cAy = (boxA[2] - boxA[0]) / 2., (boxA[3] - boxA[1]) / 2.\n    cBx, cBy = (boxB[2] - boxB[0]) / 2., (boxB[3] - boxB[1]) / 2.\n    ccA, ccB = np.sqrt(cAx**2 + cAy**2), np.sqrt(cBx**2 + cBy**2)\n    return multiplier * max(ccA/(ccB + 1e-5), ccB/(ccA + 1e-5))\n\n\ndef find_unique_cots(sequence_df, dist_func=bbox_center_distance, dist_thresh=30.0, verbose=1):\n    # Check that the df is valid\n    unique_diff = np.unique(sequence_df.sequence_frame.diff())\n    assert unique_diff[~np.isnan(unique_diff)] == np.ones(shape=(1, ))\n    \n    prev_bboxes_with_uuids = None\n    annots_with_cots_ids, min_intra_distances = [], []\n    \n    # Some stats\n    all_cots_ids, max_inter_distances, cots_per_frame = set(), [], []\n    \n    def create_new_cots_uuid():\n        new_cots_id = uuid.uuid4().hex\n        all_cots_ids.add(new_cots_id)\n        return new_cots_id\n    \n    for idx, row in sequence_df.iterrows():\n        raw_annots = ast.literal_eval(row.annotations)\n        bboxes = [\n            [s['x'], s['y'], s['x'] + s['width'], s['y'] + s['height']]\n            for s in raw_annots]\n        cots_per_frame.append(len(bboxes))\n        \n        # For frames with no annotation, reset everything.\n        if len(bboxes) == 0:\n            prev_bboxes_with_uuids = None\n            annots_with_cots_ids.append('[]')\n            min_intra_distances.append(0)\n            max_inter_distances.append(0)\n            continue\n        \n        # Intra-frame ious\n        intra_distances = np.zeros(shape=(len(bboxes), len(bboxes)))\n        for i, bbox_0 in enumerate(bboxes):\n            for j, bbox_1 in enumerate(bboxes):\n                if j == i:\n                    intra_distances[i, j] = 1000000.\n                else:\n                    intra_distances[i, j] = dist_func(bbox_0, bbox_1)\n\n        # Tracking COTS bounding-boxes and assign UUID to each COTS\n        if prev_bboxes_with_uuids is None:\n            prev_bboxes_with_uuids = [{\n                'bbox': bbox,\n                'cid': create_new_cots_uuid(),\n                } for bbox in bboxes]\n            max_inter_distances.append(0)\n        else:\n            # Calculate inter-frame IOUs\n            distances = np.zeros(shape=(len(prev_bboxes_with_uuids), len(bboxes)))\n            for i, bbox_with_uuid_0 in enumerate(prev_bboxes_with_uuids):\n                for j, bbox in enumerate(bboxes):\n                    distances[i, j] = dist_func(bbox_with_uuid_0['bbox'], bbox)\n                    if distances[i, j] > dist_thresh:\n                        distances[i, j] = 1000000.\n            max_inter_distances.append(distances.max())\n            \n            row_ids, col_ids = scipy.optimize.linear_sum_assignment(distances)\n            curr_bboxes_with_uuids, curr_matched_ids = [], []\n            for prev_id, curr_id in zip(row_ids, col_ids):\n                if distances[prev_id, curr_id] <= dist_thresh:\n                    curr_matched_ids.append(curr_id)\n                    curr_bboxes_with_uuids.append({\n                        'bbox': bboxes[curr_id],\n                        'cid': prev_bboxes_with_uuids[prev_id]['cid'],\n                        'prev_bbox': prev_bboxes_with_uuids[prev_id]['bbox']\n                    })\n            for curr_id in range(len(bboxes)):\n                if curr_id not in curr_matched_ids:\n                    curr_bboxes_with_uuids.append({\n                        'bbox': bboxes[curr_id],\n                        'cid': create_new_cots_uuid(),\n                    })\n            \n            # Prepare for next iteration\n            prev_bboxes_with_uuids = curr_bboxes_with_uuids\n        \n        # Append calculated info\n        min_intra_distances.append(intra_distances.min())\n        annots_with_cots_ids.append(repr([\n            {\n                'cots_id': bb_w_id['cid'],\n                'x': bb_w_id['bbox'][0],\n                'y': bb_w_id['bbox'][1],\n                'width': bb_w_id['bbox'][2] - bb_w_id['bbox'][0],\n                'height': bb_w_id['bbox'][3] - bb_w_id['bbox'][1],\n                'prev_bbox': bb_w_id.get('prev_bbox', None)\n            }\n            for bb_w_id in prev_bboxes_with_uuids]))\n\n    sequence_df['min_intra_dist'] = min_intra_distances\n    sequence_df['annots_with_cots_id'] = annots_with_cots_ids\n    \n    stats = {\n        'unique_cots': len(all_cots_ids),\n        'max_inter_frame_dist': np.max(max_inter_distances),\n        'max_cots_per_frame': np.max(cots_per_frame),\n    }\n    return sequence_df, stats","metadata":{"execution":{"iopub.status.busy":"2021-12-27T22:23:09.496066Z","iopub.execute_input":"2021-12-27T22:23:09.496423Z","iopub.status.idle":"2021-12-27T22:23:09.532960Z","shell.execute_reply.started":"2021-12-27T22:23:09.496387Z","shell.execute_reply":"2021-12-27T22:23:09.531904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sequence_id = np.unique(df_train.sequence)[2]\nprint(test_sequence_id)\ntest_sequence_df = df_train[df_train.sequence == test_sequence_id]\ntest_sequence_df = test_sequence_df.sort_values(by='sequence_frame')\ntest_sequence_df = test_sequence_df.head(500)\ntest_sequence_df","metadata":{"execution":{"iopub.status.busy":"2021-12-27T22:20:57.594239Z","iopub.execute_input":"2021-12-27T22:20:57.594773Z","iopub.status.idle":"2021-12-27T22:20:57.632725Z","shell.execute_reply.started":"2021-12-27T22:20:57.594732Z","shell.execute_reply":"2021-12-27T22:20:57.631898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seq_df_with_cots_ids, stats = find_unique_cots(\n    test_sequence_df,\n    dist_func=lambda boxA, boxB: bbox_center_distance(boxA, boxB), # + bbox_center_rel_difference(boxA, boxB, multiplier=20.),\n    dist_thresh=50.0)\n\nprint(f'Summary:')\nfor k, v in stats.items():\n    print(f'  - {k}: {v}')\n\nseq_df_with_cots_ids","metadata":{"execution":{"iopub.status.busy":"2021-12-27T22:20:57.634959Z","iopub.execute_input":"2021-12-27T22:20:57.635558Z","iopub.status.idle":"2021-12-27T22:20:57.866954Z","shell.execute_reply.started":"2021-12-27T22:20:57.635524Z","shell.execute_reply":"2021-12-27T22:20:57.865916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_idx, best_row, most_cots = None, None, 0\nfor idx, row in seq_df_with_cots_ids.iterrows():\n    raw_annots = ast.literal_eval(row.annots_with_cots_id)\n    if len(raw_annots) > most_cots:\n        best_idx, best_row, most_cots = idx, row, len(raw_annots)\n\nprint(best_idx, most_cots)\nprint(best_row)\nprint('\\n'.join(str(s) for s in ast.literal_eval(best_row.annots_with_cots_id)))","metadata":{"execution":{"iopub.status.busy":"2021-12-27T22:20:57.868626Z","iopub.execute_input":"2021-12-27T22:20:57.869121Z","iopub.status.idle":"2021-12-27T22:20:57.986154Z","shell.execute_reply.started":"2021-12-27T22:20:57.869076Z","shell.execute_reply":"2021-12-27T22:20:57.985204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_image(video_id, video_frame, image_dir):\n    img_path = f'{image_dir}/video_{video_id}/{video_frame}.jpg'\n    assert os.path.exists(img_path), f'{img_path} does not exist.'\n    img = cv2.imread(img_path)\n    return img\n\n\ndef load_image_with_annotations(row, image_dir):\n    video_id = row.video_id\n    video_frame = row.video_frame\n    annotaitons_str = row.annots_with_cots_id\n\n    img = load_image(video_id, video_frame, image_dir)\n    img_h, img_w = img.shape[:2]\n    annotations = ast.literal_eval(annotaitons_str)\n    palette = (np.array(sns.color_palette(\"hls\", 16)) * 255).astype(np.int).tolist()\n    \n    font = cv2.FONT_HERSHEY_SIMPLEX\n    txt_anno = f'vid: {row.video_id} | seq: {row.sequence} | vid frm: {row.video_frame} | ' \\\n               f'seq frm: {row.sequence_frame} | cots: {len(annotations)}'\n    cv2.putText(img, txt_anno, (10,25), font, 0.7, (30, 30, 30), 2)\n\n    if len(annotations) > 0:\n        for ann in annotations:\n            main_box_thickness = 16\n            instance_color = palette[int(ann['cots_id'][0], 16)][::-1]\n            if ann['prev_bbox'] is not None:\n                pbb = ann['prev_bbox']\n                c_cx, c_cy = ann['x'] + ann['width'] // 2, ann['y'] + ann['height'] // 2\n                p_cx, p_cy = (pbb[0] + pbb[2]) // 2, (pbb[1] + pbb[3]) // 2\n                cv2.line(img, (p_cx, p_cy), (c_cx, c_cy), list(instance_color), thickness=8,)\n                main_box_thickness = 4\n            cv2.rectangle(img, (ann['x'], ann['y']),\n                (ann['x'] + ann['width'], ann['y'] + ann['height']),\n                list(instance_color), thickness=main_box_thickness,)\n    return img\n\n#test\nvideo_id = best_row.video_id\nvideo_frame = best_row.video_frame\nannotations_str = best_row.annots_with_cots_id\nimage_dir = '../input/tensorflow-great-barrier-reef/train_images'\nimg = load_image_with_annotations(best_row, image_dir)\n\nplt.figure(figsize=(15, 10))\nplt.imshow(img[:, :, ::-1])\nplt.axis('off')","metadata":{"execution":{"iopub.status.busy":"2021-12-27T22:20:57.987691Z","iopub.execute_input":"2021-12-27T22:20:57.988211Z","iopub.status.idle":"2021-12-27T22:20:58.809707Z","shell.execute_reply.started":"2021-12-27T22:20:57.988172Z","shell.execute_reply":"2021-12-27T22:20:58.808818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.auto import tqdm\nimport subprocess\n\ndef make_video(df, video_name, image_dir):\n    # partly borrowed from https://github.com/RobMulla/helmet-assignment/blob/main/helmet_assignment/video.py\n    fps = 15 # don't know exact value\n    width = 1280\n    height = 720\n    save_path = f'{video_name}.mp4'\n    tmp_path = \"tmp_\" + save_path\n    output_video = cv2.VideoWriter(tmp_path, cv2.VideoWriter_fourcc(*\"MP4V\"), fps, (width, height))\n    \n    video_df = df\n    for _, row in tqdm(video_df.iterrows(), total=len(video_df)):\n        img = load_image_with_annotations(row, image_dir)\n        output_video.write(img)\n    \n    output_video.release()\n    # Not all browsers support the codec, we will re-load the file at tmp_output_path\n    # and convert to a codec that is more broadly readable using ffmpeg\n    if os.path.exists(save_path):\n        os.remove(save_path)\n    subprocess.run(\n        [\"ffmpeg\", \"-i\", tmp_path, \"-crf\", \"18\", \"-preset\", \"veryfast\", \"-vcodec\", \"libx264\", save_path]\n    )\n    os.remove(tmp_path)\n\n\nmake_video(seq_df_with_cots_ids, 'test_video', image_dir)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T22:20:58.811217Z","iopub.execute_input":"2021-12-27T22:20:58.811454Z","iopub.status.idle":"2021-12-27T22:21:35.018172Z","shell.execute_reply.started":"2021-12-27T22:20:58.811426Z","shell.execute_reply":"2021-12-27T22:21:35.016947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import Video, display\nVideo('test_video.mp4')","metadata":{"execution":{"iopub.status.busy":"2021-12-27T22:21:35.020280Z","iopub.execute_input":"2021-12-27T22:21:35.020613Z","iopub.status.idle":"2021-12-27T22:21:35.028516Z","shell.execute_reply.started":"2021-12-27T22:21:35.020547Z","shell.execute_reply":"2021-12-27T22:21:35.027560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate videos for each sequence","metadata":{}},{"cell_type":"code","source":"additional_columns_by_seqid = []\n\nfor sequence_id in np.unique(df_train.sequence):\n    sequence_df = df_train[df_train.sequence == sequence_id]\n    sequence_df = sequence_df.sort_values(by='sequence_frame')\n    \n    seq_df_with_cots_ids, stats = find_unique_cots(\n        sequence_df,\n        dist_func=lambda boxA, boxB: bbox_center_distance(boxA, boxB),\n        dist_thresh=50.0)\n\n    print(f'Sequence {sequence_id:05d} summary:')\n    for k, v in stats.items():\n        print(f'  - {k}: {v}')\n    \n    additional_columns_by_seqid.append(seq_df_with_cots_ids.loc[:, ['annots_with_cots_id', 'min_intra_dist']])\n    make_video(seq_df_with_cots_ids, f'sequence_{sequence_id:05d}', image_dir)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T22:25:33.644522Z","iopub.execute_input":"2021-12-27T22:25:33.644850Z","iopub.status.idle":"2021-12-27T22:25:36.496925Z","shell.execute_reply.started":"2021-12-27T22:25:33.644817Z","shell.execute_reply":"2021-12-27T22:25:36.496054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.join(pd.concat(additional_columns_by_seqid)).to_csv('train_with_cots_ids.csv')","metadata":{"execution":{"iopub.status.busy":"2021-12-27T22:27:31.147658Z","iopub.execute_input":"2021-12-27T22:27:31.147958Z","iopub.status.idle":"2021-12-27T22:27:31.376880Z","shell.execute_reply.started":"2021-12-27T22:27:31.147928Z","shell.execute_reply":"2021-12-27T22:27:31.376006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}