{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport cv2\nimport ast\nimport json\nimport subprocess\nfrom glob import glob\nfrom tqdm.notebook import tqdm\nfrom pprint import pprint\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom IPython.display import Video","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:54:03.301454Z","iopub.execute_input":"2021-12-13T10:54:03.301906Z","iopub.status.idle":"2021-12-13T10:54:03.308509Z","shell.execute_reply.started":"2021-12-13T10:54:03.301872Z","shell.execute_reply":"2021-12-13T10:54:03.307822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Introduction\n\n# This notebook is to show image with bboxes and make videos, and to check the jump of bbox number in the sequential video frames.","metadata":{}},{"cell_type":"code","source":"# Root of input\nINPUT_PATH = '../input/tensorflow-great-barrier-reef'\nHEIGHT = 720 # image height\nWIDTH  = 1280 # image width","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:54:03.327881Z","iopub.execute_input":"2021-12-13T10:54:03.328187Z","iopub.status.idle":"2021-12-13T10:54:03.333882Z","shell.execute_reply.started":"2021-12-13T10:54:03.328153Z","shell.execute_reply":"2021-12-13T10:54:03.332949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📝Input data","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(INPUT_PATH + '/train.csv')\ndisplay(df_train)\nprint(df_train.info())","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:54:03.335874Z","iopub.execute_input":"2021-12-13T10:54:03.336246Z","iopub.status.idle":"2021-12-13T10:54:03.4289Z","shell.execute_reply.started":"2021-12-13T10:54:03.3362Z","shell.execute_reply":"2021-12-13T10:54:03.427755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for video_id in df_train['video_id'].unique():\n    print(f'video_id: {video_id}')\n    print(f'w   annotations:  {sum(df_train[df_train[\"video_id\"]==video_id][\"annotations\"] == \"[]\")}')\n    print(f'w/o annotations:  {sum(df_train[df_train[\"video_id\"]==video_id][\"annotations\"] != \"[]\")}\\n')","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:54:03.430421Z","iopub.execute_input":"2021-12-13T10:54:03.430832Z","iopub.status.idle":"2021-12-13T10:54:03.466518Z","shell.execute_reply.started":"2021-12-13T10:54:03.430776Z","shell.execute_reply":"2021-12-13T10:54:03.465346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Change the type of 'annotations' from str to list\ndf_train['annotations'] = df_train['annotations'].apply(ast.literal_eval) # str -> list\n# Add columns of image path and number of bboxes, and the difference.\ndf_train['image_path'] = INPUT_PATH + '/train_images/video_' + df_train['video_id'].astype(str) + '/' + df_train['video_frame'].astype(str) + \".jpg\"\ndf_train['num_bboxes'] = df_train['annotations'].apply(lambda x: len(x))\ndf_train['diff_num_bboxes'] = df_train['num_bboxes'].diff().fillna(0).astype(int)\ndisplay(df_train)","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:54:03.469041Z","iopub.execute_input":"2021-12-13T10:54:03.469462Z","iopub.status.idle":"2021-12-13T10:54:04.157Z","shell.execute_reply.started":"2021-12-13T10:54:03.46941Z","shell.execute_reply":"2021-12-13T10:54:04.156087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get indexes with 2 or more diff_num_bboxes\nindexes = df_train[abs(df_train['diff_num_bboxes'])>2].index.values\ndisplay(df_train.iloc[indexes])","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:54:04.158522Z","iopub.execute_input":"2021-12-13T10:54:04.158939Z","iopub.status.idle":"2021-12-13T10:54:04.193967Z","shell.execute_reply.started":"2021-12-13T10:54:04.158892Z","shell.execute_reply":"2021-12-13T10:54:04.192702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot number of bboxes\n\nPlot number of bboxes and the diff.","metadata":{}},{"cell_type":"code","source":"def plot_num_bboxes_and_diff(df, video_id):\n    \n    df['num_bboxes_lag1'] = df.shift(1)['num_bboxes']\n    df_video = df[df['video_id']==video_id]\n    \n    sequence_start_idx = df_video[df_video['sequence_frame']==0].index\n    fig, ax = plt.subplots(1, len(sequence_start_idx), figsize=(len(df_video)/600, 3))\n    plt.subplots_adjust(wspace=0.1)\n    start_idx = df_video[:1].index[0]\n    end_idx = df_video[-1:].index[0]\n    for i in range(len(sequence_start_idx-1)):\n        \n        if i < len(sequence_start_idx)-1:\n            width = sequence_start_idx[i+1] - sequence_start_idx[i]\n        else: # last sequence\n            width = end_idx - sequence_start_idx[i]\n            \n        # plot #bbox\n        df_sequence = df.iloc[sequence_start_idx[i]:sequence_start_idx[i] + width]\n        ax[i].plot(df_sequence['video_frame'], df_sequence['num_bboxes'],\n                   linewidth=0.5, color='gray', linestyle='--', label='#bbox')\n        # plot jump of #bbox\n        df_diff_2 = df_sequence[abs(df_sequence['diff_num_bboxes'])==2]\n        df_diff_3 = df_sequence[abs(df_sequence['diff_num_bboxes'])>=3]\n        ax[i].vlines(df_diff_2['video_frame'], ymin=df_diff_2['num_bboxes_lag1'], ymax=df_diff_2['num_bboxes'],\n                     color='orange', alpha=0.5, label='diff = 2')\n        ax[i].vlines(df_diff_3['video_frame'], ymin=df_diff_3['num_bboxes_lag1'], ymax=df_diff_3['num_bboxes'],\n                     color='red',    alpha=0.5, label='diff >= 3')\n            \n        # visual setting\n        ax[i].set_title(f'{i+1}')\n        ax[i].set_position([(sequence_start_idx[i]-start_idx)/len(df_video), 0.05, len(df_sequence)/len(df_video), 0.8])    \n        ax[i].set_yticks(np.arange(0,20,5))\n        ax[i].set_xlabel('video_frame')\n        if width<300:\n            ax[i].get_xaxis().set_visible(False)\n        ax[i].set_xlim([df_sequence['video_frame'].min()-5, df_sequence['video_frame'].max()])\n        ax[i].set_ylim([0,20])\n        ax[i].spines[\"top\"].set_linewidth(0)\n        ax[i].spines[\"right\"].set_linewidth(0)\n        ax[i].spines[\"bottom\"].set_linewidth(1)\n        ax[i].grid(axis='y', linestyle = \"--\")\n        ax[i].tick_params(color='w')\n        if i>=1:\n            ax[i].axes.yaxis.set_ticklabels([])\n            # ax[i].spines[\"left\"].set_linewidth(0)\n\n    ax[0].set_ylabel('Number of bboxes')\n    ax[i].legend(loc=(0.7, 0.5))\n    plt.suptitle(f'video_id: {video_id}', x=0.5, y=1.2, fontsize=15)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:54:04.196077Z","iopub.execute_input":"2021-12-13T10:54:04.196473Z","iopub.status.idle":"2021-12-13T10:54:04.219464Z","shell.execute_reply.started":"2021-12-13T10:54:04.196433Z","shell.execute_reply":"2021-12-13T10:54:04.218427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(3):\n    plot_num_bboxes_and_diff(df=df_train, video_id=i)","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:54:04.221621Z","iopub.execute_input":"2021-12-13T10:54:04.222324Z","iopub.status.idle":"2021-12-13T10:54:05.99351Z","shell.execute_reply.started":"2021-12-13T10:54:04.222273Z","shell.execute_reply":"2021-12-13T10:54:05.992451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🐟Sample image\n\nSampling index = 12637, which has 3 diff. of number of bboxes.","metadata":{}},{"cell_type":"code","source":"sample_idx = 12637\nsample = df_train.iloc[sample_idx]\nprint(sample)","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:54:05.99519Z","iopub.execute_input":"2021-12-13T10:54:05.995726Z","iopub.status.idle":"2021-12-13T10:54:06.003903Z","shell.execute_reply.started":"2021-12-13T10:54:05.99569Z","shell.execute_reply":"2021-12-13T10:54:06.002902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_bboxes(annotations):\n    \"\"\"\n    annotations: list of annotations\n    return: bboxes as [x_min, y_min, x_max, y_max]\n    \"\"\"\n    if len(annotations)==0:\n        return []\n    boxes = pd.DataFrame(annotations, columns=['x', 'y', 'width', 'height']).astype(np.int32).values\n    # [x_min, y_min, w, h] -> [x_min, y_min, x_max, y_max]\n    boxes[:, 2] = boxes[:, 0] + boxes[:, 2]\n    boxes[:, 3] = boxes[:, 1] + boxes[:, 3]\n    return boxes   \n\ndef plot_img_and_bbox(img_path, anntations):\n    img = cv2.imread(img_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    fig, ax = plt.subplots(1, 1, figsize=(16,10))\n    if len(annotations)>0:\n        bboxes = get_bboxes(annotations)\n        for i, box in enumerate(bboxes):\n            # pur bbox on image\n            cv2.rectangle(img,\n                          (box[0], box[1]),\n                          (box[2], box[3]),\n                          color = (255, 0, 0),\n                          thickness = 2)\n            # numbering\n            ax.text(box[0], box[1]-5, i+1, color='red')\n\n    ax.set_axis_off()\n    ax.imshow(img)\n\n\ndef zoom_bbox(img_path, annotations):\n    img = cv2.imread(img_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    bboxes = get_bboxes(annotations)\n    \n    col = 7 if len(bboxes)>=7 else len(bboxes)\n    row = np.ceil(len(bboxes)/7).astype(int) if len(bboxes)>7 else 1\n    fig, ax = plt.subplots(row, col, figsize=(col*2, row*3))\n    cnt = 0\n    for i in range(row):\n        \n        for j in range(col):\n                        \n            bbox = bboxes[cnt]\n            sliced_img = img[bbox[1]:bbox[3], bbox[0]:bbox[2]]\n            \n            if row==1:\n                ax[j].imshow(sliced_img)\n                ax[j].set_title(cnt+1, color='red')\n                ax[j].set_axis_off()\n            else:\n                ax[i,j].imshow(sliced_img)\n                ax[i,j].set_title(cnt+1, color='red')\n                ax[i,j].set_axis_off()\n                \n            cnt += 1\n            \n            if cnt==len(bboxes):\n                break\n    \n        if cnt==len(bboxes):\n            break      \n            \n    plt.show() ","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:54:06.005287Z","iopub.execute_input":"2021-12-13T10:54:06.005629Z","iopub.status.idle":"2021-12-13T10:54:06.025926Z","shell.execute_reply.started":"2021-12-13T10:54:06.005595Z","shell.execute_reply":"2021-12-13T10:54:06.024778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_path    = sample['image_path']\nannotations = sample['annotations']\nprint('image_id:', sample['image_id'])\n# plot image with bboxes\nplot_img_and_bbox(img_path, annotations)\n# plot zoom of bboxes\nzoom_bbox(img_path, annotations)","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:54:06.028442Z","iopub.execute_input":"2021-12-13T10:54:06.029289Z","iopub.status.idle":"2021-12-13T10:54:07.288249Z","shell.execute_reply.started":"2021-12-13T10:54:06.029236Z","shell.execute_reply":"2021-12-13T10:54:07.287603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate 200-frame video around the image with maximum number of bboxes.\nref. https://www.kaggle.com/bamps53/create-annotated-video#kln-23","metadata":{}},{"cell_type":"code","source":"def get_img_with_annotations(img_path, annotations):\n    img = cv2.imread(img_path)\n    video_id = img_path.split('/')[-2].split('_')[-1]\n    frame_id = img_path.split('/')[-1].split('.')[0]\n    img_id = video_id + '-' + frame_id\n    #img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    if len(annotations)>0:\n        bboxes = get_bboxes(annotations)\n        for i, box in enumerate(bboxes):\n            # put bbox\n            cv2.rectangle(img,\n                          (box[0], box[1]),\n                          (box[2], box[3]),\n                          color = (0, 0, 255),\n                          thickness = 2)\n    # put image_id, #bbox\n    cv2.putText(img,\n                f'image_id: {img_id}, #bbox: {len(annotations)}',\n                org = (30, 50), \n                color = (0, 0, 255), \n                fontFace=cv2.FONT_HERSHEY_SIMPLEX,\n                fontScale=1.0,\n                thickness=3)\n    \n    return img\n\ndef make_video(df, video_id, start_frame, end_frame, fps=15, width=WIDTH, height=HEIGHT):\n    '''\n    df          : DataFrame\n    video_id    : 0, 1, or 2\n    start_frame : video_frame at start of video\n    num_frame   : video_frame at end of video\n    return      : path to video\n    '''\n    video_path = f'video_{video_id}_{start_frame}_to_{end_frame}.mp4' # video after encode\n    tmp_path = 'tmp_' + video_path # video before encode (removed after encode)\n    video = cv2.VideoWriter(tmp_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (width, height))\n    \n    df = df[df['video_id']==video_id].reset_index(drop=True)\n    start_idx = df[df['video_frame']==start_frame].index[0]\n    end_idx   = df[df['video_frame']==end_frame].index[0]\n    df = df.iloc[start_idx:end_idx]\n    for idx, row in tqdm(df.iterrows(), total=len(df)):\n        image_path  = row['image_path']\n        annotations = row['annotations']\n        frame = get_img_with_annotations(image_path, annotations)\n        video.write(frame)\n    \n    video.release()\n    \n    if os.path.exists(video_path):\n        os.remove(video_path)\n    \n    # encode by ffmpeg command \n    subprocess.run(\n        ['ffmpeg', \n         '-i', tmp_path, \n         '-loglevel', 'quiet', \n         '-crf', '18', \n         '-preset', 'veryfast', \n         '-vcodec', 'libx264', \n         video_path]\n    )\n    os.remove(tmp_path)\n    \n    return video_path","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:54:07.289281Z","iopub.execute_input":"2021-12-13T10:54:07.289848Z","iopub.status.idle":"2021-12-13T10:54:07.304666Z","shell.execute_reply.started":"2021-12-13T10:54:07.289814Z","shell.execute_reply":"2021-12-13T10:54:07.304022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"video_id    = sample['video_id']\nstart_frame = sample['video_frame'] - 100 # peek before 100 frames\nend_frame   = sample['video_frame'] + 100 # peek after 100 frames\nprint(f'video_id: {video_id}, video_frame: {start_frame} to {end_frame}')\nprint('Create video ...')\nvideo_path = make_video(df_train,\n                        video_id=video_id,\n                        start_frame=start_frame,\n                        end_frame=end_frame)","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:54:07.305887Z","iopub.execute_input":"2021-12-13T10:54:07.306303Z","iopub.status.idle":"2021-12-13T10:54:20.305398Z","shell.execute_reply.started":"2021-12-13T10:54:07.30627Z","shell.execute_reply":"2021-12-13T10:54:20.304163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Video(video_path, width=WIDTH*0.7, height=HEIGHT*0.7)","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:54:20.306986Z","iopub.execute_input":"2021-12-13T10:54:20.307449Z","iopub.status.idle":"2021-12-13T10:54:20.315668Z","shell.execute_reply.started":"2021-12-13T10:54:20.307404Z","shell.execute_reply":"2021-12-13T10:54:20.314602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The change from id=1-9071 to 9072 (around at 6 sec in this video) is small but the number of bboxes jumps up from 4 to 7 as shown below, so some starfishes are not annotated in id=1-9071.","metadata":{}},{"cell_type":"code","source":"def show_jump_of_bbox_num(df, index):\n    fig, ax = plt.subplots(1, 2, figsize=(20,12))\n    for i, idx in enumerate([index-1, index]):\n        img_path    = df.iloc[idx]['image_path']\n        annotations = df.iloc[idx]['annotations']\n        img = get_img_with_annotations(img_path, annotations)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        ax[i].imshow(img)\n        ax[i].set_axis_off()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:54:20.31772Z","iopub.execute_input":"2021-12-13T10:54:20.318074Z","iopub.status.idle":"2021-12-13T10:54:20.328826Z","shell.execute_reply.started":"2021-12-13T10:54:20.318024Z","shell.execute_reply":"2021-12-13T10:54:20.327851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_jump_of_bbox_num(df_train, sample_idx)","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:54:20.330508Z","iopub.execute_input":"2021-12-13T10:54:20.330848Z","iopub.status.idle":"2021-12-13T10:54:21.027759Z","shell.execute_reply.started":"2021-12-13T10:54:20.330798Z","shell.execute_reply":"2021-12-13T10:54:21.025218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Since the metric of this commpetition is F2, false positives for these non-annotated starfishes may be some tolerant.\nUpvote, if found useful.","metadata":{}},{"cell_type":"markdown","source":"# Kindly Upvote, if found useful.","metadata":{}}]}