{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 导入第三方库","metadata":{"papermill":{"duration":0.019347,"end_time":"2021-12-05T23:40:04.613757","exception":false,"start_time":"2021-12-05T23:40:04.59441","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport cv2\nimport ast\nimport json\nimport subprocess\nfrom glob import glob\nfrom tqdm.notebook import tqdm\nfrom pprint import pprint\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom IPython.display import Video","metadata":{"papermill":{"duration":0.333154,"end_time":"2021-12-05T23:40:04.967186","exception":false,"start_time":"2021-12-05T23:40:04.634032","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T11:34:54.780895Z","iopub.execute_input":"2021-12-07T11:34:54.781246Z","iopub.status.idle":"2021-12-07T11:34:54.859423Z","shell.execute_reply.started":"2021-12-07T11:34:54.781140Z","shell.execute_reply":"2021-12-07T11:34:54.858648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 参数设置","metadata":{"papermill":{"duration":0.019862,"end_time":"2021-12-05T23:40:05.007433","exception":false,"start_time":"2021-12-05T23:40:04.987571","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Root of input\nINPUT_PATH = '../input/tensorflow-great-barrier-reef'\nHEIGHT = 720 # image height\nWIDTH  = 1280 # image width","metadata":{"papermill":{"duration":0.026963,"end_time":"2021-12-05T23:40:05.054163","exception":false,"start_time":"2021-12-05T23:40:05.0272","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T11:34:54.861009Z","iopub.execute_input":"2021-12-07T11:34:54.861248Z","iopub.status.idle":"2021-12-07T11:34:54.865107Z","shell.execute_reply.started":"2021-12-07T11:34:54.861220Z","shell.execute_reply":"2021-12-07T11:34:54.864289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 输入数据","metadata":{"papermill":{"duration":0.019626,"end_time":"2021-12-05T23:40:05.093565","exception":false,"start_time":"2021-12-05T23:40:05.073939","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df_train = pd.read_csv(INPUT_PATH + '/train.csv')\ndisplay(df_train)\nprint(df_train.info())","metadata":{"papermill":{"duration":0.138483,"end_time":"2021-12-05T23:40:05.252441","exception":false,"start_time":"2021-12-05T23:40:05.113958","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T11:34:54.866066Z","iopub.execute_input":"2021-12-07T11:34:54.866827Z","iopub.status.idle":"2021-12-07T11:34:54.938619Z","shell.execute_reply.started":"2021-12-07T11:34:54.866779Z","shell.execute_reply":"2021-12-07T11:34:54.938032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for video_id in df_train['video_id'].unique():\n    print(f'video_id: {video_id}')\n    print(f'w   annotations:  {sum(df_train[df_train[\"video_id\"]==video_id][\"annotations\"] == \"[]\")}')\n    print(f'w/o annotations:  {sum(df_train[df_train[\"video_id\"]==video_id][\"annotations\"] != \"[]\")}\\n')","metadata":{"papermill":{"duration":0.052673,"end_time":"2021-12-05T23:40:05.326181","exception":false,"start_time":"2021-12-05T23:40:05.273508","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T11:34:54.939519Z","iopub.execute_input":"2021-12-07T11:34:54.940387Z","iopub.status.idle":"2021-12-07T11:34:54.965906Z","shell.execute_reply.started":"2021-12-07T11:34:54.940341Z","shell.execute_reply":"2021-12-07T11:34:54.965019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 将'annotations'的类型从str更改为list\ndf_train['annotations'] = df_train['annotations'].apply(ast.literal_eval) # str -> list\n# 添加列的图像路径和数量的盒子\ndf_train['image_path'] = INPUT_PATH + '/train_images/video_' + df_train['video_id'].astype(str) + '/' + df_train['video_frame'].astype(str) + \".jpg\"\ndf_train['num_bboxes'] = df_train['annotations'].apply(lambda x: len(x))\ndisplay(df_train)","metadata":{"papermill":{"duration":0.499856,"end_time":"2021-12-05T23:40:05.847036","exception":false,"start_time":"2021-12-05T23:40:05.34718","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T11:34:54.968042Z","iopub.execute_input":"2021-12-07T11:34:54.968263Z","iopub.status.idle":"2021-12-07T11:34:55.425606Z","shell.execute_reply.started":"2021-12-07T11:34:54.968234Z","shell.execute_reply":"2021-12-07T11:34:55.424776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_num_bboxes = max(df_train['num_bboxes'])\nindexes = df_train[df_train['num_bboxes']==max_num_bboxes].index.values\nprint(f'Maximum number of bboxes in an image: {max_num_bboxes}')\ndisplay(df_train.iloc[indexes])","metadata":{"papermill":{"duration":0.067698,"end_time":"2021-12-05T23:40:05.93678","exception":false,"start_time":"2021-12-05T23:40:05.869082","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T11:34:55.426776Z","iopub.execute_input":"2021-12-07T11:34:55.426975Z","iopub.status.idle":"2021-12-07T11:34:55.463365Z","shell.execute_reply.started":"2021-12-07T11:34:55.426950Z","shell.execute_reply":"2021-12-07T11:34:55.462768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# indexes[0] 和 indexes[1] 是连续的帧\nindexes = [indexes[0], indexes[2]]","metadata":{"_kg_hide-input":false,"papermill":{"duration":0.029782,"end_time":"2021-12-05T23:40:05.989359","exception":false,"start_time":"2021-12-05T23:40:05.959577","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T11:34:55.464323Z","iopub.execute_input":"2021-12-07T11:34:55.465080Z","iopub.status.idle":"2021-12-07T11:34:55.468395Z","shell.execute_reply.started":"2021-12-07T11:34:55.465044Z","shell.execute_reply":"2021-12-07T11:34:55.467514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 范例图片","metadata":{"papermill":{"duration":0.022843,"end_time":"2021-12-05T23:40:06.035298","exception":false,"start_time":"2021-12-05T23:40:06.012455","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_bboxes(annotations):\n    \"\"\"\n    annotations: list of annotations\n    return: bboxes as [x_min, y_min, x_max, y_max]\n    \"\"\"\n    if len(annotations)==0:\n        return []\n    boxes = pd.DataFrame(annotations, columns=['x', 'y', 'width', 'height']).astype(np.int32).values\n    # [x_min, y_min, w, h] -> [x_min, y_min, x_max, y_max]\n    boxes[:, 2] = boxes[:, 0] + boxes[:, 2]\n    boxes[:, 3] = boxes[:, 1] + boxes[:, 3]\n    return boxes   \n\ndef plot_img_and_bbox(img_path, anntations):\n    img = cv2.imread(img_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    fig, ax = plt.subplots(1, 1, figsize=(16,10))\n    if len(annotations)>0:\n        bboxes = get_bboxes(annotations)\n        for i, box in enumerate(bboxes):\n            # pur bbox on image\n            cv2.rectangle(img,\n                          (box[0], box[1]),\n                          (box[2], box[3]),\n                          color = (255, 0, 0),\n                          thickness = 2)\n            # numbering\n            ax.text(box[0], box[1]-5, i+1, color='red')\n\n    ax.set_axis_off()\n    ax.imshow(img)\n\n\ndef zoom_bbox(img_path, annotations):\n    img = cv2.imread(img_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    bboxes = get_bboxes(annotations)\n    \n    col = 6\n    row = np.ceil(len(bboxes)//6).astype(int)\n    fig, ax = plt.subplots(row, col, figsize=(16,9))\n    cnt = 0\n    for i in range(row):\n        if cnt >= len(bboxes):\n            break\n        for j in range(col):\n            bbox = bboxes[cnt]\n            sliced_img = img[bbox[1]:bbox[3], bbox[0]:bbox[2]]\n            ax[i,j].imshow(sliced_img)\n            ax[i,j].set_title(cnt+1, color='red')\n            ax[i,j].set_axis_off()\n            cnt += 1\n    plt.show() ","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.041658,"end_time":"2021-12-05T23:40:06.099687","exception":false,"start_time":"2021-12-05T23:40:06.058029","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T11:34:55.469768Z","iopub.execute_input":"2021-12-07T11:34:55.469967Z","iopub.status.idle":"2021-12-07T11:34:55.484958Z","shell.execute_reply.started":"2021-12-07T11:34:55.469942Z","shell.execute_reply":"2021-12-07T11:34:55.484294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"samples = df_train.iloc[indexes].copy()\nfor idx, row in samples.iterrows():\n    img_path    = row['image_path']\n    annotations = row['annotations']\n    print('image_id:', row['image_id'])\n    # plot image with bboxes\n    plot_img_and_bbox(img_path, annotations)\n    # plot zoom of bboxes\n    zoom_bbox(img_path, annotations)","metadata":{"papermill":{"duration":3.753632,"end_time":"2021-12-05T23:40:09.875892","exception":false,"start_time":"2021-12-05T23:40:06.12226","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T11:34:55.486040Z","iopub.execute_input":"2021-12-07T11:34:55.486901Z","iopub.status.idle":"2021-12-07T11:34:58.582807Z","shell.execute_reply.started":"2021-12-07T11:34:55.486858Z","shell.execute_reply":"2021-12-07T11:34:58.582186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 制作视频\n生成300帧视频围绕图像与最大数量的bboxes。\n","metadata":{"papermill":{"duration":0.123295,"end_time":"2021-12-05T23:40:10.128423","exception":false,"start_time":"2021-12-05T23:40:10.005128","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_img_with_annotations(img_path, annotations):\n    img = cv2.imread(img_path)\n    video_id = img_path.split('/')[-2].split('_')[-1]\n    frame_id = img_path.split('/')[-1].split('.')[0]\n    img_id = video_id + '-' + frame_id\n    #img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    if len(annotations)>0:\n        bboxes = get_bboxes(annotations)\n        for i, box in enumerate(bboxes):\n            # put bbox\n            cv2.rectangle(img,\n                          (box[0], box[1]),\n                          (box[2], box[3]),\n                          color = (0, 0, 255),\n                          thickness = 2)\n    # put image_id, #bbox\n    cv2.putText(img,\n                f'image_id: {img_id}, #bbox: {len(annotations)}',\n                org = (30, 50), \n                color = (0, 0, 255), \n                fontFace=cv2.FONT_HERSHEY_SIMPLEX,\n                fontScale=1.0,\n                thickness=3)\n    \n    return img\n\ndef make_video(df, video_id, start_frame, end_frame, fps=15, width=WIDTH, height=HEIGHT):\n    '''\n    df          : DataFrame\n    video_id    : 0, 1, or 2\n    start_frame : video_frame at start of video\n    num_frame   : video_frame at end of video\n    return      : path to video\n    '''\n    video_path = f'video_{video_id}_{start_frame}_to_{end_frame}.mp4' # video after encode\n    tmp_path = 'tmp_' + video_path # video before encode (removed after encode)\n    video = cv2.VideoWriter(tmp_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (width, height))\n    \n    df = df[df['video_id']==video_id].reset_index(drop=True)\n    start_idx = df[df['video_frame']==start_frame].index[0]\n    end_idx   = df[df['video_frame']==end_frame].index[0]\n    df = df.iloc[start_idx:end_idx]\n    for idx, row in tqdm(df.iterrows(), total=len(df)):\n        image_path  = row['image_path']\n        annotations = row['annotations']\n        frame = get_img_with_annotations(image_path, annotations)\n        video.write(frame)\n    \n    video.release()\n    \n    if os.path.exists(video_path):\n        os.remove(video_path)\n    \n    # encode by ffmpeg command \n    subprocess.run(\n        ['ffmpeg', \n         '-i', tmp_path, \n         '-loglevel', 'quiet', \n         '-crf', '18', \n         '-preset', 'veryfast', \n         '-vcodec', 'libx264', \n         video_path]\n    )\n    os.remove(tmp_path)\n    \n    return video_path","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.1418,"end_time":"2021-12-05T23:40:10.39351","exception":false,"start_time":"2021-12-05T23:40:10.25171","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T11:34:58.583737Z","iopub.execute_input":"2021-12-07T11:34:58.584288Z","iopub.status.idle":"2021-12-07T11:34:58.597200Z","shell.execute_reply.started":"2021-12-07T11:34:58.584257Z","shell.execute_reply":"2021-12-07T11:34:58.596382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"video_paths = []\nfor idx in indexes:\n    video_id    = df_train.loc[idx, 'video_id']\n    start_frame = df_train.loc[idx, 'video_frame'] - 100 # peek before 100 frames\n    end_frame   = df_train.loc[idx, 'video_frame'] + 200 # peek after 200 frames\n    print(f'video_id: {video_id}, video_frame: {start_frame} to {end_frame}')\n    print('Create video ...')\n    video_path = make_video(df_train,\n                            video_id=video_id,\n                            start_frame=start_frame,\n                            end_frame=end_frame)\n    video_paths.append(video_path)","metadata":{"_kg_hide-input":false,"papermill":{"duration":41.262513,"end_time":"2021-12-05T23:40:51.778026","exception":false,"start_time":"2021-12-05T23:40:10.515513","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T11:34:58.598244Z","iopub.execute_input":"2021-12-07T11:34:58.598465Z","iopub.status.idle":"2021-12-07T11:35:34.906744Z","shell.execute_reply.started":"2021-12-07T11:34:58.598440Z","shell.execute_reply":"2021-12-07T11:35:34.905095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 第一个视频","metadata":{"papermill":{"duration":0.12425,"end_time":"2021-12-05T23:40:52.026684","exception":false,"start_time":"2021-12-05T23:40:51.902434","status":"completed"},"tags":[]}},{"cell_type":"code","source":"Video(video_paths[0], width=WIDTH*0.7, height=HEIGHT*0.7)","metadata":{"papermill":{"duration":0.135958,"end_time":"2021-12-05T23:40:52.287779","exception":false,"start_time":"2021-12-05T23:40:52.151821","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T11:35:34.908840Z","iopub.execute_input":"2021-12-07T11:35:34.909939Z","iopub.status.idle":"2021-12-07T11:35:34.918113Z","shell.execute_reply.started":"2021-12-07T11:35:34.909897Z","shell.execute_reply":"2021-12-07T11:35:34.917008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"font-size: 120%;\">The change from id=1-9071 to 9072 (around at 3 sec in this video) is small but the number of bboxes jumps up from 4 to 7 as shown below, so some starfishes are not annotated in id=1-9071. </span>","metadata":{"papermill":{"duration":0.12328,"end_time":"2021-12-05T23:40:52.53641","exception":false,"start_time":"2021-12-05T23:40:52.41313","status":"completed"},"tags":[]}},{"cell_type":"code","source":"img_ids = ['1-9071', '1-9072']\nfig, ax = plt.subplots(1, 2, figsize=(20,10))\nfor i, img_id in enumerate(img_ids):\n    img_path    = df_train[df_train['image_id'].str.contains(img_id)]['image_path'].values[0]\n    annotations = df_train[df_train['image_id'].str.contains(img_id)]['annotations'].values[0]\n    img = get_img_with_annotations(img_path, annotations)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    ax[i].imshow(img)\n    ax[i].set_axis_off()\nplt.show()","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.89142,"end_time":"2021-12-05T23:40:53.553576","exception":false,"start_time":"2021-12-05T23:40:52.662156","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T11:35:34.919776Z","iopub.execute_input":"2021-12-07T11:35:34.920761Z","iopub.status.idle":"2021-12-07T11:35:35.650192Z","shell.execute_reply.started":"2021-12-07T11:35:34.920714Z","shell.execute_reply":"2021-12-07T11:35:35.649303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 第二个视频","metadata":{"papermill":{"duration":0.149642,"end_time":"2021-12-05T23:40:53.8539","exception":false,"start_time":"2021-12-05T23:40:53.704258","status":"completed"},"tags":[]}},{"cell_type":"code","source":"Video(video_paths[1], width=WIDTH*0.7, height=HEIGHT*0.7)","metadata":{"papermill":{"duration":0.158237,"end_time":"2021-12-05T23:40:54.160426","exception":false,"start_time":"2021-12-05T23:40:54.002189","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T11:35:35.652362Z","iopub.execute_input":"2021-12-07T11:35:35.652575Z","iopub.status.idle":"2021-12-07T11:35:35.658262Z","shell.execute_reply.started":"2021-12-07T11:35:35.652549Z","shell.execute_reply":"2021-12-07T11:35:35.657415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"类似地，在id=2-5715和5721之间，bbox的数量从5个更改为8个","metadata":{"papermill":{"duration":0.147021,"end_time":"2021-12-05T23:40:54.455943","exception":false,"start_time":"2021-12-05T23:40:54.308922","status":"completed"},"tags":[]}},{"cell_type":"code","source":"img_ids = ['2-5715', '2-5721']\nfig, ax = plt.subplots(1, 2, figsize=(20,10))\nfor i, img_id in enumerate(img_ids):\n    img_path    = df_train[df_train['image_id'].str.contains(img_id)]['image_path'].values[0]\n    annotations = df_train[df_train['image_id'].str.contains(img_id)]['annotations'].values[0]\n    img = get_img_with_annotations(img_path, annotations)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    ax[i].imshow(img)\n    ax[i].set_axis_off()\nplt.show()","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.846538,"end_time":"2021-12-05T23:40:55.451167","exception":false,"start_time":"2021-12-05T23:40:54.604629","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T11:35:35.659400Z","iopub.execute_input":"2021-12-07T11:35:35.659912Z","iopub.status.idle":"2021-12-07T11:35:36.319686Z","shell.execute_reply.started":"2021-12-07T11:35:35.659877Z","shell.execute_reply":"2021-12-07T11:35:36.318730Z"},"trusted":true},"execution_count":null,"outputs":[]}]}