{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# First Look at the Starfish Dataset\n\nThis notebook was created during a live coding session on twitch. Follow here: https://www.twitch.tv/medallionstallion_/","metadata":{}},{"cell_type":"code","source":"!pip install nb-black > /dev/null\n%load_ext lab_black","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-12-01T01:24:27.228840Z","iopub.execute_input":"2021-12-01T01:24:27.229258Z","iopub.status.idle":"2021-12-01T01:24:39.904700Z","shell.execute_reply.started":"2021-12-01T01:24:27.229138Z","shell.execute_reply":"2021-12-01T01:24:39.903478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom itertools import cycle\nimport matplotlib.pylab as plt\nfrom matplotlib.patches import Rectangle\nimport subprocess\nfrom tqdm.notebook import tqdm\n\nimport cv2\nfrom cv2 import VideoWriter, VideoWriter_fourcc\nimport os\nfrom IPython.display import Video\n\n\nplt.style.use(\"ggplot\")\ncolor_pal = plt.rcParams[\"axes.prop_cycle\"].by_key()[\"color\"]\ncolor_cycle = cycle(plt.rcParams[\"axes.prop_cycle\"].by_key()[\"color\"])","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-01T01:24:42.516551Z","iopub.execute_input":"2021-12-01T01:24:42.516861Z","iopub.status.idle":"2021-12-01T01:24:42.716479Z","shell.execute_reply.started":"2021-12-01T01:24:42.516827Z","shell.execute_reply":"2021-12-01T01:24:42.715635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/tensorflow-great-barrier-reef/train.csv\")\ntest = pd.read_csv(\"../input/tensorflow-great-barrier-reef/test.csv\")\nss = pd.read_csv(\"../input/tensorflow-great-barrier-reef/example_sample_submission.csv\")\n\ntrain.shape, test.shape","metadata":{"execution":{"iopub.status.busy":"2021-12-01T01:24:43.129802Z","iopub.execute_input":"2021-12-01T01:24:43.130115Z","iopub.status.idle":"2021-12-01T01:24:43.212416Z","shell.execute_reply.started":"2021-12-01T01:24:43.130083Z","shell.execute_reply":"2021-12-01T01:24:43.211489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Example of using the submission package.\nThe `greatbarrierreef` package loops over the test set. We have to predict each sample before we can see the next. This will impact how we must design our model.","metadata":{}},{"cell_type":"code","source":"import greatbarrierreef\n\nenv = greatbarrierreef.make_env()  # initialize the environment\niter_test = (\n    env.iter_test()\n)  # an iterator which loops over the test set and sample submission\nfor (pixel_array, sample_prediction_df) in iter_test:\n    break\n    sample_prediction_df[\n        \"annotations\"\n    ] = \"0.5 0 0 100 100\"  # make your predictions here\n    env.predict(sample_prediction_df)  # register your predictions","metadata":{"execution":{"iopub.status.busy":"2021-12-01T01:24:44.480209Z","iopub.execute_input":"2021-12-01T01:24:44.480549Z","iopub.status.idle":"2021-12-01T01:24:44.748242Z","shell.execute_reply.started":"2021-12-01T01:24:44.480511Z","shell.execute_reply":"2021-12-01T01:24:44.747652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Example Image. Can you see the starfish?","metadata":{}},{"cell_type":"code","source":"plt.style.use(\"default\")\nfig, ax = plt.subplots(figsize=(15, 10))\nax.imshow(pixel_array)\nax.axis(\"off\")\nax.set_title(\"Example Image from the Barrier Reef Dataset\", fontsize=14)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-01T01:26:45.458650Z","iopub.execute_input":"2021-12-01T01:26:45.458963Z","iopub.status.idle":"2021-12-01T01:26:46.373951Z","shell.execute_reply.started":"2021-12-01T01:26:45.458928Z","shell.execute_reply":"2021-12-01T01:26:46.373115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Metadata\n- `video_id` - Unique per video. We only have 3 videos in the training dataset.\n- `sequence` - random number to identify a group within the video of uncut footage.\n- `video_frame` - frame number within the entire video\n- `squence_frame` - frame number within the sequence (shot clip from within the video)\n- `image_id` - a combination of video_id + video_frame. Links to the image in the training images directory.\n- `annotations` - bounding boxes for starfish within the given frame.","metadata":{}},{"cell_type":"markdown","source":"## Example of Sequences within the training videos\n\n- Do we actually have 3 different videos?\n- Are the videos just subsets of a single long video?","metadata":{}},{"cell_type":"code","source":"plt.style.use(\"ggplot\")\nfig, axs = plt.subplots(3, 1, figsize=(15, 10), sharex=True, sharey=True)\n\nfor video in [0, 1, 2]:\n    for sequence, d in train.query(\"video_id == @video\").groupby(\"sequence\"):\n        d[\"sequence_frame\"].plot(ax=axs[video], label=f\"Sequence {sequence}\")\n    axs[video].set_title(f\"Video {video}: Sequence Frame vs Video Frame\")\n    axs[video].set_xlabel(\"Video Frame\")\n    axs[video].set_ylabel(\"Sequence Frame\")\n    axs[video].legend(bbox_to_anchor=(1, 1), loc=\"upper left\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-01T01:29:36.286943Z","iopub.execute_input":"2021-12-01T01:29:36.288390Z","iopub.status.idle":"2021-12-01T01:29:37.334930Z","shell.execute_reply.started":"2021-12-01T01:29:36.288332Z","shell.execute_reply":"2021-12-01T01:29:37.333839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# How many Annotations per Frame?\n- Is it different in each video?\n- Is it different in each sequence within a video?","metadata":{}},{"cell_type":"code","source":"train[\"n_annotations\"] = train[\"annotations\"].apply(lambda x: len(eval(x)))\ntrain[\"video_sequence\"] = (\n    train[\"video_id\"].astype(\"str\") + \"_\" + train[\"sequence\"].astype(\"str\")\n)\n\nax = (\n    train.groupby([\"video_sequence\"])[\"sequence_frame\"]\n    .max()\n    .sort_values()\n    .plot(kind=\"barh\", figsize=(12, 7), title=\"Length of Sequences\")\n)\nax.set_xlabel(\"Number of Frames in the Seqence\")","metadata":{"execution":{"iopub.status.busy":"2021-12-01T01:30:23.813780Z","iopub.execute_input":"2021-12-01T01:30:23.814123Z","iopub.status.idle":"2021-12-01T01:30:24.644136Z","shell.execute_reply.started":"2021-12-01T01:30:23.814087Z","shell.execute_reply":"2021-12-01T01:30:24.642880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axs = plt.subplots(1, 2, figsize=(15, 5))\ntrain.groupby([\"video_sequence\"])[\"n_annotations\"].mean().sort_values().plot(\n    kind=\"barh\", title=\"Avg of Annotations\", ax=axs[0], color=next(color_cycle)\n)\n\ntrain.groupby([\"video_sequence\"])[\"n_annotations\"].sum().sort_values().plot(\n    kind=\"barh\", title=\"Total Annotations\", ax=axs[1], color=next(color_cycle)\n)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-01T01:32:53.003804Z","iopub.execute_input":"2021-12-01T01:32:53.004917Z","iopub.status.idle":"2021-12-01T01:32:53.688926Z","shell.execute_reply.started":"2021-12-01T01:32:53.004849Z","shell.execute_reply":"2021-12-01T01:32:53.688022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axs = plt.subplots(1, 3, figsize=(15, 5), sharey=True)\nfor i, d in train.groupby([\"video_id\", \"sequence\"]):\n    d.set_index(\"sequence_frame\")[\"n_annotations\"].plot(ax=axs[i[0]])\n    axs[i[0]].set_title(f\"Video ID: {i[0]} - Sequence {i[1]}\")\nfig.suptitle(\"Number of Annotations per Frame for each Sequence\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-01T01:32:54.427696Z","iopub.execute_input":"2021-12-01T01:32:54.427972Z","iopub.status.idle":"2021-12-01T01:32:55.276452Z","shell.execute_reply.started":"2021-12-01T01:32:54.427936Z","shell.execute_reply":"2021-12-01T01:32:55.275864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Examples of Annotations\nThe below function allows us to plot an image with its annotations","metadata":{"execution":{"iopub.status.busy":"2021-11-28T23:39:45.815086Z","iopub.execute_input":"2021-11-28T23:39:45.815345Z","iopub.status.idle":"2021-11-28T23:39:46.221727Z","shell.execute_reply.started":"2021-11-28T23:39:45.815301Z","shell.execute_reply":"2021-11-28T23:39:46.220884Z"}}},{"cell_type":"code","source":"plt.style.use(\"default\")\n\n\ndef plot_reef_image(\n    image_id,\n    df,\n    ax=None,\n    show_annotations=True,\n    line_width=1,\n    line_color=\"red\",\n    figsize=(30, 5),\n    image_dir=\"../input/tensorflow-great-barrier-reef/train_images/\",\n):\n    \"\"\"\n    Plot reef image. If `show_annotations` is True, create boxes\n    with the annotations for starfish.\n    \"\"\"\n\n    example = df.query(\"image_id == @image_id\")\n    video = example[\"video_id\"].values[0]\n    frame = example[\"video_frame\"].values[0]\n    annotations = eval(example[\"annotations\"].values[0])\n\n    img = plt.imread(f\"{image_dir}video_{video}/{frame}.jpg\")\n    if ax is None:\n        fig, ax = plt.subplots(figsize=figsize)\n    ax.imshow(img)\n    ax.axis(\"off\")\n\n    n_annotations = len(annotations)\n    ax.set_title(f\"image_id: {image_id} ({n_annotations} Starfish)\", fontsize=12)\n\n    if show_annotations:\n        for a in annotations:\n            ax.add_patch(\n                Rectangle(\n                    (a[\"x\"], a[\"y\"]),\n                    a[\"width\"],\n                    a[\"height\"],\n                    lw=line_width,\n                    facecolor=\"none\",\n                    edgecolor=line_color,\n                )\n            )\n\n    return ax","metadata":{"execution":{"iopub.status.busy":"2021-12-01T01:33:14.409664Z","iopub.execute_input":"2021-12-01T01:33:14.410226Z","iopub.status.idle":"2021-12-01T01:33:14.448277Z","shell.execute_reply.started":"2021-12-01T01:33:14.410190Z","shell.execute_reply":"2021-12-01T01:33:14.447377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Find image with the most number of annotations as an example\nimage_id = train.sort_values(\"n_annotations\").tail(1)[\"image_id\"].values[0]\nax = plot_reef_image(image_id, train, line_color=\"red\")","metadata":{"execution":{"iopub.status.busy":"2021-12-01T01:33:15.849470Z","iopub.execute_input":"2021-12-01T01:33:15.849767Z","iopub.status.idle":"2021-12-01T01:33:16.357622Z","shell.execute_reply.started":"2021-12-01T01:33:15.849734Z","shell.execute_reply":"2021-12-01T01:33:16.356619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot A Bunch of Random Images with Annotations","metadata":{}},{"cell_type":"code","source":"fig, axs = plt.subplots(2, 4, figsize=(20, 10))\naxs = axs.flatten()\nimage_ids = train.sample(8, random_state=529)[\"image_id\"].values\n\nfor i, image_id in enumerate(image_ids):\n    plot_reef_image(image_id, train, ax=axs[i])\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-01T01:33:27.637670Z","iopub.execute_input":"2021-12-01T01:33:27.638660Z","iopub.status.idle":"2021-12-01T01:33:30.286064Z","shell.execute_reply.started":"2021-12-01T01:33:27.638606Z","shell.execute_reply":"2021-12-01T01:33:30.285264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Examples with >= 5 Annotations","metadata":{}},{"cell_type":"code","source":"fig, axs = plt.subplots(2, 4, figsize=(20, 10))\naxs = axs.flatten()\nimage_ids = (\n    train.query(\"n_annotations >= 5\").sample(8, random_state=529)[\"image_id\"].values\n)\n\nfor i, image_id in enumerate(image_ids):\n    plot_reef_image(image_id, train, ax=axs[i])\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-01T01:34:18.141395Z","iopub.execute_input":"2021-12-01T01:34:18.142161Z","iopub.status.idle":"2021-12-01T01:34:20.603948Z","shell.execute_reply.started":"2021-12-01T01:34:18.142120Z","shell.execute_reply":"2021-12-01T01:34:20.603310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Examples with 0 annotations","metadata":{}},{"cell_type":"code","source":"fig, axs = plt.subplots(2, 4, figsize=(20, 10))\naxs = axs.flatten()\nimage_ids = (\n    train.query(\"n_annotations == 0\").sample(8, random_state=529)[\"image_id\"].values\n)\n\nfor i, image_id in enumerate(image_ids):\n    plot_reef_image(image_id, train, ax=axs[i])\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-01T01:34:38.854145Z","iopub.execute_input":"2021-12-01T01:34:38.855270Z","iopub.status.idle":"2021-12-01T01:34:41.303283Z","shell.execute_reply.started":"2021-12-01T01:34:38.855201Z","shell.execute_reply":"2021-12-01T01:34:41.301930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create a video by merging images","metadata":{}},{"cell_type":"code","source":"def add_annotations(img, annotations, color=\"red\", thickness=3):\n    \"\"\"\n    Adds annotations to an image using cv2.\n\n    annotations: [list] of dictionaries with the annoation details\n    \"\"\"\n    if color == \"red\":\n        box_color = (0, 0, 255)  # Red\n    elif color == \"black\":\n        box_color = (0, 0, 0)  # Black\n    for a in annotations:\n        cv2.rectangle(\n            img,\n            (a[\"x\"], a[\"y\"]),\n            (a[\"x\"] + a[\"width\"], a[\"y\"] + a[\"height\"]),\n            box_color,\n            thickness=thickness,\n        )\n\n    return img\n\n\ndef create_reef_video(\n    train,\n    video_id,\n    start_video_frame,\n    end_video_frame,\n    annotate=True,\n    output_filename=\"./test.mp4\",\n    FPS=30,\n    image_dir=\"../input/tensorflow-great-barrier-reef/train_images/\",\n):\n\n    width = 1280\n    height = 720\n\n    fourcc = VideoWriter_fourcc(*\"mp4v\")\n\n    temp_fn = output_filename.replace(\".mp4\", \"\") + \"_temp.mp4\"\n\n    video_file = VideoWriter(temp_fn, fourcc, float(FPS), (width, height))\n\n    subset_df = (\n        train.query(\n            \"video_id == @video_id and video_frame >= @start_video_frame and video_frame <= @end_video_frame\"\n        )\n        .reset_index(drop=True)\n        .copy()\n    )\n    for i, example in tqdm(subset_df.iterrows(), total=len(subset_df)):\n        video = example[\"video_id\"]\n        frame = example[\"video_frame\"]\n        image_fn = f\"{image_dir}video_{video}/{frame}.jpg\"\n        img = cv2.imread(image_fn)\n        if annotate:\n            annotations = eval(example[\"annotations\"])\n            img = add_annotations(img, annotations)\n        video_file.write(img)\n\n    video_file.release()\n\n    subprocess.run(\n        [\n            \"ffmpeg\",\n            \"-i\",\n            temp_fn,\n            \"-crf\",\n            \"18\",\n            \"-preset\",\n            \"veryfast\",\n            \"-vcodec\",\n            \"libx264\",\n            output_filename,\n            \"-loglevel\",\n            \"error\",\n        ]\n    )\n\n    os.remove(temp_fn)\n\n    return output_filename","metadata":{"execution":{"iopub.status.busy":"2021-12-01T02:20:20.415223Z","iopub.execute_input":"2021-12-01T02:20:20.415957Z","iopub.status.idle":"2021-12-01T02:20:20.474071Z","shell.execute_reply.started":"2021-12-01T02:20:20.415909Z","shell.execute_reply":"2021-12-01T02:20:20.472999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_reef_video(\n    train,\n    output_filename=\"example-1.mp4\",\n    annotate=True,\n    video_id=1,\n    start_video_frame=9090,\n    end_video_frame=9172,\n)\nVideo(\"example-1.mp4\", width=800)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T02:05:59.489005Z","iopub.execute_input":"2021-12-01T02:05:59.489299Z","iopub.status.idle":"2021-12-01T02:06:04.496387Z","shell.execute_reply.started":"2021-12-01T02:05:59.489268Z","shell.execute_reply":"2021-12-01T02:06:04.495218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_reef_video(\n    train,\n    output_filename=\"example-2.mp4\",\n    annotate=True,\n    video_id=2,\n    start_video_frame=5600,\n    end_video_frame=5800,\n)\nVideo(\"example-2.mp4\", width=900)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T02:13:09.332234Z","iopub.execute_input":"2021-12-01T02:13:09.332530Z","iopub.status.idle":"2021-12-01T02:13:21.911113Z","shell.execute_reply.started":"2021-12-01T02:13:09.332498Z","shell.execute_reply":"2021-12-01T02:13:21.909839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_reef_video(\n    train,\n    output_filename=\"example-3.mp4\",\n    annotate=True,\n    video_id=0,\n    start_video_frame=4500,\n    end_video_frame=4700,\n)\nVideo(\"example-3.mp4\", width=900)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T02:21:27.851562Z","iopub.execute_input":"2021-12-01T02:21:27.852988Z","iopub.status.idle":"2021-12-01T02:21:35.523000Z","shell.execute_reply.started":"2021-12-01T02:21:27.852925Z","shell.execute_reply":"2021-12-01T02:21:35.521862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Full Annotated Videos","metadata":{}},{"cell_type":"code","source":"\nfor video, data in train.groupby(\"video_id\"):\n    print(f'======== Creating Annotated Video {video} ========')\n    start_frame = data[\"video_frame\"].min()\n    end_frame = data[\"video_frame\"].max()\n    create_reef_video(\n        train,\n        output_filename=f\"full_video{video}_annotated.mp4\",\n        annotate=True,\n        video_id=video,\n        start_video_frame=start_frame,\n        end_video_frame=end_frame,\n    )","metadata":{"execution":{"iopub.status.busy":"2021-12-01T02:20:36.183830Z","iopub.execute_input":"2021-12-01T02:20:36.184158Z","iopub.status.idle":"2021-12-01T02:21:24.245801Z","shell.execute_reply.started":"2021-12-01T02:20:36.184125Z","shell.execute_reply":"2021-12-01T02:21:24.244013Z"},"trusted":true},"execution_count":null,"outputs":[]}]}