{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\n\nfrom PIL import Image\n\npd.options.display.max_colwidth = 200\nplt.style.use('seaborn')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-05T20:16:13.2257Z","iopub.execute_input":"2021-12-05T20:16:13.225968Z","iopub.status.idle":"2021-12-05T20:16:13.23163Z","shell.execute_reply.started":"2021-12-05T20:16:13.225942Z","shell.execute_reply":"2021-12-05T20:16:13.230723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data exploration","metadata":{}},{"cell_type":"markdown","source":"## 1. Dataset files","metadata":{}},{"cell_type":"code","source":"data_path = '../input/tensorflow-great-barrier-reef'\n!ls {data_path}","metadata":{"execution":{"iopub.status.busy":"2021-12-05T20:16:14.773393Z","iopub.execute_input":"2021-12-05T20:16:14.773681Z","iopub.status.idle":"2021-12-05T20:16:15.576407Z","shell.execute_reply.started":"2021-12-05T20:16:14.773636Z","shell.execute_reply":"2021-12-05T20:16:15.575654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# greatbarrierreef/ : image delivery api\n!ls {os.path.join(data_path, 'greatbarrierreef/')}","metadata":{"execution":{"iopub.status.busy":"2021-12-05T20:16:15.578764Z","iopub.execute_input":"2021-12-05T20:16:15.579837Z","iopub.status.idle":"2021-12-05T20:16:16.395343Z","shell.execute_reply.started":"2021-12-05T20:16:15.579788Z","shell.execute_reply":"2021-12-05T20:16:16.394343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_images/ : training data folders, containing 3 videos folders : video_{video_id}\n!ls {os.path.join(data_path, 'train_images/')}","metadata":{"execution":{"iopub.status.busy":"2021-12-05T20:16:16.3975Z","iopub.execute_input":"2021-12-05T20:16:16.397842Z","iopub.status.idle":"2021-12-05T20:16:17.196808Z","shell.execute_reply.started":"2021-12-05T20:16:16.397796Z","shell.execute_reply":"2021-12-05T20:16:17.195773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# video folder : contains video frames {video_frame_number}.jpg. \n!ls {os.path.join(data_path, 'train_images/video_0/')} | head -n 5","metadata":{"execution":{"iopub.status.busy":"2021-12-05T20:16:17.199498Z","iopub.execute_input":"2021-12-05T20:16:17.199895Z","iopub.status.idle":"2021-12-05T20:16:18.002699Z","shell.execute_reply.started":"2021-12-05T20:16:17.199845Z","shell.execute_reply":"2021-12-05T20:16:18.001884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. train.csv","metadata":{}},{"cell_type":"markdown","source":"### Raw data","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(os.path.join(data_path, 'train.csv'))\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-05T20:16:18.003942Z","iopub.execute_input":"2021-12-05T20:16:18.004312Z","iopub.status.idle":"2021-12-05T20:16:18.045422Z","shell.execute_reply.started":"2021-12-05T20:16:18.004281Z","shell.execute_reply":"2021-12-05T20:16:18.044528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.info()","metadata":{"execution":{"iopub.status.busy":"2021-12-05T20:16:18.046668Z","iopub.execute_input":"2021-12-05T20:16:18.046908Z","iopub.status.idle":"2021-12-05T20:16:18.064829Z","shell.execute_reply.started":"2021-12-05T20:16:18.046878Z","shell.execute_reply":"2021-12-05T20:16:18.064084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n- video_id - ID number of the video the image was part of.\n- sequence - ID of a gap-free subset of a given video.\n- video_frame - The frame number of the image within the video.\n- sequence_frame - The frame number within a given sequence.\n- image_id - ID code for the image, in the format '{video_id}-{video_frame}'\n- annotations - The bounding boxes of any starfish detections in a string format.","metadata":{}},{"cell_type":"markdown","source":"### Starfish annotations example","metadata":{}},{"cell_type":"code","source":"# annotations : list of dict (x = x_min, y = y_min)\ndf_train.loc[df_train['annotations'] != '[]']['annotations'].sample(1).values[0]","metadata":{"execution":{"iopub.status.busy":"2021-12-05T20:16:18.065948Z","iopub.execute_input":"2021-12-05T20:16:18.066172Z","iopub.status.idle":"2021-12-05T20:16:18.075802Z","shell.execute_reply.started":"2021-12-05T20:16:18.066146Z","shell.execute_reply":"2021-12-05T20:16:18.074924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Video size","metadata":{}},{"cell_type":"code","source":"# size\nvideo_ids = df_train['video_id'].unique()\nprint(f'Video count : {len(video_ids)}')\nfor video_id in video_ids:\n    img_path = os.path.join(data_path, 'train_images', f'video_{video_id}', '0.jpg')\n    im = Image.open(img_path)\n    print(f'Video {video_id} : {im.size}, {im.mode}')\n    \nSIZE = (1280, 720)","metadata":{"execution":{"iopub.status.busy":"2021-12-05T20:16:18.07809Z","iopub.execute_input":"2021-12-05T20:16:18.078886Z","iopub.status.idle":"2021-12-05T20:16:18.09194Z","shell.execute_reply.started":"2021-12-05T20:16:18.078849Z","shell.execute_reply":"2021-12-05T20:16:18.090907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Videos","metadata":{}},{"cell_type":"markdown","source":"### Relationship between videos, sequences, frames and annotations","metadata":{"execution":{"iopub.status.busy":"2021-12-05T20:02:48.682659Z","iopub.execute_input":"2021-12-05T20:02:48.682948Z","iopub.status.idle":"2021-12-05T20:02:48.686589Z","shell.execute_reply.started":"2021-12-05T20:02:48.682917Z","shell.execute_reply":"2021-12-05T20:02:48.68573Z"}}},{"cell_type":"code","source":"def get_video_data(df_train):\n    video_data = {}\n    for video_id in df_train['video_id'].unique():\n        df_video = df_train.loc[df_train['video_id'] == video_id]\n        data_sequence = {}\n        video_data[video_id] = {}\n        video_data[video_id]['frames_count'] = 0\n        video_data[video_id]['frames_with_annot_count'] = 0\n        for sequence in df_video['sequence'].unique():\n            df_sequence = df_video.loc[df_video['sequence'] == sequence]\n            seq_annotations = {}\n            seq_annotations['frames_count'] = len(df_sequence)\n            seq_annotations['frames_with_annot_count'] = df_sequence.loc[df_train['annotations'] != '[]']['annotations'].count()\n            data_sequence[sequence] = seq_annotations\n            video_data[video_id]['frames_count'] += seq_annotations['frames_count']\n            video_data[video_id]['frames_with_annot_count'] += seq_annotations['frames_with_annot_count']\n        video_data[video_id]['sequence'] = data_sequence\n    return video_data\n\ndef print_video_data(video_data):\n    for video_id in video_data.keys():\n        frames_count = video_data[video_id]['frames_count']\n        frames_with_annot_count = video_data[video_id]['frames_with_annot_count']\n        print(f'Video {video_id} : {frames_count} frames, {frames_with_annot_count} frames with annotation(s)')\n        for sequence_id in video_data[video_id]['sequence'].keys():\n            frames_count = video_data[video_id]['sequence'][sequence_id]['frames_count']\n            annotations_count = video_data[video_id]['sequence'][sequence_id]['frames_with_annot_count']\n            print(f'  Sequence {sequence_id} : {frames_count} frames, {annotations_count} with annotation(s)')\n        print('\\n')\n\nvideo_data = get_video_data(df_train)\nprint_video_data(video_data)","metadata":{"execution":{"iopub.status.busy":"2021-12-05T20:16:18.393416Z","iopub.execute_input":"2021-12-05T20:16:18.394408Z","iopub.status.idle":"2021-12-05T20:16:18.470937Z","shell.execute_reply.started":"2021-12-05T20:16:18.394366Z","shell.execute_reply":"2021-12-05T20:16:18.470067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_video_data(video_data):\n    plt.style.use('seaborn')\n    fig, axs = plt.subplots(1, 3, figsize=((15, 5)))\n    frames = {f'Video {key}': value['frames_count'] for key, value in video_data.items()}\n    axs[0].bar(frames.keys(), frames.values(), width=0.3)\n    axs[0].set_ylabel('frames')\n    axs[0].set_title('Frames count per video')\n    seq = {f'Video {key}': len(value['sequence']) for key, value in video_data.items()}\n    axs[1].bar(seq.keys(), seq.values(), width=0.3)\n    axs[1].set_ylabel('sequences')\n    axs[1].set_title('Sequences count per video')\n    annot = {f'Video {key}': value['frames_with_annot_count'] for key, value in video_data.items()}\n    axs[2].bar(annot.keys(), annot.values(), width=0.3)\n    axs[2].set_ylabel('annotations')\n    axs[2].set_title('Annotations count per video')\n\nplot_video_data(video_data)\n","metadata":{"execution":{"iopub.status.busy":"2021-12-05T20:16:18.623967Z","iopub.execute_input":"2021-12-05T20:16:18.624266Z","iopub.status.idle":"2021-12-05T20:16:19.029956Z","shell.execute_reply.started":"2021-12-05T20:16:18.62423Z","shell.execute_reply":"2021-12-05T20:16:19.029427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Are the annotations well distributed in the videos?","metadata":{}},{"cell_type":"code","source":"def plot_frame_data(video_data):\n    fig, axs = plt.subplots(1, len(video_data.keys()), figsize=((15, 5)))\n    for video_id in video_data.keys():\n        annot = {f'Seq. {key}': value['frames_with_annot_count'] for key, value in video_data[video_id]['sequence'].items()}\n        no_annot = {f'Seq. {key}': value['frames_count'] - value['frames_with_annot_count'] for key, value in video_data[video_id]['sequence'].items()}\n        width = 0.5 * len(annot) / 8\n        axs[video_id].bar(annot.keys(), annot.values(), width=width, label='annotation(s)')\n        axs[video_id].bar(no_annot.keys(), no_annot.values(), width=width, label='no annotation', bottom=list(annot.values()))\n        axs[video_id].set_ylabel('frames')\n        axs[video_id].tick_params(axis='x', labelrotation=90)\n        axs[video_id].set_title(f'Video {video_id} : annotations count per sequence')\n        axs[video_id].legend()\n\nplot_frame_data(video_data)","metadata":{"execution":{"iopub.status.busy":"2021-12-05T20:16:19.206391Z","iopub.execute_input":"2021-12-05T20:16:19.206721Z","iopub.status.idle":"2021-12-05T20:16:19.840043Z","shell.execute_reply.started":"2021-12-05T20:16:19.206689Z","shell.execute_reply":"2021-12-05T20:16:19.839106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Annotations","metadata":{"execution":{"iopub.status.busy":"2021-12-05T18:12:26.316265Z","iopub.execute_input":"2021-12-05T18:12:26.31654Z","iopub.status.idle":"2021-12-05T18:12:26.541356Z","shell.execute_reply.started":"2021-12-05T18:12:26.316499Z","shell.execute_reply":"2021-12-05T18:12:26.540435Z"}}},{"cell_type":"markdown","source":"### How many starfish are there per frame?","metadata":{}},{"cell_type":"code","source":"def get_annotation_count(df_train):\n    df_train = df_train.sort_values(by=['video_id', 'sequence', 'sequence_frame'])\n    df_train['annots_count'] = df_train['annotations'].apply(lambda annots : len(eval(annots)))\n    return df_train\n    \ndf_train = get_annotation_count(df_train)\ndf_annot = df_train['annots_count'].value_counts()\n\ndef plot_annot_distrib(df_annot):\n    fig, ax = plt.subplots(1, 1, figsize=(8,5))\n    ax.bar(df_annot.index, df_annot, tick_label=df_annot.index)\n    ax.set_ylabel('frames')\n    ax.set_xlabel('annotation count per frame')\n    \nplot_annot_distrib(df_annot)","metadata":{"execution":{"iopub.status.busy":"2021-12-05T20:16:26.86179Z","iopub.execute_input":"2021-12-05T20:16:26.862398Z","iopub.status.idle":"2021-12-05T20:16:27.356106Z","shell.execute_reply.started":"2021-12-05T20:16:26.86236Z","shell.execute_reply":"2021-12-05T20:16:27.355227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### How are the annotations distributed over time?","metadata":{}},{"cell_type":"code","source":"def get_annotation_time(df_train):\n    annot_data = {}\n    for video_id in df_train['video_id'].unique():\n        df_video = df_train.loc[df_train['video_id'] == video_id]\n        annot_data[video_id] = {}\n        for sequence in df_video['sequence'].unique():\n            df_annot_time = df_video.loc[df_video['sequence'] == sequence]\n            df_annot_time = df_annot_time.sort_values(by='sequence_frame')['annots_count']\n            annot_data[video_id][sequence] = df_annot_time.values\n    return annot_data\n\nannotation_time =  get_annotation_time(df_train)\n\ndef plot_annotation_time(annotation_time):\n    for video_id, sequences in annotation_time.items():\n        for sequence, annot in sequences.items():\n            fig, ax = plt.subplots(1, 1, figsize=(15, 2))\n            ax.plot(annot)\n            ax.set_ylabel('annotation count')\n            ax.set_xlabel('time (frame)')\n            ax.set_title(f'Video {video_id}, sequence : {sequence}')\n            \nplot_annotation_time(annotation_time)","metadata":{"execution":{"iopub.status.busy":"2021-12-05T20:16:28.654569Z","iopub.execute_input":"2021-12-05T20:16:28.654922Z","iopub.status.idle":"2021-12-05T20:16:32.495272Z","shell.execute_reply.started":"2021-12-05T20:16:28.654883Z","shell.execute_reply":"2021-12-05T20:16:32.494491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### What is the shape of the bounding box?","metadata":{}},{"cell_type":"code","source":"def get_annotation_pos_and_size(df_train):\n    annot_data = {}\n    for video_id in df_train['video_id'].unique():\n        df_video = df_train.loc[df_train['video_id'] == video_id]\n        annot_data[video_id] = {}\n        for sequence in df_video['sequence'].unique():\n            annots = []\n            df_annot_time = df_video.loc[df_video['sequence'] == sequence]\n            raw_annots = df_annot_time['annotations'].apply(lambda annots : eval(annots)).values\n            annots = [annot for sublist in raw_annots for annot in sublist]\n            annots = [list(annot.values()) for annot in annots]\n            annots = np.array(annots)\n            annot_data[video_id][sequence] = annots\n    return annot_data\n\nannotation_pos_and_size =  get_annotation_pos_and_size(df_train)\n\ndef plot_annotation_pos(annotation_pos_and_size):\n    for video_id, sequences in annotation_pos_and_size.items():\n        for sequence, annot in sequences.items():\n            if annot.shape[0] != 0:\n                fig, ax = plt.subplots(1, 1, figsize=(8, 6))\n                ax.scatter(annot[:,0], annot[:,1], alpha=0.5)\n                ax.set_ylabel('height')\n                ax.set_xlabel('width')\n                ax.set_xbound(0, SIZE[0])\n                ax.set_ybound(0, SIZE[1])\n                ax.set_title(f'Starfish position (video {video_id}, sequence : {sequence})')\n                \ndef plot_annotation_size(annotation_pos_and_size):\n    fig, axs = plt.subplots(3, len(df_train['video_id'].unique()), figsize=(15, 15))\n    idx = 0\n    for video_id, sequences in annotation_pos_and_size.items():\n        width = []\n        height = []\n        ratio_wh = []\n        sequence_id = []\n        for sequence, annot in sequences.items():\n            if annot.shape[0] != 0:\n                sequence_id.append(sequence)\n                width.append(annot[:, 2])\n                height.append(annot[:, 3])\n                ratio_wh.append(annot[:, 2] / annot[:, 3])\n        # plot width\n        axs[0, idx].boxplot(width, labels=sequence_id)\n        axs[0, idx].set_ylabel('height')\n        axs[0, idx].tick_params(axis='x', labelrotation=90)\n        axs[0, idx].set_xlabel('sequence')\n        axs[0, idx].set_title(f'Bounding box width (video {video_id})')\n        # plot height\n        axs[1, idx].boxplot(height, labels=sequence_id)\n        axs[1, idx].set_ylabel('height')\n        axs[1, idx].tick_params(axis='x', labelrotation=90)\n        axs[1, idx].set_xlabel('sequence')\n        axs[1, idx].set_title(f'Bounding box height (video {video_id})')\n        # plot ratio width / height\n        axs[2, idx].boxplot(ratio_wh, labels=sequence_id)\n        axs[2, idx].set_ylabel('ratio')\n        axs[2, idx].tick_params(axis='x', labelrotation=90)\n        axs[2, idx].set_xlabel('sequence')\n        axs[2, idx].set_title(f'Bounding box ratio width / height (video {video_id})')\n        idx += 1\n    fig.tight_layout() \n\nplot_annotation_size(annotation_pos_and_size)\n","metadata":{"execution":{"iopub.status.busy":"2021-12-05T20:16:32.496665Z","iopub.execute_input":"2021-12-05T20:16:32.496971Z","iopub.status.idle":"2021-12-05T20:16:34.81991Z","shell.execute_reply.started":"2021-12-05T20:16:32.496942Z","shell.execute_reply":"2021-12-05T20:16:34.818974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Where are the boundings boxes?","metadata":{}},{"cell_type":"code","source":"plot_annotation_pos(annotation_pos_and_size)","metadata":{"execution":{"iopub.status.busy":"2021-12-05T20:16:34.821232Z","iopub.execute_input":"2021-12-05T20:16:34.821548Z","iopub.status.idle":"2021-12-05T20:16:38.304422Z","shell.execute_reply.started":"2021-12-05T20:16:34.821508Z","shell.execute_reply":"2021-12-05T20:16:38.303693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Finally","metadata":{}},{"cell_type":"code","source":"def get_sample_frames(df_train):\n    samples = []\n    for video_id in df_train['video_id'].unique():\n        df_video = df_train.loc[df_train['video_id'] == video_id]\n        for sequence in df_video['sequence'].unique():\n            df_sample = df_video.loc[df_video['sequence'] == sequence]\n            try:\n                df_sample = df_sample.loc[df_sample['annotations'] != '[]'].sample(1)\n            except:\n                df_sample = df_sample.sample(1)\n            samples.append(df_sample)\n    return samples\n\ndef process_frame(sample):\n    # frame\n    video_id = sample['video_id'].values[0]\n    frame = sample['video_frame'].values[0]\n    sequence = sample['sequence'].values[0]\n    img_path = os.path.join(data_path, 'train_images', f'video_{video_id}', f'{frame}.jpg')\n    frame = np.array(Image.open(img_path))\n    # bounding boxes\n    try:\n        bboxs = eval(sample['annotations'].values[0])\n        bboxs = [list(values.values()) for values in bboxs]\n        bboxs = np.array(bboxs)\n    except: # no bounding box in sequence\n        bboxs = None\n    return frame, bboxs, video_id, sequence\n\ndef display_frame(frame, bboxs, video_id, sequence):\n    plt.style.use('seaborn-dark')\n    fig, ax = plt.subplots(1, 1, figsize=(12, 8))\n    # frame\n    ax.imshow(frame)\n    ax.set_ylabel('height')\n    ax.set_xlabel('width')\n    ax.set_xbound(0, SIZE[0])\n    ax.set_ybound(0, SIZE[1])\n    ax.set_title(f'Frame sample from video {video_id}, sequence : {sequence}')\n    # bounding boxes\n    for bbox in bboxs:\n        rect = patches.Rectangle((bbox[0], bbox[1]), bbox[2], bbox[3], linewidth=3, edgecolor='r', facecolor='none')\n        ax.add_patch(rect)\n    return\n\nsample_frames = get_sample_frames(df_train)\n\nfor sample in sample_frames:\n    frame, bboxs, video_id, sequence = process_frame(sample)\n    display_frame(frame, bboxs, video_id, sequence)","metadata":{"execution":{"iopub.status.busy":"2021-12-05T20:16:38.306155Z","iopub.execute_input":"2021-12-05T20:16:38.306431Z","iopub.status.idle":"2021-12-05T20:16:49.824964Z","shell.execute_reply.started":"2021-12-05T20:16:38.306387Z","shell.execute_reply":"2021-12-05T20:16:49.824055Z"},"trusted":true},"execution_count":null,"outputs":[]}]}