{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport os\ninput_dir = '/kaggle/input'\nworking_dir = '/kaggle/working/'\ntemp_dir = '/kaggle/temp/'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-31T16:51:05.258212Z","iopub.execute_input":"2021-12-31T16:51:05.258465Z","iopub.status.idle":"2021-12-31T16:51:05.265843Z","shell.execute_reply.started":"2021-12-31T16:51:05.258439Z","shell.execute_reply":"2021-12-31T16:51:05.264997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## (A) Train CSV","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/tensorflow-great-barrier-reef/train.csv')\nprint(train_df.shape)\nprint(train_df.isna().sum())\ntrain_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T16:51:06.059794Z","iopub.execute_input":"2021-12-31T16:51:06.060076Z","iopub.status.idle":"2021-12-31T16:51:06.114900Z","shell.execute_reply.started":"2021-12-31T16:51:06.060043Z","shell.execute_reply":"2021-12-31T16:51:06.114164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## (A) 1- Let's see some metadata stats for a single video_id","metadata":{}},{"cell_type":"code","source":"def get_sequence_stats(dataframe):\n    num_seq_dataframe = len(dataframe.sequence.unique())\n    print(f'Number of sequences : {num_seq_dataframe}')\n    print('{Sequence ID}   {Num of imgs in Sequence (or Sequence length)}')\n    print(dataframe.sequence.value_counts(sort=False).sort_index())\n    fig, ax = plt.subplots()\n    dataframe.sequence.plot(ax=ax)\n    plt.show()\n\ndef get_video_frame_stats(dataframe):\n    print(f'Number of video_frames : {len(dataframe.video_frame.unique())}')\n    fig, ax = plt.subplots()\n    dataframe.video_frame.plot(ax=ax)\n    plt.show()\n    \ndef get_annotation_stats(dataframe):\n    dataframe['bbox_count'] = dataframe.annotations.apply(lambda x: x.count('{'))\n    print('{BBox Count}   {Num of images with this many BBoxes}')\n    print(dataframe['bbox_count'].value_counts(sort=False).sort_index())\n    print(f'Total Number of bboxes : {dataframe.bbox_count.sum()}')\n    num_img_with_no_bbox = (dataframe.bbox_count == 0).sum()\n    num_img_with_bbox = (dataframe.bbox_count != 0).sum()\n    print(f'Total Number of images (or video_frames) : {dataframe.shape[0]}')\n    print(f'Total Number of images having non-empty annotation (i.e no bbox) : {num_img_with_no_bbox}')\n    print(f'Total Number of images having non-empty annotation : {num_img_with_bbox}')\n    dataframe_with_bbox = get_parsed_annotation(dataframe[dataframe.bbox_count != 0].copy())\n    print(f'BBoxes Stats (for img having bboxes): ')\n    bbox_stats(dataframe_with_bbox.copy())\n    \ndef bbox_stats(dataframe):\n    bbox_area = []\n    for height_list, width_list in zip(dataframe.height, dataframe.width):\n        for height, width in zip(height_list, width_list):\n            bbox_area.append(height*width)\n    bbox_area = pd.Series(bbox_area)\n    print(f'Mean of bbox_area : {bbox_area.mean()}')\n    print(f'Median of bbox_area : {bbox_area.median()}')\n    fig, ax = plt.subplots()\n    bbox_area.plot(ax=ax)\n    plt.show()\n    fig, ax = plt.subplots()\n    bbox_area.hist(ax=ax)\n    plt.show()\n    \ndef get_parsed_annotation(dataframe):\n    dataframe['parsed_annotation'] = dataframe.annotations.apply(lambda x: list(eval(x)))\n    dataframe['width'] = dataframe.parsed_annotation.apply(lambda x: [i['width'] for i in x])\n    dataframe['height'] = dataframe.parsed_annotation.apply(lambda x: [i['height'] for i in x])\n    #dataframe['bbox_area'] = dataframe.apply(lambda x: [width*height for height, width in zip(x.width, x.height)], axis=1)\n    return dataframe\n    \ndef get_metadata_stats(dataframe):\n    print(f'Total number of videos : {len(dataframe.video_id.unique())}')\n    print()\n    print(f'Cummulative sequence stats : ')\n    get_sequence_stats(dataframe.copy())\n    print()\n    print(f'Cummulative annotation stats : ')\n    get_annotation_stats(dataframe.copy())\n    print()\n    print('- - - - - - - - - - - - - - - - - - - -')\n    \n    # get stats for each video\n    for video_id in dataframe.video_id.unique():\n        local_dataframe = dataframe[dataframe.video_id == video_id]\n        print()\n        print(f'Video ID : {video_id}')\n        print()\n        print(f'Cummulative sequence stats : ')\n        get_sequence_stats(local_dataframe.copy())\n        print()\n        print(f'Cummulative video_frame stats : ')\n        get_video_frame_stats(local_dataframe.copy())\n        print()\n        print(f'Cummulative annotation stats : ')\n        get_annotation_stats(local_dataframe.copy())\n        print()\n        print('# # # # # # # # # # # # # # # # # # # # # #')\n        ","metadata":{"execution":{"iopub.status.busy":"2021-12-31T17:09:04.363667Z","iopub.execute_input":"2021-12-31T17:09:04.363936Z","iopub.status.idle":"2021-12-31T17:09:04.384125Z","shell.execute_reply.started":"2021-12-31T17:09:04.363906Z","shell.execute_reply":"2021-12-31T17:09:04.383109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_metadata_stats(train_df.copy())","metadata":{"execution":{"iopub.status.busy":"2021-12-31T17:09:05.135473Z","iopub.execute_input":"2021-12-31T17:09:05.135761Z","iopub.status.idle":"2021-12-31T17:09:08.583927Z","shell.execute_reply.started":"2021-12-31T17:09:05.135732Z","shell.execute_reply":"2021-12-31T17:09:08.583133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### (A) 2 - Checking Images","metadata":{}},{"cell_type":"code","source":"from PIL import Image\n\nimg_dir = '/kaggle/input/tensorflow-great-barrier-reef/train_images'\n\ntrain_img_dim = {}\nfor video_dir in os.listdir(img_dir):\n    images = os.listdir(os.path.join(img_dir, video_dir))\n    for img in images:\n        image_path = os.path.join(img_dir, video_dir, img)\n        img_pil = Image.open(image_path)\n        size = img_pil.size\n        if size in train_img_dim:\n            train_img_dim[size] += 1\n        else:\n            train_img_dim[size] = 1\nprint(train_img_dim)\n        ","metadata":{"execution":{"iopub.status.busy":"2021-12-31T17:17:40.333786Z","iopub.execute_input":"2021-12-31T17:17:40.334285Z","iopub.status.idle":"2021-12-31T17:22:56.986895Z","shell.execute_reply.started":"2021-12-31T17:17:40.334232Z","shell.execute_reply":"2021-12-31T17:22:56.985906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}