{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# TensorFlow Great Barrier Reef","metadata":{}},{"cell_type":"markdown","source":"This notebook was built to quickly explore training data from tensorflow-great-barrier-reef competition. Feel free to reuse.\n\n**Objectives**\n1. Load the data\n2. Get a high-level view of the data structure\n3. Get a high-level view of data distribution\n4. Visualise some images with starfishes","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches","metadata":{"execution":{"iopub.status.busy":"2022-01-05T07:55:57.155103Z","iopub.execute_input":"2022-01-05T07:55:57.155723Z","iopub.status.idle":"2022-01-05T07:55:57.184730Z","shell.execute_reply.started":"2022-01-05T07:55:57.155614Z","shell.execute_reply":"2022-01-05T07:55:57.183970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Get the Data","metadata":{}},{"cell_type":"code","source":"# Get the data\n\npath = '../input/tensorflow-great-barrier-reef/'\ntrain = pd.read_csv(path+'train.csv')\ntest = pd.read_csv(path+'test.csv')\n\n# Add images path to data\n\ntrain['image_path'] = '../input/tensorflow-great-barrier-reef/train_images/video_'+train['video_id'].astype(str)+'/'+train['image_id'].apply(lambda x: x.split('-')[1])+'.jpg'\n\n# Reorganise columns\n\ncols = train.columns[:-2].tolist()+['image_path']+[train.columns[-2]]\ntrain = train[cols]\n\nprint('First rows of training data:\\n')\nprint(train.head(), '\\n')\n\nprint('Training data types:\\n')\nprint(train.dtypes)","metadata":{"execution":{"iopub.status.busy":"2022-01-05T07:55:57.187733Z","iopub.execute_input":"2022-01-05T07:55:57.188714Z","iopub.status.idle":"2022-01-05T07:55:57.343661Z","shell.execute_reply.started":"2022-01-05T07:55:57.188659Z","shell.execute_reply":"2022-01-05T07:55:57.342718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Exploratory Data Analysis","metadata":{}},{"cell_type":"markdown","source":"### Training data","metadata":{}},{"cell_type":"code","source":"# How many videos are there in the training set?\n\nprint('Number of unique videos in the training set: {}\\n'.format(train['video_id'].nunique()))\n\n# How many frames are there in each video of the training set?\n\nprint('Number of video frames per video in the training set:'')\ntrain.groupby('video_id', as_index=False)['video_frame'].count()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T07:55:57.345179Z","iopub.execute_input":"2022-01-05T07:55:57.345510Z","iopub.status.idle":"2022-01-05T07:55:57.364556Z","shell.execute_reply.started":"2022-01-05T07:55:57.345466Z","shell.execute_reply":"2022-01-05T07:55:57.363716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# How does the presence of a starfish materialises in the data?\n\nmask_starfish = train['annotations'] != '[]'\ntrain[mask_starfish].head()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T07:55:57.366755Z","iopub.execute_input":"2022-01-05T07:55:57.367775Z","iopub.status.idle":"2022-01-05T07:55:57.386029Z","shell.execute_reply.started":"2022-01-05T07:55:57.367737Z","shell.execute_reply":"2022-01-05T07:55:57.384986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# \"annotations\" is in str() format. Let's transform it as a list\n# It will help later\n\ntrain['annotations'] = train['annotations'].apply(lambda x: eval(x))","metadata":{"execution":{"iopub.status.busy":"2022-01-05T07:55:57.387575Z","iopub.execute_input":"2022-01-05T07:55:57.388357Z","iopub.status.idle":"2022-01-05T07:55:57.740192Z","shell.execute_reply.started":"2022-01-05T07:55:57.388308Z","shell.execute_reply":"2022-01-05T07:55:57.739080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# How are starfishes distributed in the training data?\n\ntrain['is_starfish'] = train['annotations'].apply(lambda x: 1 if len(x)>0 else 0)\n\nplt.figure(figsize=(8, 6))\nplt.title('Distribution of frames showing a starfish [1] vs. frames without starfish [0]')\nplt.xlabel('Presence of a starfish');\nplt.xticks([0, 1]);\nplt.ylabel('Number of video frames');\ntrain['is_starfish'].hist();\nplt.grid(False);","metadata":{"execution":{"iopub.status.busy":"2022-01-05T07:55:57.741572Z","iopub.execute_input":"2022-01-05T07:55:57.741834Z","iopub.status.idle":"2022-01-05T07:55:58.031418Z","shell.execute_reply.started":"2022-01-05T07:55:57.741801Z","shell.execute_reply":"2022-01-05T07:55:58.030425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# How are starfish bounding boxes distributed in the training data?\n\nplt.figure(figsize=(8, 6))\nplt.title('Number of starfishes distribution in the training data')\ntrain[mask_starfish][\"annotations\"].apply(lambda x: len(x)).value_counts().hist();\nplt.xlabel('Number of video frames');\nplt.ylabel('Number of bounding boxes per video frame');\nplt.grid(False)","metadata":{"execution":{"iopub.status.busy":"2022-01-05T07:55:58.032750Z","iopub.execute_input":"2022-01-05T07:55:58.032964Z","iopub.status.idle":"2022-01-05T07:55:58.320989Z","shell.execute_reply.started":"2022-01-05T07:55:58.032937Z","shell.execute_reply":"2022-01-05T07:55:58.319720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualise some random images","metadata":{}},{"cell_type":"code","source":"# Pick a frame in the training data\nframe = pd.DataFrame.sample(train, n=1)\nn_starfish = frame['annotations'].apply(lambda x: len(x)).tolist()[0]\nprint('There is {} starfish(es) in this frame.'.format(n_starfish)) if n_starfish > 0 else print('There is no starfish in this frame.')\n\nplt.figure(figsize=(15, 10))\nimg_path = frame['image_path'].tolist()[0]\nimg = plt.imread(img_path)\nax = plt.gca()\n\nann_mask = train['image_path'] == img_path\nannotations = train[ann_mask]['annotations'].tolist()[0]\nfor bbox in annotations:\n    x, y, w, h = bbox['x'], bbox['y'], bbox['width'], bbox['height']\n    rect = patches.Rectangle((x, y), w, h, linewidth=1, edgecolor='darkorange', facecolor='orange', alpha=.5)\n    ax.add_patch(rect)\nplt.imshow(img);\nplt.axis('off');","metadata":{"execution":{"iopub.status.busy":"2022-01-05T07:55:58.322995Z","iopub.execute_input":"2022-01-05T07:55:58.324099Z","iopub.status.idle":"2022-01-05T07:55:59.057327Z","shell.execute_reply.started":"2022-01-05T07:55:58.324049Z","shell.execute_reply":"2022-01-05T07:55:59.056668Z"},"trusted":true},"execution_count":null,"outputs":[]}]}