{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# References\n* Thanks to DIEGO GOMEZ & BAEK KYUN SHIN\n* https://www.kaggle.com/diegoalejogm/great-barrier-reefs-eda-with-animations\n* https://www.kaggle.com/werooring/basic-eda-starter-for-everyone","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-26T06:16:23.028234Z","iopub.execute_input":"2021-11-26T06:16:23.028539Z","iopub.status.idle":"2021-11-26T06:16:23.033526Z","shell.execute_reply.started":"2021-11-26T06:16:23.028507Z","shell.execute_reply":"2021-11-26T06:16:23.032644Z"}}},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec \nimport seaborn as sns\nfrom PIL import ImageDraw\nfrom matplotlib import animation, rc\nrc('animation', html='jshtml')\n\nimport ast","metadata":{"execution":{"iopub.status.busy":"2021-11-26T10:54:41.264804Z","iopub.execute_input":"2021-11-26T10:54:41.265539Z","iopub.status.idle":"2021-11-26T10:54:41.270301Z","shell.execute_reply.started":"2021-11-26T10:54:41.265487Z","shell.execute_reply":"2021-11-26T10:54:41.269659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/tensorflow-great-barrier-reef/train.csv')\ntest = pd.read_csv('../input/tensorflow-great-barrier-reef/test.csv')\nsample = pd.read_csv('../input/tensorflow-great-barrier-reef/example_sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-11-26T10:54:44.880902Z","iopub.execute_input":"2021-11-26T10:54:44.881561Z","iopub.status.idle":"2021-11-26T10:54:44.951094Z","shell.execute_reply.started":"2021-11-26T10:54:44.881519Z","shell.execute_reply":"2021-11-26T10:54:44.950258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Analyze","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-26T10:54:46.322430Z","iopub.execute_input":"2021-11-26T10:54:46.322738Z","iopub.status.idle":"2021-11-26T10:54:46.336654Z","shell.execute_reply.started":"2021-11-26T10:54:46.322700Z","shell.execute_reply":"2021-11-26T10:54:46.335826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.tail()","metadata":{"execution":{"iopub.status.busy":"2021-11-26T10:54:46.888101Z","iopub.execute_input":"2021-11-26T10:54:46.888504Z","iopub.status.idle":"2021-11-26T10:54:46.901436Z","shell.execute_reply.started":"2021-11-26T10:54:46.888474Z","shell.execute_reply":"2021-11-26T10:54:46.900567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2021-11-26T10:54:47.240586Z","iopub.execute_input":"2021-11-26T10:54:47.241340Z","iopub.status.idle":"2021-11-26T10:54:47.261548Z","shell.execute_reply.started":"2021-11-26T10:54:47.241299Z","shell.execute_reply":"2021-11-26T10:54:47.258078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Checking Duplicates","metadata":{}},{"cell_type":"code","source":"train.duplicated().sum()","metadata":{"execution":{"iopub.status.busy":"2021-11-26T10:54:48.676467Z","iopub.execute_input":"2021-11-26T10:54:48.676885Z","iopub.status.idle":"2021-11-26T10:54:48.692782Z","shell.execute_reply.started":"2021-11-26T10:54:48.676855Z","shell.execute_reply":"2021-11-26T10:54:48.692198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['video_id'].nunique()","metadata":{"execution":{"iopub.status.busy":"2021-11-26T10:54:49.491929Z","iopub.execute_input":"2021-11-26T10:54:49.492430Z","iopub.status.idle":"2021-11-26T10:54:49.499347Z","shell.execute_reply.started":"2021-11-26T10:54:49.492395Z","shell.execute_reply":"2021-11-26T10:54:49.498060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### As you can see, we have totally 3 videos in the training dataset. Now lets see row count for each videos","metadata":{}},{"cell_type":"code","source":"sns.set_theme(style=\"whitegrid\")\nax = sns.countplot(x='video_id', data=train)","metadata":{"execution":{"iopub.status.busy":"2021-11-26T10:54:51.235118Z","iopub.execute_input":"2021-11-26T10:54:51.235433Z","iopub.status.idle":"2021-11-26T10:54:51.463626Z","shell.execute_reply.started":"2021-11-26T10:54:51.235399Z","shell.execute_reply":"2021-11-26T10:54:51.462782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Not all the images have Crown-Of-Thorns Starfish (COTS) for which we have annotations as []","metadata":{}},{"cell_type":"code","source":"for i in range(3):\n    print(\"Video \" + str(i))\n    print(\"Frames with Annotations : \" + str((train[train['video_id'] != i]['annotations'] != '[]').sum()) )\n    print(\"Frames without Annotations : \" + str((train[train['video_id'] == i]['annotations'] != '[]').sum()) )\n    print(\"---------\")","metadata":{"execution":{"iopub.status.busy":"2021-11-26T10:54:52.653346Z","iopub.execute_input":"2021-11-26T10:54:52.653625Z","iopub.status.idle":"2021-11-26T10:54:52.676980Z","shell.execute_reply.started":"2021-11-26T10:54:52.653595Z","shell.execute_reply":"2021-11-26T10:54:52.676362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"code","source":"train.iloc[16].annotations\n# Note the below result is string. We need to convert it to list","metadata":{"execution":{"iopub.status.busy":"2021-11-26T10:54:54.115839Z","iopub.execute_input":"2021-11-26T10:54:54.116456Z","iopub.status.idle":"2021-11-26T10:54:54.122483Z","shell.execute_reply.started":"2021-11-26T10:54:54.116417Z","shell.execute_reply":"2021-11-26T10:54:54.121556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert String to List Type\ntrain['annotations'] = train['annotations'].apply(ast.literal_eval)","metadata":{"execution":{"iopub.status.busy":"2021-11-26T10:54:55.307608Z","iopub.execute_input":"2021-11-26T10:54:55.308399Z","iopub.status.idle":"2021-11-26T10:54:55.611555Z","shell.execute_reply.started":"2021-11-26T10:54:55.308357Z","shell.execute_reply":"2021-11-26T10:54:55.610548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Lets create a feature which have info about number of annotations per image","metadata":{}},{"cell_type":"code","source":"train['num_bboxes'] = train['annotations'].apply(lambda x: len(x))","metadata":{"execution":{"iopub.status.busy":"2021-11-26T10:54:57.782635Z","iopub.execute_input":"2021-11-26T10:54:57.782932Z","iopub.status.idle":"2021-11-26T10:54:57.798580Z","shell.execute_reply.started":"2021-11-26T10:54:57.782898Z","shell.execute_reply":"2021-11-26T10:54:57.797592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Total rows without annotations : {}'.format(train[train['num_bboxes'] == 0]['num_bboxes'].count()))","metadata":{"execution":{"iopub.status.busy":"2021-11-26T10:54:58.994256Z","iopub.execute_input":"2021-11-26T10:54:58.995120Z","iopub.status.idle":"2021-11-26T10:54:59.004276Z","shell.execute_reply.started":"2021-11-26T10:54:58.995079Z","shell.execute_reply":"2021-11-26T10:54:59.003637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Lets see the distribution of number of COTS per image","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (15,8))\nsns.countplot(x=train[train['num_bboxes'] > 0].num_bboxes,data=train)","metadata":{"execution":{"iopub.status.busy":"2021-11-26T10:55:03.889067Z","iopub.execute_input":"2021-11-26T10:55:03.889751Z","iopub.status.idle":"2021-11-26T10:55:04.265820Z","shell.execute_reply.started":"2021-11-26T10:55:03.889699Z","shell.execute_reply":"2021-11-26T10:55:04.264953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[train['annotations'].str.len() > 2]","metadata":{"execution":{"iopub.status.busy":"2021-11-26T10:55:10.148894Z","iopub.execute_input":"2021-11-26T10:55:10.149745Z","iopub.status.idle":"2021-11-26T10:55:10.184224Z","shell.execute_reply.started":"2021-11-26T10:55:10.149708Z","shell.execute_reply":"2021-11-26T10:55:10.183298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validate Images","metadata":{}},{"cell_type":"code","source":"from os import listdir\nfrom PIL import Image\n\ndef validate_images(video_id):\n    path = '/kaggle/input/tensorflow-great-barrier-reef/train_images/video_{}/'.format(video_id)\n    \n    print(\"Verifying that video {} frames are valid...\".format(video_id))\n    for filename in listdir(path):\n        if filename.endswith('.jpg'):\n            try:\n                img = Image.open(path+filename)\n                img.verify() # Verify it is in fact an image\n            except (IOError, SyntaxError) as e:\n                print('Bad file:', filename) # Print out the names of corrupt files\n    print(\"Verified! Video {} has all valid images\".format(video_id))\n\nfor video_id in range(3):\n    validate_images(video_id)","metadata":{"execution":{"iopub.status.busy":"2021-11-26T10:55:12.216740Z","iopub.execute_input":"2021-11-26T10:55:12.217072Z","iopub.status.idle":"2021-11-26T10:58:04.417037Z","shell.execute_reply.started":"2021-11-26T10:55:12.217037Z","shell.execute_reply":"2021-11-26T10:58:04.415853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fetch_image(df, video_id, frame_id):\n    # get frame\n    frame = df[(df['video_id'] == video_id) & (df['video_frame'] == frame_id)].iloc[0]\n    # get bounding_boxes\n    bounding_boxes = frame['annotations']\n    # open image\n    img = Image.open('/kaggle/input/tensorflow-great-barrier-reef/' + f'train_images/video_{video_id}/{frame_id}.jpg')\n\n    for box in bounding_boxes:\n        x0, y0, x1, y1 = (box['x'], box['y'], box['x']+box['width'], box['y']+box['height'])\n        draw = ImageDraw.Draw(img)\n        draw.rectangle( (x0, y0, x1, y1), outline=180, width=5)\n    return img\n\ndef fetch_image_list(df, video_id, num_images, start_frame_idx):\n    image_list = [np.array(fetch_image(df, video_id, start_frame_idx + index)) for index in range(num_images)]\n\n    return image_list","metadata":{"execution":{"iopub.status.busy":"2021-11-26T10:58:04.418874Z","iopub.execute_input":"2021-11-26T10:58:04.419156Z","iopub.status.idle":"2021-11-26T10:58:04.428882Z","shell.execute_reply.started":"2021-11-26T10:58:04.419122Z","shell.execute_reply":"2021-11-26T10:58:04.427869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images = fetch_image_list(train, video_id=0, num_images=80, start_frame_idx=25)\n\nprint(f'Number of images: {len(images)}')","metadata":{"execution":{"iopub.status.busy":"2021-11-26T10:58:04.429953Z","iopub.execute_input":"2021-11-26T10:58:04.430238Z","iopub.status.idle":"2021-11-26T10:58:06.902574Z","shell.execute_reply.started":"2021-11-26T10:58:04.430201Z","shell.execute_reply":"2021-11-26T10:58:06.901700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize COTS Annimation","metadata":{}},{"cell_type":"code","source":"grid = gridspec.GridSpec(4, 2) \nplt.figure(figsize=(18, 20))\n\nidx_list = [0, 5, 10, 15, 20, 25, 30, 35] \n\nfor i, idx in enumerate(idx_list): \n    ax = plt.subplot(grid[i])\n    plt.imshow(images[idx], interpolation='nearest')\n    ax.set_title(f'frame index {idx}')\n    plt.axis('off')","metadata":{"execution":{"iopub.status.busy":"2021-11-26T10:58:06.904537Z","iopub.execute_input":"2021-11-26T10:58:06.905366Z","iopub.status.idle":"2021-11-26T10:58:08.980763Z","shell.execute_reply.started":"2021-11-26T10:58:06.905327Z","shell.execute_reply":"2021-11-26T10:58:08.979326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_animation(imgs, frame_interval=130):\n    fig = plt.figure(figsize=(7, 4))\n    plt.axis('off')\n    img = plt.imshow(imgs[0])\n\n    def animate(i):\n        img.set_array(imgs[i])\n        return [img]\n\n    return animation.FuncAnimation(fig, animate, frames=len(imgs), interval=frame_interval)","metadata":{"execution":{"iopub.status.busy":"2021-11-26T10:58:08.984172Z","iopub.execute_input":"2021-11-26T10:58:08.984683Z","iopub.status.idle":"2021-11-26T10:58:08.991886Z","shell.execute_reply.started":"2021-11-26T10:58:08.984643Z","shell.execute_reply":"2021-11-26T10:58:08.990802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frame_interval = 130 # set smaller number if you want to play fast, otherwise set bigger\n\ncreate_animation(images, frame_interval=frame_interval)","metadata":{"execution":{"iopub.status.busy":"2021-11-26T10:58:08.993188Z","iopub.execute_input":"2021-11-26T10:58:08.993650Z","iopub.status.idle":"2021-11-26T10:58:22.618904Z","shell.execute_reply.started":"2021-11-26T10:58:08.993608Z","shell.execute_reply":"2021-11-26T10:58:22.618080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}