{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\n

Basic EDA Starter for Everyone! \n

Upvote is Free 😊\n

","metadata":{"execution":{"iopub.status.busy":"2021-11-24T07:47:52.939647Z","iopub.execute_input":"2021-11-24T07:47:52.939969Z","iopub.status.idle":"2021-11-24T07:47:52.963309Z","shell.execute_reply.started":"2021-11-24T07:47:52.939889Z","shell.execute_reply":"2021-11-24T07:47:52.962371Z"}}},{"cell_type":"markdown","source":"
\n \"coral\"\n
","metadata":{}},{"cell_type":"markdown","source":"#### References\n- https://www.kaggle.com/diegoalejogm/great-barrier-reefs-eda-with-animations\n- https://www.kaggle.com/debarshichanda/w-b-tables-great-barrier-reef-eda\n\n- Specially thanks to Diego Gomez! I took lots of codes from his notebook.","metadata":{}},{"cell_type":"markdown","source":"
\n

â‘  Load Train Data

","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\nDATA_PATH = '/kaggle/input/tensorflow-great-barrier-reef/'\ntrain = pd.read_csv(DATA_PATH + 'train.csv')\ntrain","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

â‘¡ Take A Look At Data

","metadata":{}},{"cell_type":"markdown","source":"## Basic information about train data","metadata":{}},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check duplicated data","metadata":{}},{"cell_type":"code","source":"train.duplicated().sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There is no duplicated data in train data","metadata":{}},{"cell_type":"markdown","source":"## Feature summary","metadata":{}},{"cell_type":"code","source":"def resumetable(df):\n '''function to create feature summary'''\n print(f'Shape: {df.shape}')\n summary = pd.DataFrame(df.dtypes, columns=['Data Type'])\n summary = summary.reset_index()\n summary = summary.rename(columns={'index': 'Features'})\n summary['Num of Null Value'] = df.isnull().sum().values\n summary['Num of Unique Value'] = df.nunique().values\n summary['1st Value'] = df.loc[0].values\n summary['2nd Value'] = df.loc[1].values\n summary['3rd Value'] = df.loc[2].values\n return summary","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resumetable(train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

â‘¢ Basic Engineering

","metadata":{}},{"cell_type":"markdown","source":"## Downcast","metadata":{}},{"cell_type":"code","source":"def downcast(df, verbose=True):\n start_mem = df.memory_usage().sum() / 1024**2\n for col in df.columns:\n dtype_name = df[col].dtype.name\n if dtype_name == 'object':\n pass\n elif dtype_name == 'bool':\n df[col] = df[col].astype('int8')\n elif dtype_name.startswith('int') or (df[col].round() == df[col]).all():\n df[col] = pd.to_numeric(df[col], downcast='integer')\n else:\n df[col] = pd.to_numeric(df[col], downcast='float')\n end_mem = df.memory_usage().sum() / 1024**2\n if verbose:\n print('{:.1f}% Compressed'.format(100 * (start_mem - end_mem) / start_mem))\n \n return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = downcast(train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"import ast\n\n# Convert String to List Type\ntrain['annotations'] = train['annotations'].apply(ast.literal_eval)\n\n# Get the number of bounding boxes for each image\ntrain['num_bboxes'] = train['annotations'].apply(lambda x: len(x))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check number of frames with bounding boxes","metadata":{}},{"cell_type":"code","source":"train[train['num_bboxes'] > 0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

â‘£ Verify if there is corrupted data

","metadata":{}},{"cell_type":"code","source":"from os import listdir\nfrom PIL import Image\n\ndef verify_images(video_id):\n path = DATA_PATH + f'train_images/video_{video_id}/' \n for filename in listdir(path):\n if filename.endswith('.jpg'):\n try:\n img = Image.open(path + filename)\n img.verify() # Verify it is in fact an image\n except (IOError, SyntaxError) as e:\n print('Bad file:', filename) # Print out the names of corrupt files\n print(f'Video {video_id} has all valid images. Verified!')\n\nfor video_id in range(3):\n verify_images(video_id)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

⑤ Plot Frame Image with Bounding Boxes

","metadata":{}},{"cell_type":"markdown","source":"## Load sequence of images with annotations","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom PIL import ImageDraw\n\ndef fetch_image(df, video_id, frame_id):\n # get frame\n frame = df[(df['video_id'] == video_id) & (df['video_frame'] == frame_id)].iloc[0]\n # get bounding_boxes\n bounding_boxes = frame['annotations']\n # open image\n img = Image.open(DATA_PATH + f'train_images/video_{video_id}/{frame_id}.jpg')\n\n for box in bounding_boxes:\n x0, y0, x1, y1 = (box['x'], box['y'], box['x']+box['width'], box['y']+box['height'])\n draw = ImageDraw.Draw(img)\n draw.rectangle( (x0, y0, x1, y1), outline=180, width=5)\n return img\n\ndef fetch_image_list(df, video_id, num_images, start_frame_idx):\n image_list = [np.array(fetch_image(df, video_id, start_frame_idx + index)) for index in range(num_images)]\n\n return image_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images = fetch_image_list(train, video_id=0, num_images=80, start_frame_idx=25)\n\nprint(f'Number of images: {len(images)}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec \n\ngrid = gridspec.GridSpec(4, 2) \nplt.figure(figsize=(18, 20))\n\nidx_list = [0, 5, 10, 15, 20, 25, 30, 35] \n\nfor i, idx in enumerate(idx_list): \n ax = plt.subplot(grid[i])\n plt.imshow(images[idx], interpolation='nearest')\n ax.set_title(f'frame index {idx}')\n plt.axis('off')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

â‘¥ Image Animation

","metadata":{}},{"cell_type":"code","source":"from matplotlib import animation, rc\nrc('animation', html='jshtml')\n\ndef create_animation(imgs, frame_interval=130):\n fig = plt.figure(figsize=(7, 4))\n plt.axis('off')\n img = plt.imshow(imgs[0])\n\n def animate(i):\n img.set_array(imgs[i])\n return [img]\n\n return animation.FuncAnimation(fig, animate, frames=len(imgs), interval=frame_interval)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frame_interval = 130 # set smaller number if you want to play fast, otherwise set bigger\n\ncreate_animation(images, frame_interval=frame_interval)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}