{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\n\ntrain = pd.read_csv('../input/tensorflow-great-barrier-reef/train.csv')\ntrain","metadata":{"papermill":{"duration":0.107471,"end_time":"2021-12-03T13:22:20.365389","exception":false,"start_time":"2021-12-03T13:22:20.257918","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-05T10:18:33.093263Z","iopub.execute_input":"2021-12-05T10:18:33.093572Z","iopub.status.idle":"2021-12-05T10:18:33.168539Z","shell.execute_reply.started":"2021-12-05T10:18:33.093534Z","shell.execute_reply":"2021-12-05T10:18:33.167647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"papermill":{"duration":0.057284,"end_time":"2021-12-03T13:22:20.525238","exception":false,"start_time":"2021-12-03T13:22:20.467954","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-05T10:18:33.170409Z","iopub.execute_input":"2021-12-05T10:18:33.170783Z","iopub.status.idle":"2021-12-05T10:18:33.200182Z","shell.execute_reply.started":"2021-12-05T10:18:33.170744Z","shell.execute_reply":"2021-12-05T10:18:33.199260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.duplicated().sum()","metadata":{"papermill":{"duration":0.044714,"end_time":"2021-12-03T13:22:20.633666","exception":false,"start_time":"2021-12-03T13:22:20.588952","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-05T10:18:33.201710Z","iopub.execute_input":"2021-12-05T10:18:33.202205Z","iopub.status.idle":"2021-12-05T10:18:33.224066Z","shell.execute_reply.started":"2021-12-05T10:18:33.202165Z","shell.execute_reply":"2021-12-05T10:18:33.223366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resumetable(df):\n    '''function to create feature summary'''\n    print(f'Shape: {df.shape}')\n    summary = pd.DataFrame(df.dtypes, columns=['Data Type'])\n    summary = summary.reset_index()\n    summary = summary.rename(columns={'index': 'Features'})\n    summary['Num of Null Value'] = df.isnull().sum().values\n    summary['Num of Unique Value'] = df.nunique().values\n    summary['1st Value'] = df.loc[0].values\n    summary['2nd Value'] = df.loc[1].values\n    summary['3rd Value'] = df.loc[2].values\n    return summary","metadata":{"papermill":{"duration":0.031874,"end_time":"2021-12-03T13:22:20.730077","exception":false,"start_time":"2021-12-03T13:22:20.698203","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-05T10:18:33.225114Z","iopub.execute_input":"2021-12-05T10:18:33.225481Z","iopub.status.idle":"2021-12-05T10:18:33.232569Z","shell.execute_reply.started":"2021-12-05T10:18:33.225429Z","shell.execute_reply":"2021-12-05T10:18:33.231578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resumetable(train)","metadata":{"papermill":{"duration":0.062216,"end_time":"2021-12-03T13:22:20.813502","exception":false,"start_time":"2021-12-03T13:22:20.751286","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-05T10:18:33.235309Z","iopub.execute_input":"2021-12-05T10:18:33.235992Z","iopub.status.idle":"2021-12-05T10:18:33.269506Z","shell.execute_reply.started":"2021-12-05T10:18:33.235955Z","shell.execute_reply":"2021-12-05T10:18:33.268756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def downcast(df, verbose=True):\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in df.columns:\n        dtype_name = df[col].dtype.name\n        if dtype_name == 'object':\n            pass\n        elif dtype_name == 'bool':\n            df[col] = df[col].astype('int8')\n        elif dtype_name.startswith('int') or (df[col].round() == df[col]).all():\n            df[col] = pd.to_numeric(df[col], downcast='integer')\n        else:\n            df[col] = pd.to_numeric(df[col], downcast='float')\n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose:\n        print('{:.1f}% Compressed'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"papermill":{"duration":0.033934,"end_time":"2021-12-03T13:22:20.960142","exception":false,"start_time":"2021-12-03T13:22:20.926208","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-05T10:18:33.270749Z","iopub.execute_input":"2021-12-05T10:18:33.271463Z","iopub.status.idle":"2021-12-05T10:18:33.281360Z","shell.execute_reply.started":"2021-12-05T10:18:33.271409Z","shell.execute_reply":"2021-12-05T10:18:33.280420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = downcast(train)","metadata":{"papermill":{"duration":0.043073,"end_time":"2021-12-03T13:22:21.025434","exception":false,"start_time":"2021-12-03T13:22:20.982361","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-05T10:18:33.282644Z","iopub.execute_input":"2021-12-05T10:18:33.283071Z","iopub.status.idle":"2021-12-05T10:18:33.301595Z","shell.execute_reply.started":"2021-12-05T10:18:33.283035Z","shell.execute_reply":"2021-12-05T10:18:33.300839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import ast\n\n# Convert String to List Type\ntrain['annotations'] = train['annotations'].apply(ast.literal_eval)\n\n# Get the number of bounding boxes for each image\ntrain['num_bboxes'] = train['annotations'].apply(lambda x: len(x))","metadata":{"papermill":{"duration":0.520099,"end_time":"2021-12-03T13:22:21.614405","exception":false,"start_time":"2021-12-03T13:22:21.094306","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-05T10:18:33.302722Z","iopub.execute_input":"2021-12-05T10:18:33.303097Z","iopub.status.idle":"2021-12-05T10:18:33.630070Z","shell.execute_reply.started":"2021-12-05T10:18:33.303063Z","shell.execute_reply":"2021-12-05T10:18:33.629367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"papermill":{"duration":0.040951,"end_time":"2021-12-03T13:22:21.679128","exception":false,"start_time":"2021-12-03T13:22:21.638177","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-05T10:18:33.632243Z","iopub.execute_input":"2021-12-05T10:18:33.632666Z","iopub.status.idle":"2021-12-05T10:18:33.646353Z","shell.execute_reply.started":"2021-12-05T10:18:33.632629Z","shell.execute_reply":"2021-12-05T10:18:33.645692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[train['num_bboxes'] > 0]","metadata":{"papermill":{"duration":0.05202,"end_time":"2021-12-03T13:22:21.802225","exception":false,"start_time":"2021-12-03T13:22:21.750205","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-05T10:18:33.649276Z","iopub.execute_input":"2021-12-05T10:18:33.649498Z","iopub.status.idle":"2021-12-05T10:18:33.675013Z","shell.execute_reply.started":"2021-12-05T10:18:33.649467Z","shell.execute_reply":"2021-12-05T10:18:33.674388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from os import listdir\nfrom PIL import Image\nDATA_PATH = '/kaggle/input/tensorflow-great-barrier-reef/'\ndef verify_images(video_id):\n    path = DATA_PATH + f'train_images/video_{video_id}/'    \n    for filename in listdir(path):\n        if filename.endswith('.jpg'):\n            try:\n                img = Image.open(path + filename)\n                img.verify() # Verify it is in fact an image\n            except (IOError, SyntaxError) as e:\n                print('Bad file:', filename) # Print out the names of corrupt files\n    print(f'Video {video_id} has all valid images. Verified!')\n\nfor video_id in range(3):\n    verify_images(video_id)","metadata":{"papermill":{"duration":245.755986,"end_time":"2021-12-03T13:26:27.629718","exception":false,"start_time":"2021-12-03T13:22:21.873732","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-05T10:18:33.677222Z","iopub.execute_input":"2021-12-05T10:18:33.677792Z","iopub.status.idle":"2021-12-05T10:22:34.986801Z","shell.execute_reply.started":"2021-12-05T10:18:33.677763Z","shell.execute_reply":"2021-12-05T10:22:34.986054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom PIL import ImageDraw\n\ndef fetch_image(df, video_id, frame_id):\n    # get frame\n    frame = df[(df['video_id'] == video_id) & (df['video_frame'] == frame_id)].iloc[0]\n    # get bounding_boxes\n    bounding_boxes = frame['annotations']\n    # open image\n    img = Image.open(DATA_PATH + f'train_images/video_{video_id}/{frame_id}.jpg')\n\n    for box in bounding_boxes:\n        x0, y0, x1, y1 = (box['x'], box['y'], box['x']+box['width'], box['y']+box['height'])\n        draw = ImageDraw.Draw(img)\n        draw.rectangle( (x0, y0, x1, y1), outline=180, width=5)\n    return img\n\ndef fetch_image_list(df, video_id, num_images, start_frame_idx):\n    image_list = [np.array(fetch_image(df, video_id, start_frame_idx + index)) for index in range(num_images)]\n\n    return image_list","metadata":{"papermill":{"duration":0.039856,"end_time":"2021-12-03T13:26:27.790969","exception":false,"start_time":"2021-12-03T13:26:27.751113","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-05T10:22:34.988089Z","iopub.execute_input":"2021-12-05T10:22:34.988342Z","iopub.status.idle":"2021-12-05T10:22:34.996618Z","shell.execute_reply.started":"2021-12-05T10:22:34.988306Z","shell.execute_reply":"2021-12-05T10:22:34.995900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images = fetch_image_list(train, video_id=0, num_images=80, start_frame_idx=25)\n\nprint(f'Number of images: {len(images)}')","metadata":{"papermill":{"duration":2.784189,"end_time":"2021-12-03T13:26:30.600035","exception":false,"start_time":"2021-12-03T13:26:27.815846","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-05T10:22:34.998234Z","iopub.execute_input":"2021-12-05T10:22:34.998723Z","iopub.status.idle":"2021-12-05T10:22:37.493943Z","shell.execute_reply.started":"2021-12-05T10:22:34.998685Z","shell.execute_reply":"2021-12-05T10:22:37.493133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec \n\ngrid = gridspec.GridSpec(4, 2) \nplt.figure(figsize=(18, 20))\n\nidx_list = [0, 5, 10, 15, 20, 25, 30, 35] \n\nfor i, idx in enumerate(idx_list): \n    ax = plt.subplot(grid[i])\n    plt.imshow(images[idx], interpolation='nearest')\n    ax.set_title(f'frame index {idx}')\n    plt.axis('off')","metadata":{"papermill":{"duration":1.5997,"end_time":"2021-12-03T13:26:32.224397","exception":false,"start_time":"2021-12-03T13:26:30.624697","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-05T10:22:37.496525Z","iopub.execute_input":"2021-12-05T10:22:37.497229Z","iopub.status.idle":"2021-12-05T10:22:38.729255Z","shell.execute_reply.started":"2021-12-05T10:22:37.497189Z","shell.execute_reply":"2021-12-05T10:22:38.728367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib import animation, rc\nrc('animation', html='jshtml')\n\ndef create_animation(imgs, frame_interval=130):\n    fig = plt.figure(figsize=(7, 4))\n    plt.axis('off')\n    img = plt.imshow(imgs[0])\n\n    def animate(i):\n        img.set_array(imgs[i])\n        return [img]\n\n    return animation.FuncAnimation(fig, animate, frames=len(imgs), interval=frame_interval)","metadata":{"papermill":{"duration":0.11891,"end_time":"2021-12-03T13:26:32.683109","exception":false,"start_time":"2021-12-03T13:26:32.564199","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-05T10:22:38.730422Z","iopub.execute_input":"2021-12-05T10:22:38.730709Z","iopub.status.idle":"2021-12-05T10:22:38.742924Z","shell.execute_reply.started":"2021-12-05T10:22:38.730669Z","shell.execute_reply":"2021-12-05T10:22:38.742015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frame_interval = 130 # set smaller number if you want to play fast, otherwise set bigger\n\ncreate_animation(images, frame_interval=frame_interval)","metadata":{"papermill":{"duration":12.048573,"end_time":"2021-12-03T13:26:44.835416","exception":false,"start_time":"2021-12-03T13:26:32.786843","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-05T10:22:38.744262Z","iopub.execute_input":"2021-12-05T10:22:38.744645Z","iopub.status.idle":"2021-12-05T10:22:48.668908Z","shell.execute_reply.started":"2021-12-05T10:22:38.744606Z","shell.execute_reply":"2021-12-05T10:22:48.668218Z"},"trusted":true},"execution_count":null,"outputs":[]}]}