{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#导入数据","metadata":{"papermill":{"duration":0.015354,"end_time":"2021-11-24T12:02:38.250327","exception":false,"start_time":"2021-11-24T12:02:38.234973","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import pandas as pd\n\ntrain = pd.read_csv('../input/tensorflow-great-barrier-reef/train.csv')\ntrain","metadata":{"papermill":{"duration":0.093043,"end_time":"2021-11-24T12:02:38.358878","exception":false,"start_time":"2021-11-24T12:02:38.265835","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T12:07:12.145725Z","iopub.execute_input":"2021-12-07T12:07:12.147810Z","iopub.status.idle":"2021-12-07T12:07:12.242384Z","shell.execute_reply.started":"2021-12-07T12:07:12.147733Z","shell.execute_reply":"2021-12-07T12:07:12.241473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 观察数据","metadata":{"papermill":{"duration":0.016208,"end_time":"2021-11-24T12:02:38.391639","exception":false,"start_time":"2021-11-24T12:02:38.375431","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## 列车数据基本信息","metadata":{"papermill":{"duration":0.016817,"end_time":"2021-11-24T12:02:38.424858","exception":false,"start_time":"2021-11-24T12:02:38.408041","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train.info()","metadata":{"papermill":{"duration":0.045379,"end_time":"2021-11-24T12:02:38.486598","exception":false,"start_time":"2021-11-24T12:02:38.441219","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T12:07:12.244785Z","iopub.execute_input":"2021-12-07T12:07:12.245314Z","iopub.status.idle":"2021-12-07T12:07:12.272640Z","shell.execute_reply.started":"2021-12-07T12:07:12.245266Z","shell.execute_reply":"2021-12-07T12:07:12.271534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 检查重复数据","metadata":{"papermill":{"duration":0.016115,"end_time":"2021-11-24T12:02:38.520915","exception":false,"start_time":"2021-11-24T12:02:38.5048","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train.duplicated().sum()","metadata":{"papermill":{"duration":0.037545,"end_time":"2021-11-24T12:02:38.575126","exception":false,"start_time":"2021-11-24T12:02:38.537581","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T12:07:12.276345Z","iopub.execute_input":"2021-12-07T12:07:12.276718Z","iopub.status.idle":"2021-12-07T12:07:12.302584Z","shell.execute_reply.started":"2021-12-07T12:07:12.276682Z","shell.execute_reply":"2021-12-07T12:07:12.301427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 功能概括","metadata":{"papermill":{"duration":0.016274,"end_time":"2021-11-24T12:02:38.641417","exception":false,"start_time":"2021-11-24T12:02:38.625143","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def resumetable(df):\n    '''function to create feature summary'''\n    print(f'Shape: {df.shape}')\n    summary = pd.DataFrame(df.dtypes, columns=['Data Type'])\n    summary = summary.reset_index()\n    summary = summary.rename(columns={'index': 'Features'})\n    summary['Num of Null Value'] = df.isnull().sum().values\n    summary['Num of Unique Value'] = df.nunique().values\n    summary['1st Value'] = df.loc[0].values\n    summary['2nd Value'] = df.loc[1].values\n    summary['3rd Value'] = df.loc[2].values\n    return summary","metadata":{"papermill":{"duration":0.026054,"end_time":"2021-11-24T12:02:38.684304","exception":false,"start_time":"2021-11-24T12:02:38.65825","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T12:07:12.304184Z","iopub.execute_input":"2021-12-07T12:07:12.305907Z","iopub.status.idle":"2021-12-07T12:07:12.314937Z","shell.execute_reply.started":"2021-12-07T12:07:12.305858Z","shell.execute_reply":"2021-12-07T12:07:12.313476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resumetable(train)","metadata":{"papermill":{"duration":0.049292,"end_time":"2021-11-24T12:02:38.750352","exception":false,"start_time":"2021-11-24T12:02:38.70106","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T12:07:12.316704Z","iopub.execute_input":"2021-12-07T12:07:12.318139Z","iopub.status.idle":"2021-12-07T12:07:12.363184Z","shell.execute_reply.started":"2021-12-07T12:07:12.318088Z","shell.execute_reply":"2021-12-07T12:07:12.361648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 基础工程","metadata":{"papermill":{"duration":0.01679,"end_time":"2021-11-24T12:02:38.784598","exception":false,"start_time":"2021-11-24T12:02:38.767808","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## 向下转换","metadata":{"papermill":{"duration":0.016691,"end_time":"2021-11-24T12:02:38.818357","exception":false,"start_time":"2021-11-24T12:02:38.801666","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def downcast(df, verbose=True):\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in df.columns:\n        dtype_name = df[col].dtype.name\n        if dtype_name == 'object':\n            pass\n        elif dtype_name == 'bool':\n            df[col] = df[col].astype('int8')\n        elif dtype_name.startswith('int') or (df[col].round() == df[col]).all():\n            df[col] = pd.to_numeric(df[col], downcast='integer')\n        else:\n            df[col] = pd.to_numeric(df[col], downcast='float')\n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose:\n        print('{:.1f}% Compressed'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"papermill":{"duration":0.026618,"end_time":"2021-11-24T12:02:38.862076","exception":false,"start_time":"2021-11-24T12:02:38.835458","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T12:07:12.364896Z","iopub.execute_input":"2021-12-07T12:07:12.365110Z","iopub.status.idle":"2021-12-07T12:07:12.379200Z","shell.execute_reply.started":"2021-12-07T12:07:12.365084Z","shell.execute_reply":"2021-12-07T12:07:12.377340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = downcast(train)","metadata":{"papermill":{"duration":0.036196,"end_time":"2021-11-24T12:02:38.915663","exception":false,"start_time":"2021-11-24T12:02:38.879467","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T12:07:12.380634Z","iopub.execute_input":"2021-12-07T12:07:12.381582Z","iopub.status.idle":"2021-12-07T12:07:12.408930Z","shell.execute_reply.started":"2021-12-07T12:07:12.381539Z","shell.execute_reply":"2021-12-07T12:07:12.407895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 特征工程","metadata":{"papermill":{"duration":0.017068,"end_time":"2021-11-24T12:02:38.950563","exception":false,"start_time":"2021-11-24T12:02:38.933495","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import ast\n\n# Convert String to List Type\ntrain['annotations'] = train['annotations'].apply(ast.literal_eval)\n\n# Get the number of bounding boxes for each image\ntrain['num_bboxes'] = train['annotations'].apply(lambda x: len(x))","metadata":{"papermill":{"duration":0.327369,"end_time":"2021-11-24T12:02:39.295514","exception":false,"start_time":"2021-11-24T12:02:38.968145","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T12:07:12.411221Z","iopub.execute_input":"2021-12-07T12:07:12.411839Z","iopub.status.idle":"2021-12-07T12:07:12.789722Z","shell.execute_reply.started":"2021-12-07T12:07:12.411756Z","shell.execute_reply":"2021-12-07T12:07:12.788633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"papermill":{"duration":0.041067,"end_time":"2021-11-24T12:02:39.354639","exception":false,"start_time":"2021-11-24T12:02:39.313572","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T12:07:12.792471Z","iopub.execute_input":"2021-12-07T12:07:12.792849Z","iopub.status.idle":"2021-12-07T12:07:12.808560Z","shell.execute_reply.started":"2021-12-07T12:07:12.792786Z","shell.execute_reply":"2021-12-07T12:07:12.807416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 检查带有边框的帧数","metadata":{"papermill":{"duration":0.021109,"end_time":"2021-11-24T12:02:39.40369","exception":false,"start_time":"2021-11-24T12:02:39.382581","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train[train['num_bboxes'] > 0]","metadata":{"papermill":{"duration":0.051725,"end_time":"2021-11-24T12:02:39.475147","exception":false,"start_time":"2021-11-24T12:02:39.423422","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T12:07:12.810154Z","iopub.execute_input":"2021-12-07T12:07:12.810465Z","iopub.status.idle":"2021-12-07T12:07:12.847605Z","shell.execute_reply.started":"2021-12-07T12:07:12.810415Z","shell.execute_reply":"2021-12-07T12:07:12.846568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 验证是否有损坏的数据","metadata":{"papermill":{"duration":0.0193,"end_time":"2021-11-24T12:02:39.514482","exception":false,"start_time":"2021-11-24T12:02:39.495182","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from os import listdir\nfrom PIL import Image\nDATA_PATH = '/kaggle/input/tensorflow-great-barrier-reef/'\ndef verify_images(video_id):\n    path = DATA_PATH + f'train_images/video_{video_id}/'    \n    for filename in listdir(path):\n        if filename.endswith('.jpg'):\n            try:\n                img = Image.open(path + filename)\n                img.verify() # Verify it is in fact an image\n            except (IOError, SyntaxError) as e:\n                print('Bad file:', filename) # Print out the names of corrupt files\n    print(f'Video {video_id} has all valid images. Verified!')\n\nfor video_id in range(3):\n    verify_images(video_id)","metadata":{"papermill":{"duration":232.121284,"end_time":"2021-11-24T12:06:31.654769","exception":false,"start_time":"2021-11-24T12:02:39.533485","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T12:07:12.849089Z","iopub.execute_input":"2021-12-07T12:07:12.849422Z","iopub.status.idle":"2021-12-07T12:09:14.489890Z","shell.execute_reply.started":"2021-12-07T12:07:12.849377Z","shell.execute_reply":"2021-12-07T12:09:14.488792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 用边框绘制帧图像","metadata":{"papermill":{"duration":0.018866,"end_time":"2021-11-24T12:06:31.692985","exception":false,"start_time":"2021-11-24T12:06:31.674119","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## 加载带有注释的图像序列","metadata":{"papermill":{"duration":0.018411,"end_time":"2021-11-24T12:06:31.730386","exception":false,"start_time":"2021-11-24T12:06:31.711975","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import numpy as np\nfrom PIL import ImageDraw\n\ndef fetch_image(df, video_id, frame_id):\n    # get frame\n    frame = df[(df['video_id'] == video_id) & (df['video_frame'] == frame_id)].iloc[0]\n    # get bounding_boxes\n    bounding_boxes = frame['annotations']\n    # open image\n    img = Image.open(DATA_PATH + f'train_images/video_{video_id}/{frame_id}.jpg')\n\n    for box in bounding_boxes:\n        x0, y0, x1, y1 = (box['x'], box['y'], box['x']+box['width'], box['y']+box['height'])\n        draw = ImageDraw.Draw(img)\n        draw.rectangle( (x0, y0, x1, y1), outline=180, width=5)\n    return img\n\ndef fetch_image_list(df, video_id, num_images, start_frame_idx):\n    image_list = [np.array(fetch_image(df, video_id, start_frame_idx + index)) for index in range(num_images)]\n\n    return image_list","metadata":{"papermill":{"duration":0.031604,"end_time":"2021-11-24T12:06:31.78101","exception":false,"start_time":"2021-11-24T12:06:31.749406","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T12:09:14.491467Z","iopub.execute_input":"2021-12-07T12:09:14.491848Z","iopub.status.idle":"2021-12-07T12:09:14.504783Z","shell.execute_reply.started":"2021-12-07T12:09:14.491779Z","shell.execute_reply":"2021-12-07T12:09:14.503980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images = fetch_image_list(train, video_id=0, num_images=80, start_frame_idx=25)\n\nprint(f'Number of images: {len(images)}')","metadata":{"papermill":{"duration":2.814046,"end_time":"2021-11-24T12:06:34.61438","exception":false,"start_time":"2021-11-24T12:06:31.800334","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T12:09:14.506443Z","iopub.execute_input":"2021-12-07T12:09:14.506688Z","iopub.status.idle":"2021-12-07T12:09:17.120067Z","shell.execute_reply.started":"2021-12-07T12:09:14.506661Z","shell.execute_reply":"2021-12-07T12:09:17.118390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec \n\ngrid = gridspec.GridSpec(4, 2) \nplt.figure(figsize=(18, 20))\n\nidx_list = [0, 5, 10, 15, 20, 25, 30, 35] \n\nfor i, idx in enumerate(idx_list): \n    ax = plt.subplot(grid[i])\n    plt.imshow(images[idx], interpolation='nearest')\n    ax.set_title(f'frame index {idx}')\n    plt.axis('off')","metadata":{"papermill":{"duration":1.723974,"end_time":"2021-11-24T12:06:36.359607","exception":false,"start_time":"2021-11-24T12:06:34.635633","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T12:09:17.121515Z","iopub.execute_input":"2021-12-07T12:09:17.121781Z","iopub.status.idle":"2021-12-07T12:09:18.824192Z","shell.execute_reply.started":"2021-12-07T12:09:17.121742Z","shell.execute_reply":"2021-12-07T12:09:18.823522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 图像动画","metadata":{"papermill":{"duration":0.106289,"end_time":"2021-11-24T12:06:36.576856","exception":false,"start_time":"2021-11-24T12:06:36.470567","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from matplotlib import animation, rc\nrc('animation', html='jshtml')\n\ndef create_animation(imgs, frame_interval=130):\n    fig = plt.figure(figsize=(7, 4))\n    plt.axis('off')\n    img = plt.imshow(imgs[0])\n\n    def animate(i):\n        img.set_array(imgs[i])\n        return [img]\n\n    return animation.FuncAnimation(fig, animate, frames=len(imgs), interval=frame_interval)","metadata":{"papermill":{"duration":0.117912,"end_time":"2021-11-24T12:06:36.800948","exception":false,"start_time":"2021-11-24T12:06:36.683036","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T12:09:18.825258Z","iopub.execute_input":"2021-12-07T12:09:18.825612Z","iopub.status.idle":"2021-12-07T12:09:18.837087Z","shell.execute_reply.started":"2021-12-07T12:09:18.825583Z","shell.execute_reply":"2021-12-07T12:09:18.835955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frame_interval = 130 # set smaller number if you want to play fast, otherwise set bigger\n\ncreate_animation(images, frame_interval=frame_interval)","metadata":{"papermill":{"duration":11.976804,"end_time":"2021-11-24T12:06:48.884476","exception":false,"start_time":"2021-11-24T12:06:36.907672","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-07T12:09:18.838380Z","iopub.execute_input":"2021-12-07T12:09:18.839373Z","iopub.status.idle":"2021-12-07T12:09:30.525900Z","shell.execute_reply.started":"2021-12-07T12:09:18.839327Z","shell.execute_reply":"2021-12-07T12:09:30.524886Z"},"trusted":true},"execution_count":null,"outputs":[]}]}