{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🐠 Coral Reef - Minimalistic EDA\n\n![](https://storage.googleapis.com/kaggle-competitions/kaggle/31703/logos/header.png)\n\n## Hello! This is just a functional check for some simple conditions in the input data and the submission API.\n\n# Please, _DO_ upvote if you found it useful!!","metadata":{}},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import os\nimport PIL\nimport numpy as np\nimport pandas as pd","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:24.518793Z","iopub.execute_input":"2021-11-22T22:42:24.519682Z","iopub.status.idle":"2021-11-22T22:42:24.550839Z","shell.execute_reply.started":"2021-11-22T22:42:24.519553Z","shell.execute_reply":"2021-11-22T22:42:24.54984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(\"../input/tensorflow-great-barrier-reef/train.csv\")\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:24.552394Z","iopub.execute_input":"2021-11-22T22:42:24.552657Z","iopub.status.idle":"2021-11-22T22:42:24.628895Z","shell.execute_reply.started":"2021-11-22T22:42:24.552629Z","shell.execute_reply":"2021-11-22T22:42:24.627685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[df_train['annotations'] != \"[]\"].head()","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:24.630302Z","iopub.execute_input":"2021-11-22T22:42:24.630781Z","iopub.status.idle":"2021-11-22T22:42:24.650926Z","shell.execute_reply.started":"2021-11-22T22:42:24.630746Z","shell.execute_reply":"2021-11-22T22:42:24.649927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cast annotations to list of dictionaries\ndf_train['annotations'] = df_train['annotations'].apply(eval)\ndf_train[df_train['annotations'].str.len() > 1].iloc[0]['annotations']","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:24.652724Z","iopub.execute_input":"2021-11-22T22:42:24.653595Z","iopub.status.idle":"2021-11-22T22:42:24.856747Z","shell.execute_reply.started":"2021-11-22T22:42:24.653545Z","shell.execute_reply":"2021-11-22T22:42:24.856145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Most images have no objects in them","metadata":{}},{"cell_type":"code","source":"df_train['n_objects'] = df_train['annotations'].str.len()\ndf_train.value_counts(\"n_objects\").plot.bar(figsize=(10, 5), alpha=0.5, rot=0, title=\"# images per number of objects in them\");","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:24.857566Z","iopub.execute_input":"2021-11-22T22:42:24.858747Z","iopub.status.idle":"2021-11-22T22:42:25.163684Z","shell.execute_reply.started":"2021-11-22T22:42:24.858692Z","shell.execute_reply":"2021-11-22T22:42:25.162785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(df_train['n_objects'] == 0).value_counts(normalize=True).rename({True: \"No objects\", False: \"At least one object\"})\\\n                            .plot.bar(figsize=(10, 5), alpha=0.5, rot=0, title=\"Has objects?\");","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:25.164955Z","iopub.execute_input":"2021-11-22T22:42:25.165155Z","iopub.status.idle":"2021-11-22T22:42:25.336639Z","shell.execute_reply.started":"2021-11-22T22:42:25.165134Z","shell.execute_reply":"2021-11-22T22:42:25.335344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(\"../input/tensorflow-great-barrier-reef/test.csv\")\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:25.337987Z","iopub.execute_input":"2021-11-22T22:42:25.338242Z","iopub.status.idle":"2021-11-22T22:42:25.356582Z","shell.execute_reply.started":"2021-11-22T22:42:25.338218Z","shell.execute_reply":"2021-11-22T22:42:25.355479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check `df_train` vs `train_images` folder","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:21:00.261209Z","iopub.execute_input":"2021-11-22T22:21:00.262105Z","iopub.status.idle":"2021-11-22T22:21:00.266372Z","shell.execute_reply.started":"2021-11-22T22:21:00.262062Z","shell.execute_reply":"2021-11-22T22:21:00.265504Z"}}},{"cell_type":"code","source":"!ls -l ../input/tensorflow-great-barrier-reef/train_images/","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:25.357648Z","iopub.execute_input":"2021-11-22T22:42:25.358725Z","iopub.status.idle":"2021-11-22T22:42:25.653804Z","shell.execute_reply.started":"2021-11-22T22:42:25.358677Z","shell.execute_reply":"2021-11-22T22:42:25.652651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs_0 = os.listdir(\"../input/tensorflow-great-barrier-reef/train_images/video_0\")\nimgs_1 = os.listdir(\"../input/tensorflow-great-barrier-reef/train_images/video_1\")\nimgs_2 = os.listdir(\"../input/tensorflow-great-barrier-reef/train_images/video_2\")","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:25.656346Z","iopub.execute_input":"2021-11-22T22:42:25.65712Z","iopub.status.idle":"2021-11-22T22:42:26.327898Z","shell.execute_reply.started":"2021-11-22T22:42:25.657083Z","shell.execute_reply":"2021-11-22T22:42:26.326623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(imgs_0)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:26.330258Z","iopub.execute_input":"2021-11-22T22:42:26.330471Z","iopub.status.idle":"2021-11-22T22:42:26.33666Z","shell.execute_reply.started":"2021-11-22T22:42:26.330448Z","shell.execute_reply":"2021-11-22T22:42:26.336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(imgs_1)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:26.337705Z","iopub.execute_input":"2021-11-22T22:42:26.338377Z","iopub.status.idle":"2021-11-22T22:42:26.353082Z","shell.execute_reply.started":"2021-11-22T22:42:26.338338Z","shell.execute_reply":"2021-11-22T22:42:26.352008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(imgs_2)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:26.356249Z","iopub.execute_input":"2021-11-22T22:42:26.356973Z","iopub.status.idle":"2021-11-22T22:42:26.368857Z","shell.execute_reply.started":"2021-11-22T22:42:26.356949Z","shell.execute_reply":"2021-11-22T22:42:26.367835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df_train)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:26.369996Z","iopub.execute_input":"2021-11-22T22:42:26.370181Z","iopub.status.idle":"2021-11-22T22:42:26.383981Z","shell.execute_reply.started":"2021-11-22T22:42:26.370159Z","shell.execute_reply":"2021-11-22T22:42:26.383601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# There is one row per image. Only 3 videos for training.","metadata":{}},{"cell_type":"code","source":"\nlen(imgs_0) + len(imgs_1) + len(imgs_2)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:26.384709Z","iopub.execute_input":"2021-11-22T22:42:26.385317Z","iopub.status.idle":"2021-11-22T22:42:26.493486Z","shell.execute_reply.started":"2021-11-22T22:42:26.385213Z","shell.execute_reply":"2021-11-22T22:42:26.492735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs_0[0]","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:26.494808Z","iopub.execute_input":"2021-11-22T22:42:26.495237Z","iopub.status.idle":"2021-11-22T22:42:26.512426Z","shell.execute_reply.started":"2021-11-22T22:42:26.495211Z","shell.execute_reply":"2021-11-22T22:42:26.510601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['image_path'] = \"video_\" + df_train['video_id'].astype(str) + \"/\" + df_train['video_frame'].astype(str) + \".jpg\"\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:26.513745Z","iopub.execute_input":"2021-11-22T22:42:26.513995Z","iopub.status.idle":"2021-11-22T22:42:26.574698Z","shell.execute_reply.started":"2021-11-22T22:42:26.513962Z","shell.execute_reply":"2021-11-22T22:42:26.573907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['image_path'].apply(lambda f_name: os.path.isfile(\"../input/tensorflow-great-barrier-reef/train_images/\"+ f_name)).all()","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:26.576329Z","iopub.execute_input":"2021-11-22T22:42:26.576524Z","iopub.status.idle":"2021-11-22T22:42:49.579313Z","shell.execute_reply.started":"2021-11-22T22:42:26.576501Z","shell.execute_reply":"2021-11-22T22:42:49.578651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_image(df_train, idx):\n    f_name = df_train.iloc[idx]['image_path']\n    return PIL.Image.open(\"../input/tensorflow-great-barrier-reef/train_images/\"+f_name)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:49.580155Z","iopub.execute_input":"2021-11-22T22:42:49.580351Z","iopub.status.idle":"2021-11-22T22:42:49.584422Z","shell.execute_reply.started":"2021-11-22T22:42:49.580328Z","shell.execute_reply":"2021-11-22T22:42:49.583439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_image(df_train, 0).resize((400, 256))","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:49.585579Z","iopub.execute_input":"2021-11-22T22:42:49.585792Z","iopub.status.idle":"2021-11-22T22:42:49.706523Z","shell.execute_reply.started":"2021-11-22T22:42:49.585763Z","shell.execute_reply":"2021-11-22T22:42:49.706006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_image(df_train, 16000).resize((400, 256))","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:49.70719Z","iopub.execute_input":"2021-11-22T22:42:49.707355Z","iopub.status.idle":"2021-11-22T22:42:49.809864Z","shell.execute_reply.started":"2021-11-22T22:42:49.707333Z","shell.execute_reply":"2021-11-22T22:42:49.80909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check the npy file","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:33:57.971392Z","iopub.execute_input":"2021-11-22T22:33:57.972025Z","iopub.status.idle":"2021-11-22T22:33:57.97558Z","shell.execute_reply.started":"2021-11-22T22:33:57.971986Z","shell.execute_reply":"2021-11-22T22:33:57.974658Z"}}},{"cell_type":"code","source":"pixels = np.load(\"../input/tensorflow-great-barrier-reef/example_test.npy\")","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:49.811079Z","iopub.execute_input":"2021-11-22T22:42:49.811465Z","iopub.status.idle":"2021-11-22T22:42:50.126081Z","shell.execute_reply.started":"2021-11-22T22:42:49.81143Z","shell.execute_reply":"2021-11-22T22:42:50.125409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pixels.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:50.127228Z","iopub.execute_input":"2021-11-22T22:42:50.127441Z","iopub.status.idle":"2021-11-22T22:42:50.132346Z","shell.execute_reply.started":"2021-11-22T22:42:50.127417Z","shell.execute_reply":"2021-11-22T22:42:50.131586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PIL.Image.fromarray(pixels[0, :]).resize((400, 256))","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:50.133132Z","iopub.execute_input":"2021-11-22T22:42:50.133411Z","iopub.status.idle":"2021-11-22T22:42:50.206662Z","shell.execute_reply.started":"2021-11-22T22:42:50.133384Z","shell.execute_reply":"2021-11-22T22:42:50.206209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PIL.Image.fromarray(pixels[2, :]).resize((400, 256))","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:50.207614Z","iopub.execute_input":"2021-11-22T22:42:50.207883Z","iopub.status.idle":"2021-11-22T22:42:50.26964Z","shell.execute_reply.started":"2021-11-22T22:42:50.207854Z","shell.execute_reply":"2021-11-22T22:42:50.268589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub = pd.read_csv(\"../input/tensorflow-great-barrier-reef/example_sample_submission.csv\")\ndf_sub.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:50.270889Z","iopub.execute_input":"2021-11-22T22:42:50.271114Z","iopub.status.idle":"2021-11-22T22:42:50.293897Z","shell.execute_reply.started":"2021-11-22T22:42:50.271083Z","shell.execute_reply":"2021-11-22T22:42:50.293125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check the submission API\n\n## See: [Great Barrier Reef API Tutorial](https://www.kaggle.com/sohier/great-barrier-reef-api-tutorial)","metadata":{}},{"cell_type":"code","source":"import greatbarrierreef\nenv = greatbarrierreef.make_env()   # initialize the environment\niter_test = env.iter_test()    # an iterator which loops over the test set and sample submission","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:50.296132Z","iopub.execute_input":"2021-11-22T22:42:50.296808Z","iopub.status.idle":"2021-11-22T22:42:50.331367Z","shell.execute_reply.started":"2021-11-22T22:42:50.296773Z","shell.execute_reply":"2021-11-22T22:42:50.33033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (pixel_array, df_pred) in iter_test:  # iterate through all test set images\n    print(f\"Pixels shape: {pixel_array.shape}\")\n    print(f\"df_pred shape: {df_pred.shape}\")\n    \n    img = PIL.Image.fromarray(pixel_array).resize((400, 256))\n    display(img)\n    \n    display(df_pred.head())\n    df_pred['annotations'] = '0.5 123 456 10 10'  # make your predictions here\n    env.predict(df_pred)   # register your predictions","metadata":{"execution":{"iopub.status.busy":"2021-11-22T22:42:50.332597Z","iopub.execute_input":"2021-11-22T22:42:50.332954Z","iopub.status.idle":"2021-11-22T22:42:50.547183Z","shell.execute_reply.started":"2021-11-22T22:42:50.332921Z","shell.execute_reply":"2021-11-22T22:42:50.546478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Please, _DO_ upvote if you found it useful!!","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}