{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nfrom PIL import Image, ImageDraw\nimport matplotlib.pyplot as plt\nimport plotly\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\n\nimport pandas as pd\nimport os\nimport numpy as np\nimport cv2\nimport ast","metadata":{"execution":{"iopub.status.busy":"2021-12-09T17:59:16.368282Z","iopub.execute_input":"2021-12-09T17:59:16.368632Z","iopub.status.idle":"2021-12-09T17:59:16.375317Z","shell.execute_reply.started":"2021-12-09T17:59:16.368589Z","shell.execute_reply":"2021-12-09T17:59:16.374441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pre process","metadata":{}},{"cell_type":"code","source":"%%time\n\ntrain_df = pd.read_csv('../input/tensorflow-great-barrier-reef/train.csv')\ntrain_df = train_df[train_df['annotations'].str.len() > 2]\ntrain_df['ann'] = train_df['annotations'].apply(lambda x: ast.literal_eval(x))\ntrain_df['count'] = train_df['ann'].str.len()\ntrain_df['bboxes'] = train_df['ann'].apply(lambda x: [[ann['x'], ann['y'], ann['x'] + ann['width'], ann['y']+ann['height']] for ann in x])\ntrain_df['areas'] = train_df['ann'].apply(lambda x: [np.array([ann['width']*ann['height'] for ann in x])])","metadata":{"execution":{"iopub.status.busy":"2021-12-09T17:59:16.376932Z","iopub.execute_input":"2021-12-09T17:59:16.377233Z","iopub.status.idle":"2021-12-09T17:59:16.947918Z","shell.execute_reply.started":"2021-12-09T17:59:16.377188Z","shell.execute_reply":"2021-12-09T17:59:16.946763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.sample(5)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T17:59:16.949375Z","iopub.execute_input":"2021-12-09T17:59:16.949599Z","iopub.status.idle":"2021-12-09T17:59:16.984047Z","shell.execute_reply.started":"2021-12-09T17:59:16.949570Z","shell.execute_reply":"2021-12-09T17:59:16.983165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Filtering reefs lies on last 10% Area","metadata":{}},{"cell_type":"code","source":"area_list = np.concatenate(train_df['areas'].values.flatten(), axis=1).ravel().tolist()\nprint(\"Total crops : \" + str(len(area_list)))\nprint(\"Mean \", np.mean(area_list), \" Median \", np.median(area_list))","metadata":{"execution":{"iopub.status.busy":"2021-12-09T17:59:16.986150Z","iopub.execute_input":"2021-12-09T17:59:16.986411Z","iopub.status.idle":"2021-12-09T17:59:17.018560Z","shell.execute_reply.started":"2021-12-09T17:59:16.986381Z","shell.execute_reply":"2021-12-09T17:59:17.017404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filter_per = 0.10 # bottom 10 %\nf_index = int(filter_per * len(area_list))\ns_area_list = np.sort(area_list)\nf_area_list = s_area_list[f_index:]\nmin_area = f_area_list[0]\nprint(\"Actual min area:\", s_area_list[0], ' After 10% filtered min area : ', min_area)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T18:00:19.519460Z","iopub.execute_input":"2021-12-09T18:00:19.519969Z","iopub.status.idle":"2021-12-09T18:00:19.529989Z","shell.execute_reply.started":"2021-12-09T18:00:19.519917Z","shell.execute_reply":"2021-12-09T18:00:19.528923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = make_subplots(rows=1, cols=2)\nfig.add_trace(go.Histogram(x=area_list, name='Area'), 1, 1)\nfig.add_vrect(x0=0, x1=min_area,fillcolor=\"LightSalmon\", opacity=0.5, layer=\"below\", line_width=0, col=1)\nfig.add_trace(go.Box(x=area_list, name='Area'), 1, 2)\nfig.add_trace(go.Box(x=f_area_list, name='Filtered_Area'), 1, 2)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T17:59:44.959736Z","iopub.execute_input":"2021-12-09T17:59:44.960070Z","iopub.status.idle":"2021-12-09T17:59:45.224239Z","shell.execute_reply.started":"2021-12-09T17:59:44.960039Z","shell.execute_reply":"2021-12-09T17:59:45.223456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_df['f_bboxes'] = train_df['bboxes'].apply(lambda x : [i for i in x if ((i[2]-i[0])*(i[3]-i[1])) >= min_area])\ntrain_df['f_count'] = train_df['f_bboxes'].str.len()\nf_train_df = train_df[train_df['f_count']!=0]\n\nprint(\"Actual data frame len :\", len(train_df), \" after filtered : \", len(f_train_df))","metadata":{"execution":{"iopub.status.busy":"2021-12-09T17:59:17.299294Z","iopub.execute_input":"2021-12-09T17:59:17.299515Z","iopub.status.idle":"2021-12-09T17:59:17.360305Z","shell.execute_reply.started":"2021-12-09T17:59:17.299487Z","shell.execute_reply":"2021-12-09T17:59:17.359111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df = f_train_df[(f_train_df['f_count'] > 3) & (f_train_df['f_count']!=f_train_df['count'])].sample(5)\nsample_df","metadata":{"execution":{"iopub.status.busy":"2021-12-09T17:59:17.361621Z","iopub.execute_input":"2021-12-09T17:59:17.361923Z","iopub.status.idle":"2021-12-09T17:59:17.449074Z","shell.execute_reply.started":"2021-12-09T17:59:17.361882Z","shell.execute_reply":"2021-12-09T17:59:17.448242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualizing filter effect","metadata":{}},{"cell_type":"code","source":"train_dir = '../input/tensorflow-great-barrier-reef/train_images'\nvideo_pre = 'video_'\ndef get_img_path(v_id, img_id):\n    return os.path.join(train_dir, video_pre + str(v_id), str(img_id) + '.jpg')\n\ndef get_cropped_images(v_id, image_id, bboxes): \n    img = Image.open(get_img_path(v_id, image_id))\n    images = list()\n    for bbox in bboxes:\n        images.append(img.crop(bbox))\n    return images\n\ndef draw_bboxes(v_id, image_id, bboxes): \n    img = Image.open(get_img_path(v_id, image_id))\n    draw = ImageDraw.Draw(img)\n    for bbox in bboxes:\n        draw.rectangle(bbox, outline='Red', width=10)\n    return img\n\nfor i, row in sample_df.iterrows():\n    plt.figure(figsize=(20, 15))\n    \n    img1 = draw_bboxes(row.video_id, row.video_frame, row.bboxes)\n    plt.subplot(1, 2, 1)\n    plt.imshow(img1)\n    \n    img2 = draw_bboxes(row.video_id, row.video_frame, row.f_bboxes)\n    plt.subplot(1, 2, 2)\n    plt.imshow(img2)\n    \n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T17:59:17.450659Z","iopub.execute_input":"2021-12-09T17:59:17.450953Z","iopub.status.idle":"2021-12-09T17:59:20.953778Z","shell.execute_reply.started":"2021-12-09T17:59:17.450913Z","shell.execute_reply":"2021-12-09T17:59:20.952832Z"},"trusted":true},"execution_count":null,"outputs":[]}]}