{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n# u! Hope this notebook can be helpful for getting a better understanding about the data.\n\nimport os\n\nfrom PIL import Image, ImageDraw\nimport cv2\nimport re\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\n\nfrom matplotlib import pyplot as plt\nimport matplotlib.patches as patches\nimport seaborn as sns\nfrom IPython.display import Video, display\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-04T16:31:53.648846Z","iopub.execute_input":"2021-12-04T16:31:53.649167Z","iopub.status.idle":"2021-12-04T16:32:01.753653Z","shell.execute_reply.started":"2021-12-04T16:31:53.649083Z","shell.execute_reply":"2021-12-04T16:32:01.752928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = {\n    'root_dir': '../input/tensorflow-great-barrier-reef',\n    'train_csv': '../input/tensorflow-great-barrier-reef/train.csv',\n    'test_csv': '../input/tensorflow-great-barrier-reef/test.csv',\n    'sample_submission_csv': '../input/tensorflow-great-barrier-reef/example_sample_submission.csv',\n    'video_img_dir': '../input/tensorflow-great-barrier-reef/train_images'\n}","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:01.755038Z","iopub.execute_input":"2021-12-04T16:32:01.755264Z","iopub.status.idle":"2021-12-04T16:32:01.761874Z","shell.execute_reply.started":"2021-12-04T16:32:01.755229Z","shell.execute_reply":"2021-12-04T16:32:01.761087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv = pd.read_csv(dataset['train_csv'])\ntest_csv = pd.read_csv(dataset['test_csv'])","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:01.763289Z","iopub.execute_input":"2021-12-04T16:32:01.763509Z","iopub.status.idle":"2021-12-04T16:32:01.829566Z","shell.execute_reply.started":"2021-12-04T16:32:01.763478Z","shell.execute_reply":"2021-12-04T16:32:01.828987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:01.831571Z","iopub.execute_input":"2021-12-04T16:32:01.832050Z","iopub.status.idle":"2021-12-04T16:32:01.852758Z","shell.execute_reply.started":"2021-12-04T16:32:01.832007Z","shell.execute_reply":"2021-12-04T16:32:01.852142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"number of frames:\", len(train_csv))","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:01.853937Z","iopub.execute_input":"2021-12-04T16:32:01.854206Z","iopub.status.idle":"2021-12-04T16:32:01.861855Z","shell.execute_reply.started":"2021-12-04T16:32:01.854160Z","shell.execute_reply":"2021-12-04T16:32:01.861023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frame_counts = train_csv['video_id'].value_counts().sort_values().to_frame()\nframe_counts.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:01.863448Z","iopub.execute_input":"2021-12-04T16:32:01.864030Z","iopub.status.idle":"2021-12-04T16:32:01.877429Z","shell.execute_reply.started":"2021-12-04T16:32:01.863992Z","shell.execute_reply":"2021-12-04T16:32:01.876725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"number of records in video_0 matched: \", frame_counts.loc[0]['video_id'] == len(os.listdir(os.path.join(dataset['video_img_dir'], 'video_0'))))\nprint(\"number of records in video_0 matched: \", frame_counts.loc[1]['video_id'] == len(os.listdir(os.path.join(dataset['video_img_dir'], 'video_1'))))\nprint(\"number of records in video_0 matched: \", frame_counts.loc[2]['video_id'] == len(os.listdir(os.path.join(dataset['video_img_dir'], 'video_2'))))","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:01.878994Z","iopub.execute_input":"2021-12-04T16:32:01.879553Z","iopub.status.idle":"2021-12-04T16:32:01.898204Z","shell.execute_reply.started":"2021-12-04T16:32:01.879507Z","shell.execute_reply":"2021-12-04T16:32:01.897517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence_counts = train_csv['sequence'].value_counts().sort_values().reset_index()\nsequence_counts.columns = [['sequence', 'num_frames']]\nprint(\"number of sequences:\", len(sequence_counts))\nsequence_counts.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:02.040779Z","iopub.execute_input":"2021-12-04T16:32:02.041084Z","iopub.status.idle":"2021-12-04T16:32:02.058805Z","shell.execute_reply.started":"2021-12-04T16:32:02.041058Z","shell.execute_reply":"2021-12-04T16:32:02.058072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_csv","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:02.104538Z","iopub.execute_input":"2021-12-04T16:32:02.104815Z","iopub.status.idle":"2021-12-04T16:32:02.112407Z","shell.execute_reply.started":"2021-12-04T16:32:02.104789Z","shell.execute_reply":"2021-12-04T16:32:02.111725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_no_obj_frame = train_csv[train_csv.annotations == '[]']['annotations'].count()\nprint(\"number of frames without objects:\", num_no_obj_frame)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:02.683153Z","iopub.execute_input":"2021-12-04T16:32:02.683550Z","iopub.status.idle":"2021-12-04T16:32:02.698761Z","shell.execute_reply.started":"2021-12-04T16:32:02.683511Z","shell.execute_reply":"2021-12-04T16:32:02.697753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_with_obj_frame = train_csv[train_csv.annotations != '[]']['annotations'].count()\nprint(\"number of frames with objects:\", num_with_obj_frame)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:02.700424Z","iopub.execute_input":"2021-12-04T16:32:02.700890Z","iopub.status.idle":"2021-12-04T16:32:02.711835Z","shell.execute_reply.started":"2021-12-04T16:32:02.700852Z","shell.execute_reply":"2021-12-04T16:32:02.710962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv[train_csv.annotations != '[]'].head()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:03.210706Z","iopub.execute_input":"2021-12-04T16:32:03.211573Z","iopub.status.idle":"2021-12-04T16:32:03.226454Z","shell.execute_reply.started":"2021-12-04T16:32:03.211521Z","shell.execute_reply":"2021-12-04T16:32:03.225669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('ratio of frames with objects:', num_with_obj_frame / len(train_csv))\n\nfig, axes = plt.subplots(1,1, figsize=(12, 6))\n\nsns.barplot(ax=axes, x=['Number of Frames with Objects', 'Number of Frames with No Objects'], y=[num_with_obj_frame, num_no_obj_frame])\naxes.set_title(\"Distribution of Frames with/without Objects\")\naxes.set_xlabel(\"Frame Types\")\naxes.set_ylabel(\"Count\")\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:03.761004Z","iopub.execute_input":"2021-12-04T16:32:03.761264Z","iopub.status.idle":"2021-12-04T16:32:03.970719Z","shell.execute_reply.started":"2021-12-04T16:32:03.761236Z","shell.execute_reply":"2021-12-04T16:32:03.970073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_annotation(annot_line):\n    # annot_line example: [{'x': 540, 'y': 310, 'width': 113, 'height': 105}, {'x': 657, 'y': 501, 'width': 95, 'height': 56}]\n    boxes = []\n    \n    box_pattern = r'\\{\\'\\w\\'\\:\\s\\d+\\,\\s\\'\\w\\'\\:\\s\\d+\\,\\s\\'\\w+\\'\\:\\s\\d+\\,\\s\\'\\w+\\'\\:\\s\\d+\\}'\n    val_pattern = r'\\d+'\n    \n    annotations = re.findall(box_pattern, annot_line)\n    for annot in annotations:\n        x, y, width, height = re.findall(val_pattern, annot)\n        x, y, width, height = float(x), float(y), float(width), float(height)\n        confidence = 1.0\n        \n        box = [x, y, width, height, confidence]\n        boxes.append(box)\n        \n    return boxes\n\ndef count_boxes(annot_line):\n    \n    annot_line  = annot_line[1:-1]\n    box_pattern = r'\\{\\'\\w\\'\\:\\s\\d+\\,\\s\\'\\w\\'\\:\\s\\d+\\,\\s\\'\\w+\\'\\:\\s\\d+\\,\\s\\'\\w+\\'\\:\\s\\d+\\}'\n    val_pattern = r'\\d+'\n    \n    annotations = re.findall(box_pattern, annot_line)\n    \n    return len(annotations)\n\n\ndef test_decode_annotation(annot_line):\n    print(\"sample:\", annot_line)\n    boxes = decode_annotation(annot_line)\n    for i, box in enumerate(boxes):\n        print(f\"box {i}:\", box)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:03.972390Z","iopub.execute_input":"2021-12-04T16:32:03.972648Z","iopub.status.idle":"2021-12-04T16:32:03.981823Z","shell.execute_reply.started":"2021-12-04T16:32:03.972613Z","shell.execute_reply":"2021-12-04T16:32:03.981002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_samples = [\n    \"[{'x': 540, 'y': 310, 'width': 113, 'height': 105}, {'x': 657, 'y': 501, 'width': 95, 'height': 56}, {'x': 257, 'y': 101, 'width': 42, 'height': 59}]\",\n    \"[{'x': 540, 'y': 310, 'width': 113, 'height': 105}, {'x': 657, 'y': 501, 'width': 95, 'height': 59}]\",\n    \"[{'x': 12, 'y': 250, 'width': 143, 'height': 82}]\",\n    \"[]\"\n]\n\nfor i, sample in enumerate(test_samples):\n    num_boxes = count_boxes(sample)\n    print(f\"Test {i+1}:\", f\"found {num_boxes} boxes\")\n    \n    test_decode_annotation(sample)\n    print(\"\")","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:03.983337Z","iopub.execute_input":"2021-12-04T16:32:03.983592Z","iopub.status.idle":"2021-12-04T16:32:03.999808Z","shell.execute_reply.started":"2021-12-04T16:32:03.983557Z","shell.execute_reply":"2021-12-04T16:32:03.998912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv['num_boxes'] = train_csv['annotations'].apply(count_boxes)\n","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:04.650411Z","iopub.execute_input":"2021-12-04T16:32:04.650925Z","iopub.status.idle":"2021-12-04T16:32:04.711623Z","shell.execute_reply.started":"2021-12-04T16:32:04.650889Z","shell.execute_reply":"2021-12-04T16:32:04.710945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv[train_csv.annotations != '[]'].head()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:05.362698Z","iopub.execute_input":"2021-12-04T16:32:05.363111Z","iopub.status.idle":"2021-12-04T16:32:05.389873Z","shell.execute_reply.started":"2021-12-04T16:32:05.363070Z","shell.execute_reply":"2021-12-04T16:32:05.389244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"boxes_dist = train_csv[train_csv.annotations != '[]']['num_boxes'].value_counts().sort_values(ascending=False).reset_index()\nboxes_dist.columns = ['num_boxes', 'num_frames']\nboxes_dist","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:05.915991Z","iopub.execute_input":"2021-12-04T16:32:05.916259Z","iopub.status.idle":"2021-12-04T16:32:05.933624Z","shell.execute_reply.started":"2021-12-04T16:32:05.916228Z","shell.execute_reply":"2021-12-04T16:32:05.932942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(24, 8))\nsns.barplot(x=boxes_dist.num_boxes, y=boxes_dist.num_frames)\n\nplt.title(\"Box Distribution\")\nplt.xlabel(\"Number of Boxes\")\nplt.ylabel(\"Frame Counts\")\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:06.574879Z","iopub.execute_input":"2021-12-04T16:32:06.575118Z","iopub.status.idle":"2021-12-04T16:32:06.879626Z","shell.execute_reply.started":"2021-12-04T16:32:06.575091Z","shell.execute_reply":"2021-12-04T16:32:06.878971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gen_file_path(image_id):\n    # extract file path by using the image_id in the train file\n    video_id = image_id.split('-')[0]\n    image_id = image_id.split('-')[1]\n    return os.path.join(dataset['video_img_dir'], 'video_' + video_id, image_id + '.jpg')\n\ndef draw_boxes(image_path, annot_line):\n    \n    boxes = decode_annotation(annot_line)\n\n    coords = [] \n    for box in boxes: \n        coord = [] \n        coord.append(box[0]) \n        coord.append(box[1]) \n        coord.append(box[0] + box[2]) \n        coord.append(box[1] + box[3]) \n        coords.append(coord) \n\n    image = Image.open(image_path)\n    imgcp = image.copy()\n    imgcp_draw = ImageDraw.Draw(imgcp)\n\n    for coord in  coords:\n         imgcp_draw.rectangle(coord, fill = None, outline = \"red\", width=5)\n\n    return imgcp","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:07.510735Z","iopub.execute_input":"2021-12-04T16:32:07.511209Z","iopub.status.idle":"2021-12-04T16:32:07.519151Z","shell.execute_reply.started":"2021-12-04T16:32:07.511158Z","shell.execute_reply":"2021-12-04T16:32:07.518480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv['file_path'] = train_csv['image_id'].apply(gen_file_path)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:08.358136Z","iopub.execute_input":"2021-12-04T16:32:08.358667Z","iopub.status.idle":"2021-12-04T16:32:08.439017Z","shell.execute_reply.started":"2021-12-04T16:32:08.358630Z","shell.execute_reply":"2021-12-04T16:32:08.438383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:08.932965Z","iopub.execute_input":"2021-12-04T16:32:08.933745Z","iopub.status.idle":"2021-12-04T16:32:08.945304Z","shell.execute_reply.started":"2021-12-04T16:32:08.933704Z","shell.execute_reply":"2021-12-04T16:32:08.944582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"samples = train_csv.groupby('num_boxes').first()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:09.389666Z","iopub.execute_input":"2021-12-04T16:32:09.390208Z","iopub.status.idle":"2021-12-04T16:32:09.412588Z","shell.execute_reply.started":"2021-12-04T16:32:09.390156Z","shell.execute_reply":"2021-12-04T16:32:09.411907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(24, 36))\n\nr, c = 7, 3\nfor index, row in samples.iterrows():\n    image_path = row['file_path']\n    annot_line = row['annotations']\n    plt.subplot(r, c, index + 1)\n    dimg = draw_boxes(image_path, annot_line)\n    plt.imshow(dimg)\n    \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:10.114210Z","iopub.execute_input":"2021-12-04T16:32:10.114755Z","iopub.status.idle":"2021-12-04T16:32:17.543874Z","shell.execute_reply.started":"2021-12-04T16:32:10.114719Z","shell.execute_reply":"2021-12-04T16:32:17.542276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_boxes_xy = []\nall_boxes_wh = []\n\nfor index, row in tqdm(train_csv.iterrows(), total=len(train_csv)):\n    if row['annotations'] != '[]':\n        boxes = decode_annotation(row['annotations'])\n        \n        for box in boxes:\n            all_boxes_xy.append([box[0], box[1]])\n            all_boxes_wh.append([box[2], box[3]])\n            \nall_boxes_xy = np.array(all_boxes_xy)\nall_boxes_wh = np.array(all_boxes_wh)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:17.545751Z","iopub.execute_input":"2021-12-04T16:32:17.546126Z","iopub.status.idle":"2021-12-04T16:32:18.806943Z","shell.execute_reply.started":"2021-12-04T16:32:17.546080Z","shell.execute_reply":"2021-12-04T16:32:18.806235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"box_center_df = pd.DataFrame.from_records(all_boxes_xy, columns=['x', 'y'])\n\nbox_shape_df  = pd.DataFrame.from_records(all_boxes_wh, columns=['width', 'height'])\nbox_shape_df['area'] = box_shape_df['width'] * box_shape_df['height']","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:18.808241Z","iopub.execute_input":"2021-12-04T16:32:18.808953Z","iopub.status.idle":"2021-12-04T16:32:18.869218Z","shell.execute_reply.started":"2021-12-04T16:32:18.808916Z","shell.execute_reply":"2021-12-04T16:32:18.868544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"box_center_df.describe()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:18.871162Z","iopub.execute_input":"2021-12-04T16:32:18.871508Z","iopub.status.idle":"2021-12-04T16:32:18.891464Z","shell.execute_reply.started":"2021-12-04T16:32:18.871471Z","shell.execute_reply":"2021-12-04T16:32:18.890833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"box_shape_df.describe()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:18.892731Z","iopub.execute_input":"2021-12-04T16:32:18.893057Z","iopub.status.idle":"2021-12-04T16:32:18.915502Z","shell.execute_reply.started":"2021-12-04T16:32:18.893009Z","shell.execute_reply":"2021-12-04T16:32:18.914792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(28, 16))\nplt.scatter(x=all_boxes_xy[:,0], y=all_boxes_xy[:,1], s=0.5, color = 'red')\nplt.title(\"Distribution of Box Center Coordinate on Image\")\nplt.xlabel(\"X value\")\nplt.ylabel(\"Y value\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:18.916868Z","iopub.execute_input":"2021-12-04T16:32:18.917144Z","iopub.status.idle":"2021-12-04T16:32:19.246417Z","shell.execute_reply.started":"2021-12-04T16:32:18.917106Z","shell.execute_reply":"2021-12-04T16:32:19.242446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sequence","metadata":{}},{"cell_type":"code","source":"train_csv.groupby('sequence')['num_boxes'].sum().sort_values(ascending=False).to_frame().T","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:19.247593Z","iopub.execute_input":"2021-12-04T16:32:19.247913Z","iopub.status.idle":"2021-12-04T16:32:19.266199Z","shell.execute_reply.started":"2021-12-04T16:32:19.247874Z","shell.execute_reply":"2021-12-04T16:32:19.265438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.groupby('sequence')['image_id'].count().sort_values(ascending=False).to_frame().T\n","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:19.267302Z","iopub.execute_input":"2021-12-04T16:32:19.267612Z","iopub.status.idle":"2021-12-04T16:32:19.287655Z","shell.execute_reply.started":"2021-12-04T16:32:19.267576Z","shell.execute_reply":"2021-12-04T16:32:19.286890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_seq = train_csv[train_csv.sequence == 22643]\nsample_seq","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:32:19.289010Z","iopub.execute_input":"2021-12-04T16:32:19.289403Z","iopub.status.idle":"2021-12-04T16:32:19.305843Z","shell.execute_reply.started":"2021-12-04T16:32:19.289369Z","shell.execute_reply":"2021-12-04T16:32:19.305230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\nimg = Image.open(train_csv['file_path'][0])","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:33:38.137709Z","iopub.execute_input":"2021-12-04T16:33:38.137971Z","iopub.status.idle":"2021-12-04T16:33:38.144202Z","shell.execute_reply.started":"2021-12-04T16:33:38.137941Z","shell.execute_reply":"2021-12-04T16:33:38.143470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(np.array(img))\nimg.size","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:34:28.157600Z","iopub.execute_input":"2021-12-04T16:34:28.158255Z","iopub.status.idle":"2021-12-04T16:34:28.694487Z","shell.execute_reply.started":"2021-12-04T16:34:28.158211Z","shell.execute_reply":"2021-12-04T16:34:28.693575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Dataset","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import DataLoader\n!pip install -qU torch_snippets\nfrom torch_snippets import *","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:34:39.420607Z","iopub.execute_input":"2021-12-04T16:34:39.420868Z","iopub.status.idle":"2021-12-04T16:34:53.287605Z","shell.execute_reply.started":"2021-12-04T16:34:39.420840Z","shell.execute_reply":"2021-12-04T16:34:53.286712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\nlabel2target = {}\nlabel2target['starfish'] = 1\nlabel2target['background'] = 0\ntarget2label = {t:l for l,t in label2target.items()}\ndef preprocess_image(img):\n    img = torch.tensor(img).permute(2,0,1)\n    return img.to(device).float()\nclass OpenDataset(torch.utils.data.Dataset):\n    w, h = 1280 , 720\n    def __init__(self, df):\n#         self.image_dir = image_dir\n#         self.files = glob.glob(self.image_dir+'/*')\n        self.df = df\n        self.image_infos = df['file_path'].values\n    def __getitem__(self, ix):\n        # load images and masks\n#         image_id = self.image_infos[ix]\n        img_path = self.image_infos[ix]\n        img = Image.open(img_path).convert(\"RGB\")\n        img = np.array(img.resize((self.w, self.h), resample=Image.BILINEAR))/255.\n#         data = self.df[self.df['Image_ID'] == image_id]\n#         if self.df['annotations'].values[ix] != '[]':\n        data = []\n        boxes = decode_annotation(self.df['annotations'].values[ix]) \n        for box in boxes:\n            data.append(pd.Series([box[0], box[1],box[0]+box[2],box[1]+box[3]]).astype(np.uint32).tolist())\n#             data = data.astype(np.uint32).tolist() # convert to absolute coordinates\n        # torch FRCNN expects ground truths as a dictionary of tensors\n\n        labels =['starfish']*len(data)\n        target = {}\n        target[\"boxes\"] = torch.Tensor(data).float()\n        target[\"labels\"] = torch.Tensor([1]*len(data)).long()\n        img = preprocess_image(img)\n        return img, target\n    def collate_fn(self, batch):\n        return tuple(zip(*batch)) \n\n    def __len__(self):\n        return len(self.image_infos)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:34:53.289520Z","iopub.execute_input":"2021-12-04T16:34:53.289770Z","iopub.status.idle":"2021-12-04T16:34:53.305718Z","shell.execute_reply.started":"2021-12-04T16:34:53.289740Z","shell.execute_reply":"2021-12-04T16:34:53.305079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = train_csv[train_csv.annotations != \"[]\"]\nfrom sklearn.model_selection import train_test_split\ntrn_ids, val_ids = train_test_split(data.image_id.unique(), test_size=0.1, random_state=99)\ntrn_df, val_df = data[data['image_id'].isin(trn_ids)], data[data['image_id'].isin(val_ids)]\n\ntrain_ds = OpenDataset(trn_df)\ntest_ds = OpenDataset(val_df)\ntrain_loader = DataLoader(train_ds, batch_size=4, collate_fn=train_ds.collate_fn, drop_last=True)\ntest_loader = DataLoader(test_ds, batch_size=4, collate_fn=test_ds.collate_fn, drop_last=True)\nlen(trn_df), len(val_df)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:34:55.525581Z","iopub.execute_input":"2021-12-04T16:34:55.526145Z","iopub.status.idle":"2021-12-04T16:34:55.549735Z","shell.execute_reply.started":"2021-12-04T16:34:55.526107Z","shell.execute_reply":"2021-12-04T16:34:55.549105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Testing","metadata":{}},{"cell_type":"code","source":"num_classes = 2\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nimport torchvision\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\n\n\ndef get_model():\n    model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True)\n    in_features = model.roi_heads.box_predictor.cls_score.in_features\n    model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:34:56.564115Z","iopub.execute_input":"2021-12-04T16:34:56.564651Z","iopub.status.idle":"2021-12-04T16:34:56.572266Z","shell.execute_reply.started":"2021-12-04T16:34:56.564608Z","shell.execute_reply":"2021-12-04T16:34:56.571376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining training and validation functions for a single batch\ndef train_batch(inputs, model, optimizer):\n    model.train()\n    input, targets = inputs\n    input = list(image.to(device) for image in input)\n    targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n    optimizer.zero_grad()\n    losses = model(input, targets)\n    loss = sum(loss for loss in losses.values())\n    loss.backward()\n    optimizer.step()\n    return loss, losses\n\n@torch.no_grad() # this will disable gradient computation in the function below\ndef validate_batch(inputs, model):\n    model.train() # to obtain the losses, model needs to be in train mode only. # #Note that here we are not defining the model's forward method \n#and hence need to work per the way the model class is defined\n    input, targets = inputs\n    input = list(image.to(device) for image in input)\n    targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n\n    optimizer.zero_grad()\n    losses = model(input, targets)\n    loss = sum(loss for loss in losses.values())\n    return loss, losses","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:34:57.420168Z","iopub.execute_input":"2021-12-04T16:34:57.420742Z","iopub.status.idle":"2021-12-04T16:34:57.431249Z","shell.execute_reply.started":"2021-12-04T16:34:57.420702Z","shell.execute_reply":"2021-12-04T16:34:57.430386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = get_model().to(device)\noptimizer = torch.optim.SGD(model.parameters(), lr=0.005,\n                            momentum=0.9, weight_decay=0.0005)\nn_epochs = 8\nlog = Report(n_epochs)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:34:57.983107Z","iopub.execute_input":"2021-12-04T16:34:57.984047Z","iopub.status.idle":"2021-12-04T16:35:07.934102Z","shell.execute_reply.started":"2021-12-04T16:34:57.984000Z","shell.execute_reply":"2021-12-04T16:35:07.933343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for epoch in range(n_epochs):\n    _n = len(train_loader)\n    for ix, inputs in enumerate(train_loader):\n        loss, losses = train_batch(inputs, model, optimizer)\n        loc_loss, regr_loss, loss_objectness, loss_rpn_box_reg = \\\n            [losses[k] for k in ['loss_classifier','loss_box_reg','loss_objectness','loss_rpn_box_reg']]\n        pos = (epoch + (ix+1)/_n)\n        log.record(pos, trn_loss=loss.item(), trn_loc_loss=loc_loss.item(), \n                   trn_regr_loss=regr_loss.item(), trn_objectness_loss=loss_objectness.item(),\n                   trn_rpn_box_reg_loss=loss_rpn_box_reg.item(), end='\\r')\n\n    _n = len(test_loader)\n    for ix,inputs in enumerate(test_loader):\n        loss, losses = validate_batch(inputs, model)\n        loc_loss, regr_loss, loss_objectness, loss_rpn_box_reg = \\\n          [losses[k] for k in ['loss_classifier','loss_box_reg','loss_objectness','loss_rpn_box_reg']]\n        pos = (epoch + (ix+1)/_n)\n        log.record(pos, val_loss=loss.item(), val_loc_loss=loc_loss.item(), \n                  val_regr_loss=regr_loss.item(), val_objectness_loss=loss_objectness.item(),\n                  val_rpn_box_reg_loss=loss_rpn_box_reg.item(), end='\\r')\n    if (epoch+1)%(n_epochs//2)==0: log.report_avgs(epoch+1)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:35:10.579998Z","iopub.execute_input":"2021-12-04T16:35:10.580730Z","iopub.status.idle":"2021-12-04T17:42:40.878979Z","shell.execute_reply.started":"2021-12-04T16:35:10.580692Z","shell.execute_reply":"2021-12-04T17:42:40.877341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"log.plot_epochs(['trn_loss','val_loss'])","metadata":{"execution":{"iopub.status.busy":"2021-12-04T17:44:03.178632Z","iopub.execute_input":"2021-12-04T17:44:03.178889Z","iopub.status.idle":"2021-12-04T17:44:03.559033Z","shell.execute_reply.started":"2021-12-04T17:44:03.178862Z","shell.execute_reply":"2021-12-04T17:44:03.558319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputt = next(iter(test_loader))[0]\ninput1 = next(iter(test_loader))[1]","metadata":{"execution":{"iopub.status.busy":"2021-12-04T17:44:35.000836Z","iopub.execute_input":"2021-12-04T17:44:35.001402Z","iopub.status.idle":"2021-12-04T17:44:35.409934Z","shell.execute_reply.started":"2021-12-04T17:44:35.001363Z","shell.execute_reply":"2021-12-04T17:44:35.409210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()\nmodel(inputt[0].unsqueeze(0))","metadata":{"execution":{"iopub.status.busy":"2021-12-04T17:45:18.228720Z","iopub.execute_input":"2021-12-04T17:45:18.228972Z","iopub.status.idle":"2021-12-04T17:45:18.322993Z","shell.execute_reply.started":"2021-12-04T17:45:18.228943Z","shell.execute_reply":"2021-12-04T17:45:18.322200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchvision.ops import nms\ndef decode_output(output):\n    'convert tensors to numpy arrays'\n    bbs = output['boxes'].cpu().detach().numpy().astype(np.uint16)\n    labels = np.array([target2label[i] for i in output['labels'].cpu().detach().numpy()])\n    confs = output['scores'].cpu().detach().numpy()\n    ixs = nms(torch.tensor(bbs.astype(np.float32)), torch.tensor(confs), 0.05)\n    bbs, confs, labels = [tensor[ixs] for tensor in [bbs, confs, labels]]\n\n    if len(ixs) == 1:\n        bbs, confs, labels = [np.array([tensor]) for tensor in [bbs, confs, labels]]\n    return bbs.tolist(), confs.tolist(), labels.tolist()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T17:48:01.760294Z","iopub.execute_input":"2021-12-04T17:48:01.760945Z","iopub.status.idle":"2021-12-04T17:48:01.769368Z","shell.execute_reply.started":"2021-12-04T17:48:01.760900Z","shell.execute_reply":"2021-12-04T17:48:01.768467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()\nfor ix, (images, targets) in enumerate(test_loader):\n    if ix==3: break\n    images = [im for im in images]\n    outputs = model(images)\n    for ix, output in enumerate(outputs):\n        bbs, confs, labels = decode_output(output)\n        info = [f'{l}@{c:.2f}' for l,c in zip(labels, confs)]\n        show(images[ix].cpu().permute(1,2,0), bbs=bbs, texts=labels, sz=7)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T17:49:38.450958Z","iopub.execute_input":"2021-12-04T17:49:38.453671Z","iopub.status.idle":"2021-12-04T17:49:43.861825Z","shell.execute_reply.started":"2021-12-04T17:49:38.453607Z","shell.execute_reply":"2021-12-04T17:49:43.861105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH ='TF-OD1.pth'\nPATH1 ='TF-OD2.pth'\ntorch.save(model.state_dict(), PATH)\ntorch.save(model, PATH1)","metadata":{},"execution_count":null,"outputs":[]}]}