{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **References**","metadata":{}},{"cell_type":"markdown","source":"https://www.kaggle.com/julian3833/reef-starter-torch-fasterrcnn-train-lb-0-361\n\nhttps://www.kaggle.com/rhythmcam/ast-basic-string-expression\n\nhttps://www.kaggle.com/vexxingbanana/sartorius-coco-dataset-notebook\n\nhttps://www.kaggle.com/c/tensorflow-great-barrier-reef/discussion/293723","metadata":{}},{"cell_type":"markdown","source":"# This Coco Json File Format is useful for training models with [MMTracking](http://github.com/open-mmlab/mmtracking)","metadata":{}},{"cell_type":"markdown","source":"# **Import Libraries**","metadata":{}},{"cell_type":"code","source":"import sklearn\nfrom sklearn.model_selection import train_test_split\nimport numpy as np\nimport pandas as pd\nimport os\nimport matplotlib.pyplot as plt\nimport json\nimport glob\nimport random\nimport cv2\nimport tqdm\nfrom tqdm import tqdm\nimport re\nimport ast","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:33:54.45956Z","iopub.execute_input":"2021-12-31T02:33:54.460809Z","iopub.status.idle":"2021-12-31T02:33:54.467938Z","shell.execute_reply.started":"2021-12-31T02:33:54.460731Z","shell.execute_reply":"2021-12-31T02:33:54.467279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Helper Functions**","metadata":{}},{"cell_type":"code","source":"def get_boxes(row):\n    \"\"\"Return the bboxes for a given row as a 3D matrix \"\"\"\n    #if len(row['annotations']) == 0:\n    #    row['annotations'] = [{'x': -1, 'y': -1, 'width': -1, 'height': -1}]\n    return pd.DataFrame(row['annotations'], columns=['x', 'y', 'width', 'height']).astype(float).values","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:33:25.415321Z","iopub.execute_input":"2021-12-31T02:33:25.416075Z","iopub.status.idle":"2021-12-31T02:33:25.422116Z","shell.execute_reply.started":"2021-12-31T02:33:25.416022Z","shell.execute_reply":"2021-12-31T02:33:25.42143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Create CocoVid Json File**","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/tensorflow-great-barrier-reef/train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:33:24.036484Z","iopub.execute_input":"2021-12-31T02:33:24.036794Z","iopub.status.idle":"2021-12-31T02:33:24.095574Z","shell.execute_reply.started":"2021-12-31T02:33:24.036762Z","shell.execute_reply":"2021-12-31T02:33:24.094503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:33:24.327342Z","iopub.execute_input":"2021-12-31T02:33:24.328265Z","iopub.status.idle":"2021-12-31T02:33:24.34673Z","shell.execute_reply.started":"2021-12-31T02:33:24.328202Z","shell.execute_reply":"2021-12-31T02:33:24.345727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_json_dict = {\n    \"images\": [],\n    \"videos\": [],\n    \"annotations\": [],\n    \"categories\": []\n}\ncategory_dict = {\"id\": 1, \"name\": \"starfish\", \"supercategory\": \"none\"}\noutput_json_dict[\"categories\"].append(category_dict)\nvideo_0_dict = {\"name\": \"video_0\", \"id\": 0}\noutput_json_dict[\"videos\"].append(video_0_dict)\nvideo_1_dict = {\"name\": \"video_1\", \"id\": 1}\noutput_json_dict[\"videos\"].append(video_1_dict)\n# video_2_dict = {\"name\": \"video_2\", \"id\": 2}\n# output_json_dict[\"videos\"].append(video_2_dict)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:33:24.660071Z","iopub.execute_input":"2021-12-31T02:33:24.660386Z","iopub.status.idle":"2021-12-31T02:33:24.686251Z","shell.execute_reply.started":"2021-12-31T02:33:24.660354Z","shell.execute_reply":"2021-12-31T02:33:24.684996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"annot_id = 0\nimg_id = 0\nvideo_0_frames = 0\nvideo_1_frames = 0","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:33:25.081801Z","iopub.execute_input":"2021-12-31T02:33:25.082098Z","iopub.status.idle":"2021-12-31T02:33:25.087448Z","shell.execute_reply.started":"2021-12-31T02:33:25.082066Z","shell.execute_reply":"2021-12-31T02:33:25.086454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for f in tqdm(train_df.itertuples(), total=len(train_df)):\n    if f[1] != 2:\n        img_path = '../input/tensorflow-great-barrier-reef/train_images/video_' + str(f[1]) + '/' + f[5].split('-')[1] + '.jpg'\n        img = cv2.imread(img_path)\n        height, width, channels = img.shape\n#         frame_id = f[3]\n        video_id = f[1]\n        if video_id == 0:\n            frame_id = video_0_frames\n        else:\n            frame_id = video_1_frames\n\n        img_info = {\n            \"id\": f[0],\n            \"width\": width,\n            \"height\": height,\n            \"file_name\": img_path,\n            \"frame_id\": frame_id,\n            \"video_id\": video_id,\n        }\n        output_json_dict[\"images\"].append(img_info)\n        if f[6] != '[]':\n    #         print(train_df.iloc[f[0]]['annotations'])\n            bbox_list = ast.literal_eval(f[6])\n            for bbox in bbox_list:\n    #             bbox = ast.literal_eval(bbox)\n                if bbox['height'] + bbox['y'] > 720:\n                    bbox['height'] = 720 - bbox['y']\n                annot = {\n                    \"category_id\": 1,\n                    \"bbox\": [bbox['x'], bbox['y'], bbox['width'], bbox['height']],\n                    \"id\": annot_id,\n                    \"image_id\": f[0],\n                    \"area\": bbox['width'] * bbox['height'],\n                    \"segmentation\": [],\n                    \"iscrowd\": 0,\n                    \"video_id\": video_id,\n                }\n                output_json_dict[\"annotations\"].append(annot)\n                annot_id += 1\n        img_id += 1\n        if video_id == 0:\n            video_0_frames += 1\n        else:\n            video_1_frames += 1","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:34:17.797966Z","iopub.execute_input":"2021-12-31T02:34:17.798275Z","iopub.status.idle":"2021-12-31T02:44:58.11751Z","shell.execute_reply.started":"2021-12-31T02:34:17.798242Z","shell.execute_reply":"2021-12-31T02:44:58.11661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('train_dataset.json', 'w') as f:\n    output_json = json.dumps(output_json_dict)\n    f.write(output_json)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_json_dict = {\n    \"images\": [],\n    \"videos\": [],\n    \"annotations\": [],\n    \"categories\": []\n}\ncategory_dict = {\"id\": 1, \"name\": \"starfish\", \"supercategory\": \"none\"}\noutput_json_dict[\"categories\"].append(category_dict)\n# video_0_dict = {\"name\": \"video_0\", \"id\": 0}\n# output_json_dict[\"videos\"].append(video_0_dict)\n# video_1_dict = {\"name\": \"video_1\", \"id\": 1}\n# output_json_dict[\"videos\"].append(video_1_dict)\nvideo_2_dict = {\"name\": \"video_2\", \"id\": 2}\noutput_json_dict[\"videos\"].append(video_2_dict)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"annot_id = 0\nimg_id = 0\nvideo_2_frames = 0","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for f in tqdm(train_df.itertuples(), total=len(train_df)):\n    if f[1] == 2:\n        img_path = '../input/tensorflow-great-barrier-reef/train_images/video_' + str(f[1]) + '/' + f[5].split('-')[1] + '.jpg'\n        img = cv2.imread(img_path)\n        height, width, channels = img.shape\n        frame_id = video_2_frames\n        video_id = f[1]\n\n        img_info = {\n            \"id\": img_id,\n            \"width\": width,\n            \"height\": height,\n            \"file_name\": img_path,\n            \"frame_id\": frame_id,\n            \"video_id\": video_id\n        }\n        output_json_dict[\"images\"].append(img_info)\n        if f[6] != '[]':\n    #         print(train_df.iloc[f[0]]['annotations'])\n            bbox_list = ast.literal_eval(f[6])\n            for bbox in bbox_list:\n    #             bbox = ast.literal_eval(bbox)\n                if bbox['height'] + bbox['y'] > 720:\n                    bbox['height'] = 720 - bbox['y']\n                annot = {\n                    \"category_id\": 1,\n                    \"bbox\": [bbox['x'], bbox['y'], bbox['width'], bbox['height']],\n                    \"id\": annot_id,\n                    \"image_id\": img_id,\n                    \"area\": bbox['width'] * bbox['height'],\n                    \"segmentation\": [],\n                    \"iscrowd\": 0,\n                    \"video_id\": video_id,\n                }\n                output_json_dict[\"annotations\"].append(annot)\n                annot_id += 1\n        img_id += 1\n        video_2_frames += 1","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('val_dataset.json', 'w') as f:\n    output_json = json.dumps(output_json_dict)\n    f.write(output_json)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}