{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"ref: https://www.kaggle.com/ammarnassanalhajali/barrier-reef-yolox-training","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nimport ast\nimport os\nimport json\nimport pandas as pd\nimport torch\nimport importlib\nimport cv2 \n\nimport shutil\nfrom shutil import copyfile\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\nfrom sklearn.model_selection import GroupKFold\nfrom PIL import Image\nfrom string import Template\nfrom IPython.display import display\nTRAIN_PATH = '/kaggle/input/tensorflow-great-barrier-reef'","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read in the data CSV files\ndf = pd.read_csv(\"/kaggle/input/tensorflow-great-barrier-reef/train.csv\")\ndf.head(5)\ndf[\"NumBBox\"]=df['annotations'].apply(lambda x: str.count(x, 'x'))\ndf_train=df[df[\"NumBBox\"]>0]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_bbox(annots):\n    bboxes = [list(annot.values()) for annot in annots]\n    return bboxes\ndef get_path(row):\n    row['image_path'] = f'{TRAIN_PATH}/train_images/video_{row.video_id}/{row.video_frame}.jpg'\n    return row\ndef load_image(image_path):\n    return cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['annotations'] = df_train['annotations'].progress_apply(lambda x: ast.literal_eval(x))\ndf_train['bboxes'] = df_train.annotations.progress_apply(get_bbox)\ndf_train[\"width\"]=1280\ndf_train[\"height\"]=720","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train.progress_apply(get_path, axis=1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_spt=5\nSelected_Fold=0  #0..4\n\nfrom sklearn.model_selection import StratifiedKFold\nkf = StratifiedKFold(n_splits = n_spt) \ndf_train = df_train.reset_index(drop=True)\ndf_train['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(kf.split(df_train, y = df_train.video_id.tolist())):\n    df_train.loc[val_idx, 'fold'] = fold\ndisplay(df_train.fold.value_counts())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Work_Dir = '/kaggle/working/' \nDataSet_Path = 'dataset/images'\n\nos.makedirs(f'{Work_Dir}{DataSet_Path}/train2017', exist_ok=True)\nos.makedirs(f'{Work_Dir}{DataSet_Path}/val2017', exist_ok=True)\nos.makedirs(f'{Work_Dir}{DataSet_Path}/annotations', exist_ok=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in tqdm(range(len(df_train))):\n    row = df_train.loc[i]\n    if row.fold != Selected_Fold:\n        copyfile(f'{row.image_path}', f'{Work_Dir}{DataSet_Path}/train2017/{row.image_id}.jpg')\n    else:\n        copyfile(f'{row.image_path}', f'{Work_Dir}{DataSet_Path}/val2017/{row.image_id}.jpg') ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Number of training files: {len(os.listdir(f\"{Work_Dir}{DataSet_Path}/train2017/\"))}')\nprint(f'Number of validation files: {len(os.listdir(f\"{Work_Dir}{DataSet_Path}/val2017/\"))}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"annotion_id = 0\n\ndef save_annot_json(json_annotation, filename):\n    with open(filename, 'w') as f:\n        output_json = json.dumps(json_annotation)\n        f.write(output_json)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dataset2coco(df, dest_path):\n    \n    global annotion_id\n    \n    annotations_json = {\n        \"info\": [],\n        \"licenses\": [],\n        \"categories\": [],\n        \"images\": [],\n        \"annotations\": []\n    }\n    \n    info = {\n        \"year\": \"2021\",\n        \"version\": \"1\",\n        \"description\": \"COTS dataset - COCO format\",\n        \"contributor\": \"\",\n        \"url\": \"https://kaggle.com\",\n        \"date_created\": \"2021-11-30T15:01:26+00:00\"\n    }\n    annotations_json[\"info\"].append(info)\n    \n    lic = {\n            \"id\": 1,\n            \"url\": \"\",\n            \"name\": \"Unknown\"\n        }\n    annotations_json[\"licenses\"].append(lic)\n\n    classes = {\"id\": 0, \"name\": \"starfish\", \"supercategory\": \"none\"}\n\n    annotations_json[\"categories\"].append(classes)\n\n    \n    for ann_row in df.itertuples():\n            \n        images = {\n            \"id\": ann_row[0],\n            \"license\": 1,\n            \"file_name\": ann_row.image_id + '.jpg',\n            \"height\": ann_row.height,\n            \"width\": ann_row.width,\n            \"date_captured\": \"2021-11-30T15:01:26+00:00\"\n        }\n        \n        annotations_json[\"images\"].append(images)\n        \n        bbox_list = ann_row.bboxes\n        \n        for bbox in bbox_list:\n            b_width = bbox[2]\n            b_height = bbox[3]\n            \n            # some boxes in COTS are outside the image height and width\n            if (bbox[0] + bbox[2] > 1280):\n                b_width = 1280 - bbox[0] \n            if (bbox[1] + bbox[3] > 720):\n                b_height = 720 - bbox[1] \n                \n            image_annotations = {\n                \"id\": annotion_id,\n                \"image_id\": ann_row[0],\n                \"category_id\": 0,\n                \"bbox\": [bbox[0], bbox[1], b_width, b_height],\n                \"area\": bbox[2] * bbox[3],\n                \"segmentation\": [],\n                \"iscrowd\": 0\n            }\n            \n            annotion_id += 1\n            annotations_json[\"annotations\"].append(image_annotations)\n        \n        \n    print(f\"Dataset COTS annotation to COCO json format completed! Files: {len(df)}\")\n    return annotations_json","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert COTS dataset to JSON COCO\ntrain_annot_json = dataset2coco(df_train[df_train.fold != Selected_Fold], f\"{Work_Dir}{DataSet_Path}/train2017/\")\nval_annot_json = dataset2coco(df_train[df_train.fold == Selected_Fold], f\"{Work_Dir}{DataSet_Path}/val2017/\")\n\n# Save converted annotations\nsave_annot_json(train_annot_json, f\"{Work_Dir}{DataSet_Path}/annotations/train.json\")\nsave_annot_json(val_annot_json, f\"{Work_Dir}{DataSet_Path}/annotations/valid.json\")","metadata":{},"execution_count":null,"outputs":[]}]}