{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Converting the dataset to the COCO format\n\nA lot of pre-trained instance segmentation models are trained on the [COCO](https://cocodataset.org/#home) dataset, so for easy fine tuning it would be useful to transform our datset to the same format. \n \n","metadata":{}},{"cell_type":"code","source":"import json\nfrom pathlib import Path\nimport os\nimport shutil\nimport cv2\nimport itertools\nimport numpy as np\nfrom typing import List, Dict\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2023-06-10T11:51:24.421305Z","iopub.execute_input":"2023-06-10T11:51:24.421786Z","iopub.status.idle":"2023-06-10T11:51:24.429544Z","shell.execute_reply.started":"2023-06-10T11:51:24.421744Z","shell.execute_reply":"2023-06-10T11:51:24.428198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_DIR = Path('/kaggle/input/hubmap-hacking-the-human-vasculature/')","metadata":{"execution":{"iopub.status.busy":"2023-06-10T11:51:24.431912Z","iopub.execute_input":"2023-06-10T11:51:24.433252Z","iopub.status.idle":"2023-06-10T11:51:24.451304Z","shell.execute_reply.started":"2023-06-10T11:51:24.433203Z","shell.execute_reply":"2023-06-10T11:51:24.449893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read .jsonl file and convert it to a list of dicts\n# The dicts contain IDs, class names and segmentation masks\n# from https://www.kaggle.com/code/leonidkulyk/eda-hubmap-hhv-interactive-annotations\nwith open('/kaggle/input/hubmap-hacking-the-human-vasculature/polygons.jsonl', 'r') as json_file:\n    json_list = list(json_file)\n    \ntiles_dicts = []\nfor json_str in json_list:\n    tiles_dicts.append(json.loads(json_str))","metadata":{"execution":{"iopub.status.busy":"2023-06-10T11:51:24.453067Z","iopub.execute_input":"2023-06-10T11:51:24.454195Z","iopub.status.idle":"2023-06-10T11:51:28.723209Z","shell.execute_reply.started":"2023-06-10T11:51:24.454157Z","shell.execute_reply":"2023-06-10T11:51:28.722093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define a conversion between class name and number\nid_dict = {'blood_vessel': 0, 'glomerulus': 1, 'unsure': 2}","metadata":{"execution":{"iopub.status.busy":"2023-06-10T11:51:28.726412Z","iopub.execute_input":"2023-06-10T11:51:28.727095Z","iopub.status.idle":"2023-06-10T11:51:28.732500Z","shell.execute_reply.started":"2023-06-10T11:51:28.727051Z","shell.execute_reply":"2023-06-10T11:51:28.731272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to copy images and transform labels to \n# coco formatted .txt files\ndef tile_to_coco(tile: List[Dict], output_folder: Path):\n    tile_id = tile['id']    \n    \n    # Copy image\n    shutil.copyfile(DATA_DIR / f'train/{tile_id}.tif', output_folder / f'{tile_id}.tif')\n    \n    # Create text file and write formatted labels to it\n    with open(output_folder / f'{tile_id}.txt', 'w') as text_file:\n        for annotation in tile['annotations']:\n            \n            class_id = id_dict[annotation['type']]\n            flat_mask_polygon = list(itertools.chain(*annotation['coordinates'][0]))\n            # Divide by 512 because coco labels expect positions between 0 and 1\n            # not pixel indices\n            array = np.array(flat_mask_polygon)/512.\n            text_file.write(f'{class_id} {\" \".join(map(str, array))}\\n')\n            \n        ","metadata":{"execution":{"iopub.status.busy":"2023-06-10T11:51:28.733957Z","iopub.execute_input":"2023-06-10T11:51:28.734298Z","iopub.status.idle":"2023-06-10T11:51:28.747888Z","shell.execute_reply.started":"2023-06-10T11:51:28.734265Z","shell.execute_reply":"2023-06-10T11:51:28.746487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split into train and validation \ntrain_dicts, valid_dicts = train_test_split(tiles_dicts, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T11:51:28.750626Z","iopub.execute_input":"2023-06-10T11:51:28.751061Z","iopub.status.idle":"2023-06-10T11:51:29.115397Z","shell.execute_reply.started":"2023-06-10T11:51:28.751028Z","shell.execute_reply":"2023-06-10T11:51:29.114425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir('/kaggle/working/train/')\nos.mkdir('/kaggle/working/valid/')","metadata":{"execution":{"iopub.status.busy":"2023-06-10T11:51:29.116930Z","iopub.execute_input":"2023-06-10T11:51:29.117604Z","iopub.status.idle":"2023-06-10T11:51:29.167292Z","shell.execute_reply.started":"2023-06-10T11:51:29.117567Z","shell.execute_reply":"2023-06-10T11:51:29.164907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for train_dict in train_dicts: \n    tile_to_coco(train_dict, Path('/kaggle/working/train/'))","metadata":{"execution":{"iopub.status.busy":"2023-06-10T11:51:29.168392Z","iopub.status.idle":"2023-06-10T11:51:29.168859Z","shell.execute_reply.started":"2023-06-10T11:51:29.168626Z","shell.execute_reply":"2023-06-10T11:51:29.168645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for valid_dict in valid_dicts: \n    tile_to_coco(valid_dict, Path('/kaggle/working/valid/'))","metadata":{"execution":{"iopub.status.busy":"2023-06-10T11:51:29.170190Z","iopub.status.idle":"2023-06-10T11:51:29.170630Z","shell.execute_reply.started":"2023-06-10T11:51:29.170424Z","shell.execute_reply":"2023-06-10T11:51:29.170442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a yaml file as expected by YOLOv7 (and others)\nyaml_text = \"\"\"\n# HuBMAP - Hacking the Human Vasculature dataset \n# https://www.kaggle.com/competitions/hubmap-hacking-the-human-vasculature\n\n\n# train and val data as 1) directory: path/images/, 2) file: path/images.txt, or 3) list: [path1/images/, path2/images/]\ntrain: /kaggle/input/hubmap-hhv-coco/train/\nval: /kaggle/input/hubmap-hhv-coco/valid/\n\n# class names\nnames: \n  0: blood_vessel\n  1: glomerulus\n  2: unsure\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-06-10T14:35:41.073179Z","iopub.execute_input":"2023-06-10T14:35:41.074016Z","iopub.status.idle":"2023-06-10T14:35:41.082640Z","shell.execute_reply.started":"2023-06-10T14:35:41.073602Z","shell.execute_reply":"2023-06-10T14:35:41.080570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('/kaggle/working/hubmap-coco.yaml', 'w') as text_file:\n    text_file.write(yaml_text)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T14:35:41.285950Z","iopub.execute_input":"2023-06-10T14:35:41.286380Z","iopub.status.idle":"2023-06-10T14:35:41.295352Z","shell.execute_reply.started":"2023-06-10T14:35:41.286346Z","shell.execute_reply":"2023-06-10T14:35:41.292879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat /kaggle/working/hubmap-coco.yaml","metadata":{"execution":{"iopub.status.busy":"2023-06-10T14:35:41.630178Z","iopub.execute_input":"2023-06-10T14:35:41.631479Z","iopub.status.idle":"2023-06-10T14:35:41.904735Z","shell.execute_reply.started":"2023-06-10T14:35:41.631430Z","shell.execute_reply":"2023-06-10T14:35:41.903311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}