{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Converting the dataset to the COCO format\n\nA lot of pre-trained instance segmentation models are trained on the [COCO](https://cocodataset.org/#home) dataset, so for easy fine tuning it would be useful to transform our datset to the same format. \n \n","metadata":{}},{"cell_type":"code","source":"import json\nfrom pathlib import Path\nimport os\nimport shutil\nimport cv2\nimport itertools\nimport numpy as np\nfrom typing import List, Dict\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2023-06-22T08:56:28.345585Z","iopub.execute_input":"2023-06-22T08:56:28.345957Z","iopub.status.idle":"2023-06-22T08:56:30.340389Z","shell.execute_reply.started":"2023-06-22T08:56:28.345927Z","shell.execute_reply":"2023-06-22T08:56:30.338937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_DIR = Path('/kaggle/input/hubmap-hacking-the-human-vasculature/')","metadata":{"execution":{"iopub.status.busy":"2023-06-22T08:56:36.015740Z","iopub.execute_input":"2023-06-22T08:56:36.016174Z","iopub.status.idle":"2023-06-22T08:56:36.022503Z","shell.execute_reply.started":"2023-06-22T08:56:36.016138Z","shell.execute_reply":"2023-06-22T08:56:36.020796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read .jsonl file and convert it to a list of dicts\n# The dicts contain IDs, class names and segmentation masks\n# from https://www.kaggle.com/code/leonidkulyk/eda-hubmap-hhv-interactive-annotations\nwith open('/kaggle/input/hubmap-hacking-the-human-vasculature/polygons.jsonl', 'r') as json_file:\n    json_list = list(json_file)\n    \ntiles_dicts = []\nfor json_str in json_list:\n    tiles_dicts.append(json.loads(json_str))","metadata":{"execution":{"iopub.status.busy":"2023-06-22T08:57:01.635844Z","iopub.execute_input":"2023-06-22T08:57:01.636264Z","iopub.status.idle":"2023-06-22T08:57:06.580417Z","shell.execute_reply.started":"2023-06-22T08:57:01.636234Z","shell.execute_reply":"2023-06-22T08:57:06.578989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define a conversion between class name and number\nid_dict = {'blood_vessel': 0, 'glomerulus': 1, 'unsure': 2}","metadata":{"execution":{"iopub.status.busy":"2023-06-22T09:02:36.126825Z","iopub.execute_input":"2023-06-22T09:02:36.127271Z","iopub.status.idle":"2023-06-22T09:02:36.134277Z","shell.execute_reply.started":"2023-06-22T09:02:36.127237Z","shell.execute_reply":"2023-06-22T09:02:36.131392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"array=list(itertools.chain(*tiles_dicts[0]['annotations'][0]['coordinates'][0]))","metadata":{"execution":{"iopub.status.busy":"2023-06-22T09:18:02.731163Z","iopub.execute_input":"2023-06-22T09:18:02.731569Z","iopub.status.idle":"2023-06-22T09:18:02.738271Z","shell.execute_reply.started":"2023-06-22T09:18:02.731534Z","shell.execute_reply":"2023-06-22T09:18:02.736623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"array= np.array(array)/512","metadata":{"execution":{"iopub.status.busy":"2023-06-22T09:18:23.895648Z","iopub.execute_input":"2023-06-22T09:18:23.897412Z","iopub.status.idle":"2023-06-22T09:18:23.904007Z","shell.execute_reply.started":"2023-06-22T09:18:23.897355Z","shell.execute_reply":"2023-06-22T09:18:23.902458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_id=1","metadata":{"execution":{"iopub.status.busy":"2023-06-22T09:18:31.905593Z","iopub.execute_input":"2023-06-22T09:18:31.906080Z","iopub.status.idle":"2023-06-22T09:18:31.911843Z","shell.execute_reply.started":"2023-06-22T09:18:31.906023Z","shell.execute_reply":"2023-06-22T09:18:31.910482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# f'{class_id} {\" \".join(map(str, array))}\\n'.count(\" \")","metadata":{"execution":{"iopub.status.busy":"2023-06-22T09:20:32.435807Z","iopub.execute_input":"2023-06-22T09:20:32.436265Z","iopub.status.idle":"2023-06-22T09:20:32.441672Z","shell.execute_reply.started":"2023-06-22T09:20:32.436232Z","shell.execute_reply":"2023-06-22T09:20:32.440454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to copy images and transform labels to \n# coco formatted .txt files\ndef tile_to_coco(tile: List[Dict], output_folder: Path):\n    tile_id = tile['id']    \n    \n    # Copy image\n    shutil.copyfile(DATA_DIR / f'train/{tile_id}.tif', output_folder / f'{tile_id}.tif')\n    \n    # Create text file and write formatted labels to it\n    with open(output_folder / f'{tile_id}.txt', 'w') as text_file:\n        for annotation in tile['annotations']:\n            \n            class_id = id_dict[annotation['type']]\n            flat_mask_polygon = list(itertools.chain(*annotation['coordinates'][0]))\n            # Divide by 512 because coco labels expect positions between 0 and 1\n            # not pixel indices\n            array = np.array(flat_mask_polygon)/512.\n            text_file.write(f'{class_id} {\" \".join(map(str, array))}\\n')\n            \n        ","metadata":{"execution":{"iopub.status.busy":"2023-06-22T09:20:43.565564Z","iopub.execute_input":"2023-06-22T09:20:43.565982Z","iopub.status.idle":"2023-06-22T09:20:43.575013Z","shell.execute_reply.started":"2023-06-22T09:20:43.565949Z","shell.execute_reply":"2023-06-22T09:20:43.573508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split into train and validation \ntrain_dicts, valid_dicts = train_test_split(tiles_dicts, test_size=0.30, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-06-22T09:24:08.547024Z","iopub.execute_input":"2023-06-22T09:24:08.547465Z","iopub.status.idle":"2023-06-22T09:24:08.554663Z","shell.execute_reply.started":"2023-06-22T09:24:08.547431Z","shell.execute_reply":"2023-06-22T09:24:08.553360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir('/kaggle/working/train/')\nos.mkdir('/kaggle/working/valid/')","metadata":{"execution":{"iopub.status.busy":"2023-06-22T09:24:46.457404Z","iopub.execute_input":"2023-06-22T09:24:46.458203Z","iopub.status.idle":"2023-06-22T09:24:46.463626Z","shell.execute_reply.started":"2023-06-22T09:24:46.458162Z","shell.execute_reply":"2023-06-22T09:24:46.462434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for train_dict in train_dicts: \n    tile_to_coco(train_dict, Path('/kaggle/working/train/'))","metadata":{"execution":{"iopub.status.busy":"2023-06-22T09:25:13.536984Z","iopub.execute_input":"2023-06-22T09:25:13.537433Z","iopub.status.idle":"2023-06-22T09:25:21.638397Z","shell.execute_reply.started":"2023-06-22T09:25:13.537399Z","shell.execute_reply":"2023-06-22T09:25:21.637087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for valid_dict in valid_dicts: \n    tile_to_coco(valid_dict, Path('/kaggle/working/valid/'))","metadata":{"execution":{"iopub.status.busy":"2023-06-22T09:25:36.345981Z","iopub.execute_input":"2023-06-22T09:25:36.346387Z","iopub.status.idle":"2023-06-22T09:25:57.099244Z","shell.execute_reply.started":"2023-06-22T09:25:36.346355Z","shell.execute_reply":"2023-06-22T09:25:57.098091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a yaml file as expected by YOLOv7 (and others)\nyaml_text = \"\"\"\n# HuBMAP - Hacking the Human Vasculature dataset \n# https://www.kaggle.com/competitions/hubmap-hacking-the-human-vasculature\n\n\n# train and val data as 1) directory: path/images/, 2) file: path/images.txt, or 3) list: [path1/images/, path2/images/]\ntrain: /kaggle/input/hubmap-hhv-coco/train/\nval: /kaggle/input/hubmap-hhv-coco/valid/\n\n# class names\nnames: \n  0: blood_vessel\n  1: glomerulus\n  2: unsure\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-06-10T14:35:41.073179Z","iopub.execute_input":"2023-06-10T14:35:41.074016Z","iopub.status.idle":"2023-06-10T14:35:41.08264Z","shell.execute_reply.started":"2023-06-10T14:35:41.073602Z","shell.execute_reply":"2023-06-10T14:35:41.08057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('/kaggle/working/hubmap-coco.yaml', 'w') as text_file:\n    text_file.write(yaml_text)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T14:35:41.28595Z","iopub.execute_input":"2023-06-10T14:35:41.28638Z","iopub.status.idle":"2023-06-10T14:35:41.295352Z","shell.execute_reply.started":"2023-06-10T14:35:41.286346Z","shell.execute_reply":"2023-06-10T14:35:41.292879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat /kaggle/working/hubmap-coco.yaml","metadata":{"execution":{"iopub.status.busy":"2023-06-10T14:35:41.630178Z","iopub.execute_input":"2023-06-10T14:35:41.631479Z","iopub.status.idle":"2023-06-10T14:35:41.904735Z","shell.execute_reply.started":"2023-06-10T14:35:41.63143Z","shell.execute_reply":"2023-06-10T14:35:41.903311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}