{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **[HuBMAP 2023] K-fold CV COCO Dataset Generator**","metadata":{}},{"cell_type":"markdown","source":"### Versions\n* V7 - 2 Categories:  **{1: 'blood_vessel',2: 'glomerulus'}**\n* V6 - 1 Category:  **{1: 'blood_vessel'}**\n* V5 - Ignore it ,I made mistake!!!\n* V4 - 3 Categories:**{1: 'glomerulus', 2: 'blood_vessel', 3: 'unsure'}**","metadata":{}},{"cell_type":"code","source":"!pip install pycocotools -Uqq","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-07-24T10:09:44.151812Z","iopub.execute_input":"2023-07-24T10:09:44.152288Z","iopub.status.idle":"2023-07-24T10:09:54.807292Z","shell.execute_reply.started":"2023-07-24T10:09:44.152242Z","shell.execute_reply":"2023-07-24T10:09:54.805944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### importing libraries","metadata":{}},{"cell_type":"code","source":"import json, cv2, numpy as np, itertools, random, pandas as pd\nfrom pycocotools.coco import COCO\nfrom pycocotools import mask as maskUtils\nfrom skimage import io\nimport matplotlib.pyplot as plt\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\nfrom sklearn import model_selection\n\nimport matplotlib.pyplot as plt\nfrom skimage import io\nfrom pycocotools.coco import COCO\nimport matplotlib.patches as mpatches\n","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:09:54.809088Z","iopub.execute_input":"2023-07-24T10:09:54.809447Z","iopub.status.idle":"2023-07-24T10:09:54.816645Z","shell.execute_reply.started":"2023-07-24T10:09:54.809412Z","shell.execute_reply":"2023-07-24T10:09:54.815430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Functions","metadata":{}},{"cell_type":"code","source":"def coordinates_to_masks(coordinates, shape):\n    masks = []\n    for coord in coordinates:\n        mask = np.zeros(shape, dtype=np.uint8)\n        cv2.fillPoly(mask, [np.array(coord)], 1)\n        masks.append(mask)\n    return masks\n\ndef rle_encoding(x):\n    dots = np.where(x.flatten() == 1)[0]\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if (b>prev+1): run_lengths.extend((b + 1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return ' '.join(map(str, run_lengths))\n\ndef binary_mask_to_rle(binary_mask):\n    rle = {'counts': [], 'size': list(binary_mask.shape)}\n    counts = rle.get('counts')\n    for i, (value, elements) in enumerate(itertools.groupby(binary_mask.ravel(order='F'))):\n        if i == 0 and value == 1:\n            counts.append(0)\n        counts.append(len(list(elements)))\n    return rle\n\ndef rle_to_binary_mask(mask_rle, shape=(512, 512)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) \n                       for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo : hi] = 1\n    return img.reshape(shape)  # Needed to align to RLE direction","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:09:54.818198Z","iopub.execute_input":"2023-07-24T10:09:54.818615Z","iopub.status.idle":"2023-07-24T10:09:54.832006Z","shell.execute_reply.started":"2023-07-24T10:09:54.818577Z","shell.execute_reply":"2023-07-24T10:09:54.830750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Loading Dataset","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/hubmap-hacking-the-human-vasculature/tile_meta.csv')\ndf = df.query('dataset != 3')\n#df=df.head(50)\ndf.reset_index(inplace=True,drop=True)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:09:54.835492Z","iopub.execute_input":"2023-07-24T10:09:54.835814Z","iopub.status.idle":"2023-07-24T10:09:54.870366Z","shell.execute_reply.started":"2023-07-24T10:09:54.835787Z","shell.execute_reply":"2023-07-24T10:09:54.869221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Spliting training & Valid","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\nn_splits=5\nskf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\nfor fold, (_, val_idx) in enumerate(skf.split(X=df, y=df['source_wsi']), 1):\n    df.loc[val_idx, 'fold'] = fold\n    \ndf['fold'] = df['fold'].astype(np.uint8)\ndf.groupby('fold').size()","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:09:54.872286Z","iopub.execute_input":"2023-07-24T10:09:54.873069Z","iopub.status.idle":"2023-07-24T10:09:54.890124Z","shell.execute_reply.started":"2023-07-24T10:09:54.873030Z","shell.execute_reply":"2023-07-24T10:09:54.889284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_fold=1\ntrain_ids = df.query(f'fold != {selected_fold}')['id'].values.tolist()\nvalid_ids = df.query(f'fold == {selected_fold}')['id'].values.tolist()\nprint(len(train_ids), len(valid_ids))","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:09:54.891838Z","iopub.execute_input":"2023-07-24T10:09:54.892619Z","iopub.status.idle":"2023-07-24T10:09:54.904850Z","shell.execute_reply.started":"2023-07-24T10:09:54.892580Z","shell.execute_reply":"2023-07-24T10:09:54.903686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Reading polygons.jsonl","metadata":{}},{"cell_type":"code","source":"jsonl_file_path = \"/kaggle/input/hubmap-hacking-the-human-vasculature/polygons.jsonl\"\ndata = []\nwith open(jsonl_file_path, \"r\") as file:\n    for line in file:\n        data.append(json.loads(line))","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:09:54.906432Z","iopub.execute_input":"2023-07-24T10:09:54.907099Z","iopub.status.idle":"2023-07-24T10:09:59.964253Z","shell.execute_reply.started":"2023-07-24T10:09:54.907058Z","shell.execute_reply":"2023-07-24T10:09:59.963315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Categories","metadata":{}},{"cell_type":"code","source":"categories_list=['blood_vessel']#,'glomerulus','unsure']\n#------------------------------------------------------------------------------\ncategories_ids = {name:id+1 for id, name in enumerate(categories_list)}  \nids_categories = {id+1:name for id, name in enumerate(categories_list)}  \ncategories =[{'id':id,'name':name} for name,id in categories_ids.items()]\n\nprint(categories_ids)\nprint(ids_categories)\nprint(categories)","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:09:59.965535Z","iopub.execute_input":"2023-07-24T10:09:59.965888Z","iopub.status.idle":"2023-07-24T10:09:59.973667Z","shell.execute_reply.started":"2023-07-24T10:09:59.965857Z","shell.execute_reply":"2023-07-24T10:09:59.972318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Creating COCO","metadata":{}},{"cell_type":"code","source":"def coco_structure(images_ids):\n    idx=1\n    annotations=[]\n    images=[]\n    for item in tqdm(data,total=int(len(images_ids))):\n        image_id=item[\"id\"]\n        if image_id in images_ids:\n            image = {\"id\": image_id, \"file_name\": image_id + \".tif\", \"height\": 512, \"width\": 512}\n            images.append(image)\n        else:continue\n        #-----------------------------\n        anns=item[\"annotations\"]\n        for an in anns:\n            category_type=an[\"type\"]\n            if category_type !=\"blood_vessel\": continue\n            category_id=categories_ids[category_type]\n            segmentation=an[\"coordinates\"]\n            mask_img = coordinates_to_masks(segmentation, (512, 512))[0]\n            ys, xs = np.where(mask_img)\n            x1, x2 = min(xs), max(xs)\n            y1, y2 = min(ys), max(ys)\n\n            rle = binary_mask_to_rle(mask_img)\n\n            seg = {\n                    \"id\": idx,\n                    \"image_id\": image_id,\n                    \"category_id\": category_id,\n                    \"segmentation\": rle,\n                    \"bbox\": [int(x1), int(y1), int(x2 - x1 + 1), int(y2 - y1 + 1)],\n                    \"area\": int(np.sum(mask_img)),\n                    \"iscrowd\": 0,\n                }\n            if image_id in images_ids:\n                annotations.append(seg)\n                idx=idx+1\n                \n    return {\"info\": {}, \"licenses\": [], \"categories\": categories, \"images\": images, \"annotations\": annotations}","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:09:59.975339Z","iopub.execute_input":"2023-07-24T10:09:59.976154Z","iopub.status.idle":"2023-07-24T10:09:59.989891Z","shell.execute_reply.started":"2023-07-24T10:09:59.976114Z","shell.execute_reply":"2023-07-24T10:09:59.988974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_coco_data = coco_structure(train_ids)\nvalid_coco_data = coco_structure(valid_ids)","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:09:59.992968Z","iopub.execute_input":"2023-07-24T10:09:59.993317Z","iopub.status.idle":"2023-07-24T10:10:09.687932Z","shell.execute_reply.started":"2023-07-24T10:09:59.993289Z","shell.execute_reply":"2023-07-24T10:10:09.686708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Saving COCO","metadata":{}},{"cell_type":"code","source":"output_file_path = f\"coco_annotations_train_class_{len(categories_list)}_folds{n_splits}_fold{selected_fold}.json\"\nwith open(output_file_path, \"w\", encoding=\"utf-8\") as output_file:\n    json.dump(train_coco_data, output_file, ensure_ascii=True, indent=4)\n    \noutput_file_path = f\"coco_annotations_valid_class_{len(categories_list)}_folds{n_splits}_fold{selected_fold}.json\"\nwith open(output_file_path, \"w\", encoding=\"utf-8\") as output_file:\n    json.dump(valid_coco_data, output_file, ensure_ascii=True, indent=4)","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:10:09.689246Z","iopub.execute_input":"2023-07-24T10:10:09.689580Z","iopub.status.idle":"2023-07-24T10:10:09.742472Z","shell.execute_reply.started":"2023-07-24T10:10:09.689550Z","shell.execute_reply":"2023-07-24T10:10:09.741379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Creating DataFrame","metadata":{}},{"cell_type":"code","source":"def dataframe_structure():\n    annotations=[]\n    for item in tqdm(data):\n        image_id=item[\"id\"]\n        anns=item[\"annotations\"]\n        #if not image_id in images_ids:continue\n        for an in anns:\n            category_type=an[\"type\"]\n            category_id=categories_ids[category_type]\n            segmentation=an[\"coordinates\"]\n            mask_img = coordinates_to_masks(segmentation, (512, 512))[0]\n            ys, xs = np.where(mask_img)\n            x1, x2 = min(xs), max(xs)\n            y1, y2 = min(ys), max(ys)\n            for index, row in df.query(f\"id=='{image_id}'\").iterrows():\n                seg = {\n                        \"image_id\": image_id,\n                        \"source_wsi\": row[\"source_wsi\"],\n                        \"dataset\": row[\"dataset\"],\n                        \"category_id\": int(category_id),\n                        \"category_name\": ids_categories[category_id],\n                        \"annotations\": rle_encoding(mask_img),\n                        \"bbox\": (int(x1), int(y1), int(x2 - x1 + 1), int(y2 - y1 + 1)),\n                        \"area\": int(np.sum(mask_img)),\n                        \"iscrowd\": 0,\n                    }\n                annotations.append(seg)\n                \n    return annotations","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:10:09.743803Z","iopub.execute_input":"2023-07-24T10:10:09.744228Z","iopub.status.idle":"2023-07-24T10:10:09.753030Z","shell.execute_reply.started":"2023-07-24T10:10:09.744198Z","shell.execute_reply":"2023-07-24T10:10:09.751981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#data_dict = dataframe_structure()\n#df_train = pd.DataFrame(data_dict)\n#df_train.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:10:09.754293Z","iopub.execute_input":"2023-07-24T10:10:09.754674Z","iopub.status.idle":"2023-07-24T10:10:09.763675Z","shell.execute_reply.started":"2023-07-24T10:10:09.754645Z","shell.execute_reply":"2023-07-24T10:10:09.762675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_train.to_csv(\"df_train.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:10:09.765293Z","iopub.execute_input":"2023-07-24T10:10:09.765795Z","iopub.status.idle":"2023-07-24T10:10:09.773507Z","shell.execute_reply.started":"2023-07-24T10:10:09.765757Z","shell.execute_reply":"2023-07-24T10:10:09.772590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!zip -r file.zip /kaggle/working\n#from IPython.display import FileLink\n#FileLink(r'file.zip')","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:10:09.774849Z","iopub.execute_input":"2023-07-24T10:10:09.775194Z","iopub.status.idle":"2023-07-24T10:10:09.782897Z","shell.execute_reply.started":"2023-07-24T10:10:09.775167Z","shell.execute_reply":"2023-07-24T10:10:09.782088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualization","metadata":{}},{"cell_type":"code","source":"dataDir = Path(\"/kaggle/input/hubmap-hacking-the-human-vasculature/train\")\nannFile = Path(f\"coco_annotations_train_class_{len(categories_list)}_folds{n_splits}_fold{selected_fold}.json\")\n\ncolors = ['Set1','Set3','Set3_r'] \nlegend = ids_categories #{1: 'blood_vessel',2:glomerulus....}\n\ncoco = COCO(annFile)\nimgIds = coco.getImgIds()\nimgs = coco.loadImgs(imgIds[0:4])\n\nfig, axs = plt.subplots(len(imgs), 2, figsize=(10, 5*len(imgs)))\nfor img, ax_row in zip(imgs, axs):\n    ax = ax_row[0]  # Access the first axis in each row\n    I = io.imread(dataDir / img[\"file_name\"])\n    annIds = coco.getAnnIds(imgIds=[img[\"id\"]])\n    anns = coco.loadAnns(annIds)\n    ax.imshow(I)\n    ax = ax_row[1]  # Access the second axis in each row\n    ax.imshow(I)\n    plt.sca(ax)\n    for i, ann in enumerate(anns):\n        category_id = ann['category_id']\n        color = colors[category_id-1]\n        #-----------------------------------------\n        mask = coco.annToMask(ann)\n        mask = np.ma.masked_where(mask == 0, mask)\n        ax.imshow(mask, cmap=color, alpha=0.8)\n        #-----------------------------------------\n        handles = []\n        for category_id in legend:\n            color = colors[category_id - 1]\n            handles.append(mpatches.Patch(color=plt.colormaps.get_cmap(color)(0)))\n        ax.legend(handles, legend.values(), bbox_to_anchor=(1.05, 1), loc='upper left')\n\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:11:02.811204Z","iopub.execute_input":"2023-07-24T10:11:02.811584Z","iopub.status.idle":"2023-07-24T10:11:05.439726Z","shell.execute_reply.started":"2023-07-24T10:11:02.811557Z","shell.execute_reply":"2023-07-24T10:11:05.438266Z"},"trusted":true},"execution_count":null,"outputs":[]}]}