{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import Libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport json\nfrom PIL import Image\nfrom collections import Counter\n\nimport numpy as np\nimport pandas as pd\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport tifffile as tiff\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\n\nimport cv2","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-05T05:58:01.426182Z","iopub.execute_input":"2023-07-05T05:58:01.426716Z","iopub.status.idle":"2023-07-05T05:58:02.499060Z","shell.execute_reply.started":"2023-07-05T05:58:01.426686Z","shell.execute_reply":"2023-07-05T05:58:02.498098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read JSONL and Convert it to a List of Dicts","metadata":{}},{"cell_type":"code","source":"POLY_ANNOT_PATH = \"/kaggle/input/hubmap-hacking-the-human-vasculature/polygons.jsonl\"\nTILE_META_PATH = \"/kaggle/input/hubmap-hacking-the-human-vasculature/tile_meta.csv\"\nTRAIN_PATH = \"/kaggle/input/hubmap-hacking-the-human-vasculature/train\"\n\ntile_meta_df = pd.read_csv(TILE_META_PATH)\n\nwith open(POLY_ANNOT_PATH, \"r\") as json_file:\n    json_list = list(json_file)\n    \ntiles_dicts = []\nfor json_str in json_list:\n    tiles_dicts.append(json.loads(json_str))","metadata":{"execution":{"iopub.status.busy":"2023-07-05T05:58:07.223343Z","iopub.execute_input":"2023-07-05T05:58:07.223726Z","iopub.status.idle":"2023-07-05T05:58:10.856999Z","shell.execute_reply.started":"2023-07-05T05:58:07.223698Z","shell.execute_reply":"2023-07-05T05:58:10.855492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Functions","metadata":{}},{"cell_type":"code","source":"def get_coords(coords, img_height):\n    coords_arr = np.array(coords).squeeze()\n    xs = coords_arr[:, 0]\n    ys = -coords_arr[:, 1] + img_height\n    return xs, ys\n\ndef plot_annotated_image(img_dict, scale_factor:int = 1):\n    filename = os.path.join(TRAIN_PATH, img_dict[\"id\"]) + \".tif\"\n    arr = tiff.imread(filename)\n    img_ex = Image.fromarray(arr)\n    annotations = img_dict[\"annotations\"]\n    \n    fig = go.Figure()\n    img_width = img_ex.size[0]\n    img_height = img_ex.size[1]\n    \n    fig.add_trace(\n        go.Scatter(\n            x=[0, img_width],\n            y=[0, img_height],\n            mode=\"markers\",\n            marker_opacity=0,\n        )\n    )\n    \n    fig.update_xaxes(\n        visible=False,\n        range=[0, img_width],\n    )\n    \n    fig.update_yaxes(\n        visible=False,\n        range=[0, img_height],\n        scaleanchor=\"x\",\n    )\n    \n    fig.add_layout_image(dict(\n        x=0,\n        sizex=img_width,\n        y=img_height,\n        sizey=img_height,\n        xref=\"x\", yref=\"y\",\n        opacity=1.0,\n        layer=\"below\",\n        sizing=\"stretch\",\n        source=img_ex,\n    ))\n    \n    for annot in annotations:\n        name = annot[\"type\"]\n        xs, ys = get_coords(annot[\"coordinates\"], img_height)\n        fig.add_trace(go.Scatter(\n            x=xs, y=ys, fill=\"toself\", name=name,\n            hovertemplate=\"%{name}\",\n            mode=\"lines\",\n        ))\n        \n    fig.update_layout(\n        width=img_width*scale_factor,\n        height=img_height*scale_factor,\n        margin={\"l\":0, \"r\":0, \"t\":0, \"b\":0},\n        showlegend=False,\n    )\n    \n    fig.show(config={\"doubleClick\": \"reset\"})","metadata":{"execution":{"iopub.status.busy":"2023-07-05T06:08:29.685487Z","iopub.execute_input":"2023-07-05T06:08:29.685876Z","iopub.status.idle":"2023-07-05T06:08:29.696435Z","shell.execute_reply.started":"2023-07-05T06:08:29.685842Z","shell.execute_reply":"2023-07-05T06:08:29.695438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_annotated_image(tiles_dicts[0])","metadata":{"execution":{"iopub.status.busy":"2023-07-05T06:08:32.241787Z","iopub.execute_input":"2023-07-05T06:08:32.242122Z","iopub.status.idle":"2023-07-05T06:08:32.653861Z","shell.execute_reply.started":"2023-07-05T06:08:32.242096Z","shell.execute_reply":"2023-07-05T06:08:32.652629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualization with lines\nmask = np.zeros((512, 512), dtype=np.float32)\nfor annot in tiles_dicts[0][\"annotations\"]:\n    cords = annot[\"coordinates\"]\n    if annot[\"type\"] == \"blood_vessel\":\n        for cd in cords:\n            rr, cc = np.array([i[0] for i in cd]), np.asarray([j[1] for j in cd])\n            mask[rr, cc] = 1\n            \nplt.imshow(mask)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T06:08:36.504258Z","iopub.execute_input":"2023-07-05T06:08:36.504640Z","iopub.status.idle":"2023-07-05T06:08:36.706595Z","shell.execute_reply.started":"2023-07-05T06:08:36.504613Z","shell.execute_reply":"2023-07-05T06:08:36.705216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualization with filled contours - to create mask\ncontours, _ = cv2.findContours((mask*255).astype(np.uint8), 1, 2)\nzero_img = np.zeros([mask.shape[0], mask.shape[1], 3], dtype=\"uint8\")\n\nfor p in contours:\n    cv2.fillPoly(zero_img, [p], (255, 255, 255))\n    \nplt.imshow(zero_img)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T06:08:40.974830Z","iopub.execute_input":"2023-07-05T06:08:40.975156Z","iopub.status.idle":"2023-07-05T06:08:41.176516Z","shell.execute_reply.started":"2023-07-05T06:08:40.975133Z","shell.execute_reply":"2023-07-05T06:08:41.175664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Segmentation Mask","metadata":{}},{"cell_type":"code","source":"def make_seg_mask(tiles_dict):\n    mask = np.zeros((512, 512), dtype=np.float32)\n    for annot in tiles_dict[\"annotations\"]:\n        cords = annot[\"coordinates\"]\n        if annot[\"type\"] == \"blood_vessel\":\n            for cd in cords:\n                rr, cc = np.array([i[1] for i in cd]), np.asarray([i[0] for i in cd])\n                mask[rr, cc] = 1\n    contours, _ = cv2.findContours((mask*255).astype(np.uint8), 1, 2)\n    zero_img = np.zeros([mask.shape[0], mask.shape[1], 3], dtype=\"uint8\")\n    \n    for p in contours:\n        cv2.fillPoly(zero_img, [p], (255, 255, 255))\n    \n    contours, hierarchy = cv2.findContours(mask.astype(\"uint8\"), cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)\n    img_with_area = zero_img\n    \n    for i in range(len(contours)):\n        cv2.fillPoly(img_with_area, [contours[i][:,0,:]], (255-4*(i+1),255-4*(i+1),255-4*(i+1)), lineType=cv2.LINE_8, shift=0)\n            \n    return img_with_area ","metadata":{"execution":{"iopub.status.busy":"2023-07-05T06:31:27.865449Z","iopub.execute_input":"2023-07-05T06:31:27.865787Z","iopub.status.idle":"2023-07-05T06:31:27.875510Z","shell.execute_reply.started":"2023-07-05T06:31:27.865762Z","shell.execute_reply":"2023-07-05T06:31:27.874026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('train/image', exist_ok=True)\nos.makedirs('train/mask', exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T06:31:29.195101Z","iopub.execute_input":"2023-07-05T06:31:29.195454Z","iopub.status.idle":"2023-07-05T06:31:29.199368Z","shell.execute_reply.started":"2023-07-05T06:31:29.195431Z","shell.execute_reply":"2023-07-05T06:31:29.198822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, tldc in enumerate(tqdm(tiles_dicts)):\n    path = os.path.join(TRAIN_PATH, tldc[\"id\"]) + \".tif\"\n    arr = tiff.imread(path)\n    img_ex = Image.fromarray(arr)\n    img = np.array(img_ex)\n    mask = make_seg_mask(tldc)\n    \n    if np.sum(mask) > 0:\n        cv2.imwrite(f\"train/image/{tldc['id']}.png\", img)\n        cv2.imwrite(f\"train/mask/{tldc['id']}.png\", mask)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T20:58:26.349981Z","iopub.execute_input":"2023-07-04T20:58:26.350390Z","iopub.status.idle":"2023-07-04T20:58:30.051741Z","shell.execute_reply.started":"2023-07-04T20:58:26.350359Z","shell.execute_reply":"2023-07-04T20:58:30.050167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Reference: https://www.kaggle.com/code/itsuki9180/hubmap-making-dataset","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}