{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport pandas as pd\nfrom tqdm import tqdm, notebook\nfrom collections import Counter\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2023-07-20T16:36:31.797734Z","iopub.execute_input":"2023-07-20T16:36:31.798023Z","iopub.status.idle":"2023-07-20T16:36:31.889450Z","shell.execute_reply.started":"2023-07-20T16:36:31.797997Z","shell.execute_reply":"2023-07-20T16:36:31.888044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import Counter","metadata":{"execution":{"iopub.status.busy":"2023-07-20T16:36:31.891058Z","iopub.execute_input":"2023-07-20T16:36:31.891412Z","iopub.status.idle":"2023-07-20T16:36:31.896240Z","shell.execute_reply.started":"2023-07-20T16:36:31.891384Z","shell.execute_reply":"2023-07-20T16:36:31.894900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#files path\npolygons_path='/kaggle/input/hubmap-hacking-the-human-vasculature/polygons.jsonl'\nsample_submission_path='/kaggle/input/hubmap-hacking-the-human-vasculature/sample_submission.csv'\ntile_meta_path='/kaggle/input/hubmap-hacking-the-human-vasculature/tile_meta.csv'\nwsi_meta_path='/kaggle/input/hubmap-hacking-the-human-vasculature/wsi_meta.csv'\n\n#folders path\ntrain_path='/kaggle/input/hubmap-hacking-the-human-vasculature/train/'\ntest_path='/kaggle/input/hubmap-hacking-the-human-vasculature/test/'","metadata":{"execution":{"iopub.status.busy":"2023-07-20T16:36:31.898875Z","iopub.execute_input":"2023-07-20T16:36:31.899367Z","iopub.status.idle":"2023-07-20T16:36:31.908697Z","shell.execute_reply.started":"2023-07-20T16:36:31.899340Z","shell.execute_reply":"2023-07-20T16:36:31.907677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission=pd.read_csv(sample_submission_path)\ntile_meta=pd.read_csv(tile_meta_path)\nwsi_meta=pd.read_csv(wsi_meta_path)","metadata":{"execution":{"iopub.status.busy":"2023-07-20T16:36:31.910419Z","iopub.execute_input":"2023-07-20T16:36:31.911014Z","iopub.status.idle":"2023-07-20T16:36:31.958134Z","shell.execute_reply.started":"2023-07-20T16:36:31.910984Z","shell.execute_reply":"2023-07-20T16:36:31.957323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The competition data includes small pieces called \"tiles\" that come from five big pictures called \"Whole Slide Images\" (WSI). These WSIs are split into two groups called datasets. In **Dataset 1**, the **tiles have been looked at by experts who reviewed and marked them**. In **Dataset 2**, the tiles are from the same big pictures but **they don't have as many marks, and the marks they do have haven't been reviewed by experts**.","metadata":{}},{"cell_type":"markdown","source":"### Number of tiles in each known WSI","metadata":{}},{"cell_type":"code","source":"tile_count=tile_meta['source_wsi'].value_counts()\ntile_count","metadata":{"execution":{"iopub.status.busy":"2023-07-20T16:53:31.395485Z","iopub.execute_input":"2023-07-20T16:53:31.395822Z","iopub.status.idle":"2023-07-20T16:53:31.404032Z","shell.execute_reply.started":"2023-07-20T16:53:31.395797Z","shell.execute_reply":"2023-07-20T16:53:31.402960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get number of tiles belonging to each wsi,every tiles has their own annotations(experted or non-experted)\nwsi4=tile_count.values[-1]\nwsi3=tile_count.values[-2]\nwsi2=tile_count.values[-3]\nwsi1=tile_count.values[-4]\n#print(wsi1,wsi2,wsi3,wsi4)","metadata":{"execution":{"iopub.status.busy":"2023-07-20T16:55:55.687860Z","iopub.execute_input":"2023-07-20T16:55:55.688212Z","iopub.status.idle":"2023-07-20T16:55:55.694424Z","shell.execute_reply.started":"2023-07-20T16:55:55.688163Z","shell.execute_reply":"2023-07-20T16:55:55.693373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"values = [tile_count.values[x] for x in range(-4,0,1)] \nvalues","metadata":{"execution":{"iopub.status.busy":"2023-07-20T17:01:43.467885Z","iopub.execute_input":"2023-07-20T17:01:43.468238Z","iopub.status.idle":"2023-07-20T17:01:43.475111Z","shell.execute_reply.started":"2023-07-20T17:01:43.468206Z","shell.execute_reply":"2023-07-20T17:01:43.474149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# make the pie circular by setting the aspect ratio to 1\n# plt.figure(figsize=plt.figaspect(1))\nvalues = [tile_count.values[x] for x in range(-4,0,1)] \nlabels = ['WSI1', 'WSI2', 'WSI3', 'WSI4'] \n\ndef make_autopct(values):\n    def my_autopct(pct):\n        total = sum(values)\n        val = int(round(pct*total/100.0))\n        # 同时显示数值和占比的饼图\n        return '{p:.2f}%  ({v:d})'.format(p=pct,v=val)\n    return my_autopct\n\nplt.pie(values, labels=labels, autopct=make_autopct(values))\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-07-20T17:02:18.041508Z","iopub.execute_input":"2023-07-20T17:02:18.041831Z","iopub.status.idle":"2023-07-20T17:02:18.150407Z","shell.execute_reply.started":"2023-07-20T17:02:18.041801Z","shell.execute_reply":"2023-07-20T17:02:18.149662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_count=tile_meta['dataset'].value_counts()\nfor a,b in enumerate(dataset_count):\n    print(a,b)","metadata":{"execution":{"iopub.status.busy":"2023-07-20T16:47:07.809575Z","iopub.execute_input":"2023-07-20T16:47:07.809890Z","iopub.status.idle":"2023-07-20T16:47:07.816532Z","shell.execute_reply.started":"2023-07-20T16:47:07.809868Z","shell.execute_reply":"2023-07-20T16:47:07.815391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.bar(list(map(str, dataset_count.index)), dataset_count.values)\n\nplt.xlabel('Dataset Number')\nplt.ylabel('Number of Examples')\nplt.title('Number of examples in each dataset')\n\n\nfor a,b in enumerate(dataset_count):  \n     plt.text(a, b+0.05, '%.0f' % b, ha='center', va= 'bottom',fontsize=11)  \n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-20T16:48:52.020835Z","iopub.execute_input":"2023-07-20T16:48:52.021222Z","iopub.status.idle":"2023-07-20T16:48:52.187094Z","shell.execute_reply.started":"2023-07-20T16:48:52.021166Z","shell.execute_reply":"2023-07-20T16:48:52.186252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### WSI source Distribution in dataset1","metadata":{}},{"cell_type":"code","source":"#WSI1\ntile_meta.loc[(tile_meta['source_wsi']==1)& (tile_meta['dataset']==1)]","metadata":{"execution":{"iopub.status.busy":"2023-07-20T16:36:32.000436Z","iopub.execute_input":"2023-07-20T16:36:32.000895Z","iopub.status.idle":"2023-07-20T16:36:32.024681Z","shell.execute_reply.started":"2023-07-20T16:36:32.000869Z","shell.execute_reply":"2023-07-20T16:36:32.023506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#number of tiles of wsi1 in the dataset1\nnum_w1d1=len(tile_meta.loc[(tile_meta['source_wsi']==1)& (tile_meta['dataset']==1)])\n#Store all ids of the WSI1 tiles in dataset1 respectively\nwsi1_inds1=tile_meta.loc[(tile_meta['source_wsi']==1)& (tile_meta['dataset']==1)]['id'].tolist()\n#number of tiles of wsi1 in the dataset2\nnum_w1d2=wsi1-num_w1d1","metadata":{"execution":{"iopub.status.busy":"2023-07-20T18:41:55.825131Z","iopub.execute_input":"2023-07-20T18:41:55.825508Z","iopub.status.idle":"2023-07-20T18:41:55.837742Z","shell.execute_reply.started":"2023-07-20T18:41:55.825477Z","shell.execute_reply":"2023-07-20T18:41:55.836112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#WSI2\ntile_meta.loc[(tile_meta['source_wsi']==2)& (tile_meta['dataset']==1)]","metadata":{"execution":{"iopub.status.busy":"2023-07-20T16:36:32.027044Z","iopub.execute_input":"2023-07-20T16:36:32.027533Z","iopub.status.idle":"2023-07-20T16:36:32.040733Z","shell.execute_reply.started":"2023-07-20T16:36:32.027507Z","shell.execute_reply":"2023-07-20T16:36:32.039602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#number of tiles of WSI2 in the dataset1\nnum_w2d1=len(tile_meta.loc[(tile_meta['source_wsi']==2)& (tile_meta['dataset']==1)])\n#Store all ids of the WSI1 tiles in dataset1 respectively\nwsi2_inds1=tile_meta.loc[(tile_meta['source_wsi']==2)& (tile_meta['dataset']==1)]['id'].tolist()\n#number of tiles of wsi2 in the dataset2\nnum_w2d2=wsi2-num_w2d1\n","metadata":{"execution":{"iopub.status.busy":"2023-07-20T18:42:32.401492Z","iopub.execute_input":"2023-07-20T18:42:32.401850Z","iopub.status.idle":"2023-07-20T18:42:32.413288Z","shell.execute_reply.started":"2023-07-20T18:42:32.401821Z","shell.execute_reply":"2023-07-20T18:42:32.411541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#WSI3\ntile_meta.loc[(tile_meta['source_wsi']==3)& (tile_meta['dataset']==1)]","metadata":{"execution":{"iopub.status.busy":"2023-07-20T16:36:32.042157Z","iopub.execute_input":"2023-07-20T16:36:32.042821Z","iopub.status.idle":"2023-07-20T16:36:32.054618Z","shell.execute_reply.started":"2023-07-20T16:36:32.042789Z","shell.execute_reply":"2023-07-20T16:36:32.053651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#number of tiles of WSI3 in the dataset1\nnum_w3d1=len(tile_meta.loc[(tile_meta['source_wsi']==3)& (tile_meta['dataset']==1)])\n#Store all ids of the WSI1 tiles in dataset1 respectively\nwsi3_inds1=tile_meta.loc[(tile_meta['source_wsi']==3)& (tile_meta['dataset']==1)]['id'].tolist()\n#number of tiles of wsi3 in the dataset2\nnum_w3d2=wsi3-num_w3d1\n","metadata":{"execution":{"iopub.status.busy":"2023-07-20T18:43:29.448474Z","iopub.execute_input":"2023-07-20T18:43:29.448897Z","iopub.status.idle":"2023-07-20T18:43:29.461310Z","shell.execute_reply.started":"2023-07-20T18:43:29.448864Z","shell.execute_reply":"2023-07-20T18:43:29.459822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#WSI4\ntile_meta.loc[(tile_meta['source_wsi']==4)& (tile_meta['dataset']==1)]","metadata":{"execution":{"iopub.status.busy":"2023-07-20T16:36:32.055936Z","iopub.execute_input":"2023-07-20T16:36:32.056438Z","iopub.status.idle":"2023-07-20T16:36:32.066599Z","shell.execute_reply.started":"2023-07-20T16:36:32.056410Z","shell.execute_reply":"2023-07-20T16:36:32.065593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#number of tiles of WSI4 in the dataset1\nnum_w4d1=len(tile_meta.loc[(tile_meta['source_wsi']==3)& (tile_meta['dataset']==1)])\n#Store all ids of the WSI1 tiles in dataset1 respectively\nwsi4_inds1=tile_meta.loc[(tile_meta['source_wsi']==3)& (tile_meta['dataset']==1)]['id'].tolist()\n#number of tiles of wsi4 in the dataset2\nnum_w4d2=wsi4-num_w4d1","metadata":{"execution":{"iopub.status.busy":"2023-07-20T18:44:06.344258Z","iopub.execute_input":"2023-07-20T18:44:06.344613Z","iopub.status.idle":"2023-07-20T18:44:06.355463Z","shell.execute_reply.started":"2023-07-20T18:44:06.344582Z","shell.execute_reply":"2023-07-20T18:44:06.354686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wsi1_inds1[:5]","metadata":{"execution":{"iopub.status.busy":"2023-07-20T18:41:10.664073Z","iopub.execute_input":"2023-07-20T18:41:10.664446Z","iopub.status.idle":"2023-07-20T18:41:10.672748Z","shell.execute_reply.started":"2023-07-20T18:41:10.664415Z","shell.execute_reply":"2023-07-20T18:41:10.671233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wsi_meta.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-20T16:36:32.095574Z","iopub.execute_input":"2023-07-20T16:36:32.096200Z","iopub.status.idle":"2023-07-20T16:36:32.115056Z","shell.execute_reply.started":"2023-07-20T16:36:32.096150Z","shell.execute_reply":"2023-07-20T16:36:32.113456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"values = [num_w1d1,num_w2d1,num_w3d1,num_w4d1] \nlabels = ['WSI1', 'WSI2', 'WSI3', 'WSI4'] \n\nplt.title(\"Distribution of each WSI's tiles in Datset1\")\nplt.pie(values, labels=labels, autopct=make_autopct(values))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-20T18:49:11.874914Z","iopub.execute_input":"2023-07-20T18:49:11.875303Z","iopub.status.idle":"2023-07-20T18:49:12.004247Z","shell.execute_reply.started":"2023-07-20T18:49:11.875273Z","shell.execute_reply":"2023-07-20T18:49:12.003493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"values = [num_w1d2,num_w2d2,num_w3d2,num_w4d2] \nlabels = ['WSI1', 'WSI2', 'WSI3', 'WSI4'] \n\nplt.title(\"Distribution of each WSI's tiles in Datset2\")\nplt.pie(values, labels=labels, autopct=make_autopct(values))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-20T18:49:35.389870Z","iopub.execute_input":"2023-07-20T18:49:35.390259Z","iopub.status.idle":"2023-07-20T18:49:35.494790Z","shell.execute_reply.started":"2023-07-20T18:49:35.390229Z","shell.execute_reply":"2023-07-20T18:49:35.493519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Polygons","metadata":{}},{"cell_type":"markdown","source":"**polygons.jsonl**\n**Polygonal segmentation masks in JSONL format**, available for Dataset 1 and Dataset 2. Each line gives JSON annotations for a single image with:<br>\n* `id`  Identifies the corresponding image in train/<br>\n* `annotations` A **list of mask annotations** with:<br>\n    * `type` Identifies the **type of structure** annotated:<br>\n      * `blood_vessel` The **target structure**. Your goal in this competition is to predict these kinds of masks on the test set.<br>\n      * `glomerulus` A capillary ball structure in the kidney. **These parts of the images were excluded from blood vessel annotation.** You should ensure none of your test set predictions occur within glomerulus structures as they will be counted as false positives. Annotations are provided for test set tiles.<br>\n      * `unsure` A structure the expert annotators **cannot confidently distinguish as a blood vessel.**<br>\n    * `coordinates` A **list of polygon coordinates defining the segmentation mask**.\n","metadata":{}},{"cell_type":"code","source":"#load json data and convert it into a list of dicts\n#dicts contain IDs, class name and polygon annotation of segmentation masks\nimport json\nwith open('/kaggle/input/hubmap-hacking-the-human-vasculature/polygons.jsonl','r') as json_file:\n    json_list=list(json_file)\n    \ntiles_dicts=[]\nfor json_str in json_list:\n    tiles_dicts.append(json.loads(json_str))","metadata":{"execution":{"iopub.status.busy":"2023-07-20T22:19:27.619865Z","iopub.execute_input":"2023-07-20T22:19:27.620240Z","iopub.status.idle":"2023-07-20T22:19:30.711099Z","shell.execute_reply.started":"2023-07-20T22:19:27.620210Z","shell.execute_reply":"2023-07-20T22:19:30.709735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict_of_tiles={}\nfor tile in tiles_dicts:\n    dict_of_tiles[tile['id']]=tile['annotations']","metadata":{"execution":{"iopub.status.busy":"2023-07-20T22:22:23.211189Z","iopub.execute_input":"2023-07-20T22:22:23.211589Z","iopub.status.idle":"2023-07-20T22:22:23.218646Z","shell.execute_reply.started":"2023-07-20T22:22:23.211562Z","shell.execute_reply":"2023-07-20T22:22:23.217192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_glomerulus=0\nnum_blood_vessel=0\nnum_unsure=0\n# i=0\nfor x in dict_of_tiles:\n    for instance in dict_of_tiles[x]:\n        if instance['type']=='glomerulus':\n            num_glomerulus+=1\n        elif instance['type']=='blood_vessel':\n            num_blood_vessel+=1\n        else:\n            num_unsure+=1\n#     i+=1\n#     if i==2:\n#         break\n\nprint(num_glomerulus,num_blood_vessel,num_unsure)","metadata":{"execution":{"iopub.status.busy":"2023-07-20T22:46:57.143856Z","iopub.execute_input":"2023-07-20T22:46:57.144266Z","iopub.status.idle":"2023-07-20T22:46:57.160099Z","shell.execute_reply.started":"2023-07-20T22:46:57.144233Z","shell.execute_reply":"2023-07-20T22:46:57.158829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"values = [num_glomerulus,num_blood_vessel,num_unsure] \nlabels = ['glomerulus', 'blood_vessel', 'unsure'] \n\nplt.title(\"Percentage of each instance's type\")\nplt.pie(values, labels=labels, autopct=make_autopct(values))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-20T22:48:23.208744Z","iopub.execute_input":"2023-07-20T22:48:23.209084Z","iopub.status.idle":"2023-07-20T22:48:23.377896Z","shell.execute_reply.started":"2023-07-20T22:48:23.209061Z","shell.execute_reply":"2023-07-20T22:48:23.377145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}