{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n#
Input Data Quality Analysis
\n\n
\n
\n \"Sentiment\n
\n
","metadata":{}},{"cell_type":"markdown","source":"This notebook contains a simple overview of what we have.\n\n# Table of contents\n
\n\n\n- [1 | IDs](#1)\n- [2 | Annotation Regions](#2)\n > - [2.1 Region Areas per Image](#2.1)\n > - [2.2 Areas of Regions](#2.2)\n- [3 | Visual Control](#3)\n- [4 | Conclusion](#4)\n\n\n\n# 1 | IDs\n
\n
\nIDs is the best mean to identify images and related data.","metadata":{}},{"cell_type":"code","source":"# Import necessary libraries\nimport os\nimport json\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\n\npath_in_base = '/kaggle/input/hubmap-hacking-the-human-vasculature/'","metadata":{"execution":{"iopub.status.busy":"2023-06-09T09:03:10.531077Z","iopub.execute_input":"2023-06-09T09:03:10.531489Z","iopub.status.idle":"2023-06-09T09:03:10.766617Z","shell.execute_reply.started":"2023-06-09T09:03:10.531457Z","shell.execute_reply":"2023-06-09T09:03:10.765483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load annotation regions\nreg_df = pd.DataFrame({'iid':[], 'type':[], 'coord':[]})\n\nwith open(path_in_base + 'polygons.jsonl', 'r') as f:\n for line in f:\n antn = json.loads(line)\n img_id = antn['id']\n img_antn = antn['annotations']\n for ant in img_antn:\n tp = ant['type']\n crd = ant['coordinates']\n reg_df.loc[len(reg_df.index)] = np.array([img_id, tp, crd], dtype = object)\n\nreg_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-06-09T09:03:10.769853Z","iopub.execute_input":"2023-06-09T09:03:10.770596Z","iopub.status.idle":"2023-06-09T09:03:41.863611Z","shell.execute_reply.started":"2023-06-09T09:03:10.770555Z","shell.execute_reply":"2023-06-09T09:03:41.861852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# IDs of training set\ndef get_train_ids():\n ids = set()\n for dirname, _, filenames in os.walk(path_in_base + 'train'):\n for filename in filenames:\n ids.add(filename[:len(filename) - 4])\n return ids\n\n# IDs of test set (actually there is only one image)\ndef get_test_ids():\n ids = set()\n for dirname, _, filenames in os.walk(path_in_base + 'test'):\n for filename in filenames:\n ids.add(filename[:len(filename) - 4])\n return ids\n\n# IDs relating to the annotation regions\ndef get_polygons_ids():\n ids = set(reg_df['iid'])\n return ids\n\n# IDs relating to the metadata\ndef get_meta_ids():\n ids = set()\n df = pd.read_csv(path_in_base + 'tile_meta.csv')\n ids.update(df['id'])\n return ids\n\n# List of \"bad\" IDs discovered visually\n# The list is nominal and is presented only as an example\ndef get_bad_ids():\n ids = set()\n ids.add('0c0fd0a91f64')\n ids.add('4aa434181421')\n ids.add('8c66c89b3e78')\n ids.add('30ad648e36a2')\n ids.add('46d9ee4f8a44')\n ids.add('596a08cbd66b')\n ids.add('22429e2b9c48')\n ids.add('c1da0ec94d0e')\n ids.add('d6fd6d5bf8ee')\n ids.add('eff3a8b0bf65')\n return ids\n\n# Display an image\ndef draw_img(img_id):\n img = Image.open(path_in_base + 'train/' + img_id + '.tif')\n print(img_id)\n display(img)","metadata":{"execution":{"iopub.status.busy":"2023-06-09T09:03:41.865314Z","iopub.execute_input":"2023-06-09T09:03:41.865925Z","iopub.status.idle":"2023-06-09T09:03:41.875344Z","shell.execute_reply.started":"2023-06-09T09:03:41.865889Z","shell.execute_reply":"2023-06-09T09:03:41.874646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# IDs of training set\ntrain_ids = get_train_ids()\n\nprint(len(train_ids))\nprint(list(train_ids)[:10])","metadata":{"execution":{"iopub.status.busy":"2023-06-09T09:03:41.877652Z","iopub.execute_input":"2023-06-09T09:03:41.878288Z","iopub.status.idle":"2023-06-09T09:04:06.670657Z","shell.execute_reply.started":"2023-06-09T09:03:41.878254Z","shell.execute_reply":"2023-06-09T09:04:06.669872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# IDs of test set\ntest_ids = get_test_ids()\n\nprint(len(test_ids))\nprint(list(test_ids)[:10])","metadata":{"execution":{"iopub.status.busy":"2023-06-09T09:04:06.672101Z","iopub.execute_input":"2023-06-09T09:04:06.672504Z","iopub.status.idle":"2023-06-09T09:04:06.681609Z","shell.execute_reply.started":"2023-06-09T09:04:06.672478Z","shell.execute_reply":"2023-06-09T09:04:06.679789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# IDs relating to the annotation regions\npolygons_ids = get_polygons_ids()\n\nprint(len(polygons_ids))\nprint(list(polygons_ids)[:10])","metadata":{"execution":{"iopub.status.busy":"2023-06-09T09:04:06.682884Z","iopub.execute_input":"2023-06-09T09:04:06.683217Z","iopub.status.idle":"2023-06-09T09:04:06.699007Z","shell.execute_reply.started":"2023-06-09T09:04:06.683190Z","shell.execute_reply":"2023-06-09T09:04:06.697034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# IDs relating to the metadata\nmeta_ids = get_meta_ids()\n\nprint(len(meta_ids))\nprint(list(meta_ids)[:10])","metadata":{"execution":{"iopub.status.busy":"2023-06-09T09:04:06.700824Z","iopub.execute_input":"2023-06-09T09:04:06.701285Z","iopub.status.idle":"2023-06-09T09:04:06.733776Z","shell.execute_reply.started":"2023-06-09T09:04:06.701254Z","shell.execute_reply":"2023-06-09T09:04:06.732822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Ok, testing set does not intersect training set\nlen(test_ids.intersection(train_ids))","metadata":{"execution":{"iopub.status.busy":"2023-06-09T09:04:06.735089Z","iopub.execute_input":"2023-06-09T09:04:06.736427Z","iopub.status.idle":"2023-06-09T09:04:06.746370Z","shell.execute_reply.started":"2023-06-09T09:04:06.736367Z","shell.execute_reply":"2023-06-09T09:04:06.744976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# There is no a metadata for the given testing set.\nlen(test_ids.intersection(meta_ids))","metadata":{"execution":{"iopub.status.busy":"2023-06-09T09:04:06.747861Z","iopub.execute_input":"2023-06-09T09:04:06.749439Z","iopub.status.idle":"2023-06-09T09:04:06.759089Z","shell.execute_reply.started":"2023-06-09T09:04:06.749388Z","shell.execute_reply":"2023-06-09T09:04:06.758375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Ok. Test does not presented in polygons.\nlen(test_ids.intersection(polygons_ids))","metadata":{"execution":{"iopub.status.busy":"2023-06-09T09:04:06.762025Z","iopub.execute_input":"2023-06-09T09:04:06.763254Z","iopub.status.idle":"2023-06-09T09:04:06.774702Z","shell.execute_reply.started":"2023-06-09T09:04:06.763195Z","shell.execute_reply":"2023-06-09T09:04:06.773642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Only 1633 images presented in polygons\nlen(train_ids.intersection(polygons_ids))","metadata":{"execution":{"iopub.status.busy":"2023-06-09T09:04:06.775903Z","iopub.execute_input":"2023-06-09T09:04:06.777350Z","iopub.status.idle":"2023-06-09T09:04:06.792293Z","shell.execute_reply.started":"2023-06-09T09:04:06.777309Z","shell.execute_reply":"2023-06-09T09:04:06.790851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Four \"bad\" images are presented in polygons\nlen(get_bad_ids().intersection(polygons_ids))","metadata":{"execution":{"iopub.status.busy":"2023-06-09T09:04:06.793540Z","iopub.execute_input":"2023-06-09T09:04:06.793889Z","iopub.status.idle":"2023-06-09T09:04:06.808548Z","shell.execute_reply.started":"2023-06-09T09:04:06.793863Z","shell.execute_reply":"2023-06-09T09:04:06.807511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lets see \"bad\" images\nbad_ids = get_bad_ids().intersection(polygons_ids)\n\nfor img_id in bad_ids:\n draw_img(img_id)","metadata":{"execution":{"iopub.status.busy":"2023-06-09T09:04:06.810020Z","iopub.execute_input":"2023-06-09T09:04:06.810394Z","iopub.status.idle":"2023-06-09T09:04:07.131869Z","shell.execute_reply.started":"2023-06-09T09:04:06.810364Z","shell.execute_reply":"2023-06-09T09:04:07.130588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# There are regions for \"bad\" regions\n# They should be included into training (or validation) set\n\nfor img_id in bad_ids:\n print(img_id, ':', end=' ')\n print(reg_df[reg_df['iid'] == img_id]['coord'], '\\n')","metadata":{"execution":{"iopub.status.busy":"2023-06-09T09:04:07.134167Z","iopub.execute_input":"2023-06-09T09:04:07.134715Z","iopub.status.idle":"2023-06-09T09:04:07.246176Z","shell.execute_reply.started":"2023-06-09T09:04:07.134656Z","shell.execute_reply":"2023-06-09T09:04:07.244929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# 2 | Annotation Regions\n
\n
\nAnnotation regions are essential in the development of the required AI model.","metadata":{}},{"cell_type":"code","source":"# NaN values (Ok)\nreg_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2023-06-09T09:04:07.248447Z","iopub.execute_input":"2023-06-09T09:04:07.248910Z","iopub.status.idle":"2023-06-09T09:04:07.275073Z","shell.execute_reply.started":"2023-06-09T09:04:07.248872Z","shell.execute_reply":"2023-06-09T09:04:07.272997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Types (Ok)\nreg_df['type'].unique()","metadata":{"execution":{"iopub.status.busy":"2023-06-09T09:04:07.276931Z","iopub.execute_input":"2023-06-09T09:04:07.277324Z","iopub.status.idle":"2023-06-09T09:04:07.299269Z","shell.execute_reply.started":"2023-06-09T09:04:07.277294Z","shell.execute_reply":"2023-06-09T09:04:07.297338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The dataset is very disbalanced by type\nvc = reg_df.type.value_counts()\nvc","metadata":{"execution":{"iopub.status.busy":"2023-06-09T09:04:07.302724Z","iopub.execute_input":"2023-06-09T09:04:07.303311Z","iopub.status.idle":"2023-06-09T09:04:07.322540Z","shell.execute_reply.started":"2023-06-09T09:04:07.303265Z","shell.execute_reply":"2023-06-09T09:04:07.320077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## 2.1 Region Areas per Image","metadata":{}},{"cell_type":"code","source":"# Return mask area (number of pixels in percent of 512 * 512) \ndef get_msk_area(img_id):\n msk = np.zeros((512, 512), dtype = np.uint8)\n mdf = reg_df[reg_df.iid == img_id]\n for index, row in mdf.iterrows():\n crd = np.array(row['coord'])\n crd = crd.reshape(-1, 1, 2)\n cv2.fillPoly(msk, [crd], 1)\n ar_img = np.asarray(msk)\n return ar_img.sum() * (100.0 / (512 * 512))\n\n# Calculate area for each region\n\nreg_areas_df = pd.DataFrame({'iid':[], 'area':[]})\n\nfor iid in polygons_ids:\n ar = get_msk_area(iid)\n reg_areas_df.loc[len(reg_areas_df.index)] = np.array([iid, ar])\n\nreg_areas_df['area'] = reg_areas_df['area'].astype(float)\n\nreg_areas_df","metadata":{"execution":{"iopub.status.busy":"2023-06-09T09:04:07.326253Z","iopub.execute_input":"2023-06-09T09:04:07.326756Z","iopub.status.idle":"2023-06-09T09:04:15.343990Z","shell.execute_reply.started":"2023-06-09T09:04:07.326717Z","shell.execute_reply":"2023-06-09T09:04:15.342311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reg_areas_df.dtypes\nreg_areas_df['area'].describe()","metadata":{"execution":{"iopub.status.busy":"2023-06-09T09:04:15.345973Z","iopub.execute_input":"2023-06-09T09:04:15.347328Z","iopub.status.idle":"2023-06-09T09:04:15.364970Z","shell.execute_reply.started":"2023-06-09T09:04:15.347245Z","shell.execute_reply":"2023-06-09T09:04:15.363209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## 2.2 Areas of Regions","metadata":{}},{"cell_type":"code","source":"# Calculate area for each region\n\nreg_areas_reg_df = pd.DataFrame({'area':[]})\n\nfor index, row in reg_df.iterrows():\n msk = np.zeros((512, 512), dtype = np.uint8)\n crd = np.array(row['coord'])\n crd = crd.reshape(-1, 1, 2)\n # Draw the polygon on the mask\n cv2.fillPoly(msk, [crd], 1)\n ar_img = np.asarray(msk)\n reg_areas_reg_df.loc[len(reg_areas_reg_df.index)] = np.array([ar_img.sum() * (100.0 / (512 * 512))])\n\nreg_areas_reg_df['area'] = reg_areas_reg_df['area'].astype(float)\n\nreg_areas_reg_df","metadata":{"execution":{"iopub.status.busy":"2023-06-09T09:04:15.367133Z","iopub.execute_input":"2023-06-09T09:04:15.367514Z","iopub.status.idle":"2023-06-09T09:04:37.165759Z","shell.execute_reply.started":"2023-06-09T09:04:15.367482Z","shell.execute_reply":"2023-06-09T09:04:37.164256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reg_areas_df.dtypes\nreg_areas_reg_df['area'].describe()","metadata":{"execution":{"iopub.status.busy":"2023-06-09T09:04:37.166938Z","iopub.execute_input":"2023-06-09T09:04:37.167325Z","iopub.status.idle":"2023-06-09T09:04:37.181280Z","shell.execute_reply.started":"2023-06-09T09:04:37.167296Z","shell.execute_reply":"2023-06-09T09:04:37.179658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# 3 | Visual Control\n
\n
\n\nVisual control of training TIFF images shows that picture of some of them differ from others.","metadata":{}},{"cell_type":"code","source":"# Display the divergent images\nbad_ids = get_bad_ids()\n\nfor img_id in bad_ids:\n draw_img(img_id)","metadata":{"execution":{"iopub.status.busy":"2023-06-09T09:04:37.183669Z","iopub.execute_input":"2023-06-09T09:04:37.184427Z","iopub.status.idle":"2023-06-09T09:04:37.764651Z","shell.execute_reply.started":"2023-06-09T09:04:37.184388Z","shell.execute_reply":"2023-06-09T09:04:37.763056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# 4 | Conclusion\n
\n\n\n1. Training list should be created according to the list of IDs from 'polygons.jsonl' file;\n2. Input data are imbalanced;\n3. Do the divergent images seem contributable to the model?","metadata":{}}]}