{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# HuBMAP - Hacking the Human Vasculature | Exploratory Data Analysis (EDA)","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport random\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport cv2\nimport tifffile\n\nimport plotly.express as px\nimport plotly.graph_objects as go\n\nimport json\n\nfrom shapely.geometry import Polygon\nimport matplotlib.patches as patches","metadata":{"execution":{"iopub.status.busy":"2023-07-02T18:15:20.295917Z","iopub.execute_input":"2023-07-02T18:15:20.296320Z","iopub.status.idle":"2023-07-02T18:15:22.478418Z","shell.execute_reply.started":"2023-07-02T18:15:20.296288Z","shell.execute_reply":"2023-07-02T18:15:22.477297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_PATH = \"../input/hubmap-hacking-the-human-vasculature\"\nTRAIN_PATH = os.path.join(BASE_PATH, \"train\")\nTEST_PATH = os.path.join(BASE_PATH, \"test\")\n\nprint(os.listdir(BASE_PATH))","metadata":{"execution":{"iopub.status.busy":"2023-07-02T18:15:22.480758Z","iopub.execute_input":"2023-07-02T18:15:22.481202Z","iopub.status.idle":"2023-07-02T18:15:22.488101Z","shell.execute_reply.started":"2023-07-02T18:15:22.481163Z","shell.execute_reply":"2023-07-02T18:15:22.486998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Tile Meta Data\n\nMetadata for each image. The hidden version of this file also contains metadata for the test set tiles.\n* **source_wsi:** Identifies the WSI this tile was extracted from.\n* **{i|j}:** The location of the upper-left corner within the WSI where the tile was extracted.\n* **dataset:** The dataset this tile belongs to, as described above.","metadata":{}},{"cell_type":"code","source":"tile_meta_df = pd.read_csv(\n    os.path.join(BASE_PATH, \"tile_meta.csv\")\n)\nprint(f\"DF SHAPE: {tile_meta_df.shape}\")\ntile_meta_df","metadata":{"execution":{"iopub.status.busy":"2023-07-02T18:15:22.489847Z","iopub.execute_input":"2023-07-02T18:15:22.490925Z","iopub.status.idle":"2023-07-02T18:15:22.563182Z","shell.execute_reply.started":"2023-07-02T18:15:22.490892Z","shell.execute_reply":"2023-07-02T18:15:22.562276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x=tile_meta_df[\"dataset\"])","metadata":{"execution":{"iopub.status.busy":"2023-07-02T18:15:22.566236Z","iopub.execute_input":"2023-07-02T18:15:22.566891Z","iopub.status.idle":"2023-07-02T18:15:22.856396Z","shell.execute_reply.started":"2023-07-02T18:15:22.566858Z","shell.execute_reply":"2023-07-02T18:15:22.854320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## WSI Meta Data","metadata":{}},{"cell_type":"markdown","source":"Metadata for the Whole Slide Images the tiles were extracted from.\n* **source_wsi:** Identifies the WSI.\n* age, sex, race, height, weight, and bmi demographic information about the tissue donor.","metadata":{}},{"cell_type":"code","source":"wsi_meta_df = pd.read_csv(\n    os.path.join(BASE_PATH, \"wsi_meta.csv\")\n)\nprint(f\"DF SHAPE: {wsi_meta_df.shape}\")\nwsi_meta_df","metadata":{"execution":{"iopub.status.busy":"2023-07-02T18:15:22.858137Z","iopub.execute_input":"2023-07-02T18:15:22.858599Z","iopub.status.idle":"2023-07-02T18:15:22.885623Z","shell.execute_reply.started":"2023-07-02T18:15:22.858567Z","shell.execute_reply":"2023-07-02T18:15:22.884186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.sunburst(\n    wsi_meta_df,\n    path=['age', 'sex', 'race', 'height', 'weight', 'bmi'],\n)\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-02T18:15:22.887762Z","iopub.execute_input":"2023-07-02T18:15:22.888485Z","iopub.status.idle":"2023-07-02T18:15:25.335516Z","shell.execute_reply.started":"2023-07-02T18:15:22.888411Z","shell.execute_reply":"2023-07-02T18:15:25.334257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Image Files and Polygons","metadata":{}},{"cell_type":"code","source":"train_files = os.listdir(TRAIN_PATH)\ntest_files = os.listdir(TEST_PATH)\n\ntrain_sz = len(train_files)\ntest_sz = len(test_files)\n\nprint(f\"Train Images: {train_sz}\")\nprint(f\"Test Images: {test_sz}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-02T18:27:15.895972Z","iopub.execute_input":"2023-07-02T18:27:15.896519Z","iopub.status.idle":"2023-07-02T18:27:15.909561Z","shell.execute_reply.started":"2023-07-02T18:27:15.896483Z","shell.execute_reply":"2023-07-02T18:27:15.908436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"POLYGON_PATH = os.path.join(BASE_PATH, \"polygons.jsonl\")\n\nwith open(POLYGON_PATH, 'r') as json_file:\n    json_list = list(json_file)\n\nfor json_str in json_list:\n    result = json.loads(json_str)\n    print(f\"result: {result}\")\n    print(isinstance(result, dict))\n    break","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-07-02T18:15:25.605633Z","iopub.execute_input":"2023-07-02T18:15:25.606034Z","iopub.status.idle":"2023-07-02T18:15:26.023186Z","shell.execute_reply.started":"2023-07-02T18:15:25.606004Z","shell.execute_reply":"2023-07-02T18:15:26.022348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Polygon Info: \", result.keys())\nprint(\"Polygon Annotations Info: \", result[\"annotations\"][0].keys())","metadata":{"execution":{"iopub.status.busy":"2023-07-02T18:15:26.024485Z","iopub.execute_input":"2023-07-02T18:15:26.025180Z","iopub.status.idle":"2023-07-02T18:15:26.030938Z","shell.execute_reply.started":"2023-07-02T18:15:26.025149Z","shell.execute_reply":"2023-07-02T18:15:26.029864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check Out Images and Annotation Masks","metadata":{}},{"cell_type":"markdown","source":"### JsonList","metadata":{}},{"cell_type":"code","source":"json_list[0]","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-07-02T18:15:26.035079Z","iopub.execute_input":"2023-07-02T18:15:26.036445Z","iopub.status.idle":"2023-07-02T18:15:26.071056Z","shell.execute_reply.started":"2023-07-02T18:15:26.036409Z","shell.execute_reply":"2023-07-02T18:15:26.070223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# JSONL TO DF\n\ndfs = []\n\n# Convert each JSON string to a DataFrame\nfor json_string in json_list:\n    data = json.loads(json_string)\n    df = pd.json_normalize(data, 'annotations', ['id'])\n    dfs.append(df)\n\n# Concatenate all the dataframes into a single dataframe\nresult_df = pd.concat(dfs, ignore_index=True)\n\nprint(result_df)","metadata":{"execution":{"iopub.status.busy":"2023-07-02T18:15:26.072256Z","iopub.execute_input":"2023-07-02T18:15:26.073476Z","iopub.status.idle":"2023-07-02T18:15:49.557745Z","shell.execute_reply.started":"2023-07-02T18:15:26.073444Z","shell.execute_reply":"2023-07-02T18:15:49.556581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_df.to_csv(\"/kaggle/working/polygons.csv\")\nresult_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-02T18:15:49.559461Z","iopub.execute_input":"2023-07-02T18:15:49.560223Z","iopub.status.idle":"2023-07-02T18:15:52.716085Z","shell.execute_reply.started":"2023-07-02T18:15:49.560192Z","shell.execute_reply":"2023-07-02T18:15:52.714761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x=result_df[\"type\"])","metadata":{"execution":{"iopub.status.busy":"2023-07-02T18:15:52.717558Z","iopub.execute_input":"2023-07-02T18:15:52.717911Z","iopub.status.idle":"2023-07-02T18:15:53.018978Z","shell.execute_reply.started":"2023-07-02T18:15:52.717882Z","shell.execute_reply":"2023-07-02T18:15:53.017802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x=result_df[\"id\"])","metadata":{"execution":{"iopub.status.busy":"2023-07-02T18:15:53.020182Z","iopub.execute_input":"2023-07-02T18:15:53.020478Z","iopub.status.idle":"2023-07-02T18:16:11.410409Z","shell.execute_reply.started":"2023-07-02T18:15:53.020452Z","shell.execute_reply":"2023-07-02T18:16:11.409029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Images and Masks","metadata":{}},{"cell_type":"code","source":"ids = random.sample(list(result_df[\"id\"]), 16)\npaths = [os.path.join(TRAIN_PATH, ids[i])+\".tif\" for i in range(len(ids))]\n\nplt.figure(figsize=(16, 16))\ni=0\nfor ind, tmp_image in zip(ids, paths):\n    plt.subplot(4, 4, i + 1)\n    plt.imshow(plt.imread(tmp_image))\n    plt.title(ind)\n    plt.axis(\"off\")\n    i+=1","metadata":{"execution":{"iopub.status.busy":"2023-07-02T18:16:11.412187Z","iopub.execute_input":"2023-07-02T18:16:11.412612Z","iopub.status.idle":"2023-07-02T18:16:15.119366Z","shell.execute_reply.started":"2023-07-02T18:16:11.412580Z","shell.execute_reply":"2023-07-02T18:16:15.118181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_result = result_df.groupby(by=\"id\")[\"coordinates\"].agg(lambda x: list(x)).reset_index()\ngrouped_result.to_csv(\"/kaggle/working/grouped_polygons.csv\")\ngrouped_result.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-02T18:21:30.050580Z","iopub.execute_input":"2023-07-02T18:21:30.052082Z","iopub.status.idle":"2023-07-02T18:21:33.085536Z","shell.execute_reply.started":"2023-07-02T18:21:30.052028Z","shell.execute_reply":"2023-07-02T18:21:33.084256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_result[\"coordinates\"] = grouped_result[\"coordinates\"].apply(lambda x: x[0])\ngrouped_result.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-02T18:21:33.088540Z","iopub.execute_input":"2023-07-02T18:21:33.089029Z","iopub.status.idle":"2023-07-02T18:21:33.288809Z","shell.execute_reply.started":"2023-07-02T18:21:33.088985Z","shell.execute_reply":"2023-07-02T18:21:33.287635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize Polygons","metadata":{}},{"cell_type":"code","source":"def show_image_and_polygon(idx):\n    total_area = 0\n    \n    img_path = os.path.join(TRAIN_PATH, grouped_result[\"id\"][idx]+\".tif\")\n    img = plt.imread(img_path)\n    \n    fig, ax = plt.subplots()\n    ax.imshow(img)\n\n    # iterate coordinates and plot the polygons\n    for polygon in grouped_result[\"coordinates\"][idx]:\n        x_coords = [point[0] for point in polygon]\n        y_coords = [point[1] for point in polygon]\n\n        patch = patches.Polygon(polygon, alpha=0.5)\n\n        ax.add_patch(patch)\n        \n        polygon_area = Polygon(polygon).area\n        total_area += polygon_area\n\n        # Display the area as text on the plot\n        ax.text(x_coords[0], y_coords[0], f\"Area: {polygon_area:.2f}\", color='red')\n        \n    ax.set_aspect('equal')\n    ax.autoscale()\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-02T18:45:14.947952Z","iopub.execute_input":"2023-07-02T18:45:14.948336Z","iopub.status.idle":"2023-07-02T18:45:14.957862Z","shell.execute_reply.started":"2023-07-02T18:45:14.948306Z","shell.execute_reply":"2023-07-02T18:45:14.956637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_image_and_polygon(25)","metadata":{"execution":{"iopub.status.busy":"2023-07-02T18:45:16.037572Z","iopub.execute_input":"2023-07-02T18:45:16.037976Z","iopub.status.idle":"2023-07-02T18:45:16.420075Z","shell.execute_reply.started":"2023-07-02T18:45:16.037945Z","shell.execute_reply":"2023-07-02T18:45:16.419174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_integers = [random.randint(1, len(grouped_result)) for _ in range(16)]\n\ni=0\nfor idx in random_integers:\n    show_image_and_polygon(idx)\n    i+=1","metadata":{"execution":{"iopub.status.busy":"2023-07-02T18:45:27.009992Z","iopub.execute_input":"2023-07-02T18:45:27.010698Z","iopub.status.idle":"2023-07-02T18:45:33.430899Z","shell.execute_reply.started":"2023-07-02T18:45:27.010663Z","shell.execute_reply":"2023-07-02T18:45:33.429536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}