{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Overview\n\n**This notebook has a few uses:**\n1. EDA - you can use the classes in this notebook to help you expand your EDA\n2. Features - this notebook creates a few features such as area (µm) and compute_polsby_popper (measures the compactness of an organ on a scale of 0 to 1 where 1 is the most compactly shaped) - these features could potentially be used to improve a ML model\n3. Judges Prize - one of the questions to answer for the judges prize is \"Is it possible to predict FTU area size distribution, given age and sex info across all organs?\" The dataset generated at the end of this notebook can help you get started on this task\n\n**Methodology**\n- I have a config class and utils file for basics\n- The FTU class represents any one functional tissue unit as a shapely Polygon. It calculates features such as the area, perimeter, compactness, etc for each FTU.\n- The organ class represents any organ sample. The organ consists of a list of FTUs. It makes it easy to analyze any organ, perform EDA, and extract features.\n- I show an example of one of each type of organ\n- Create an Organ object for each sample. Use that to create a pandas dataframe with each feature for every organ","metadata":{}},{"cell_type":"markdown","source":"# CFG","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport json\n\nimport tifffile\nimport cv2\n\nfrom shapely.geometry import Polygon\nfrom PIL import Image, ImageDraw\n\nfrom typing import List, Dict, Any\n\nimport matplotlib.pyplot as plt\nimport plotly.express as px","metadata":{"execution":{"iopub.status.busy":"2022-08-26T23:55:18.878530Z","iopub.execute_input":"2022-08-26T23:55:18.879067Z","iopub.status.idle":"2022-08-26T23:55:18.888082Z","shell.execute_reply.started":"2022-08-26T23:55:18.878946Z","shell.execute_reply":"2022-08-26T23:55:18.887216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    img_dir = \"../input/hubmap-organ-segmentation/train_images\"\n    ann_dir = \"../input/hubmap-organ-segmentation/train_annotations\"\n    meta_path = \"../input/hubmap-organ-segmentation/train.csv\"\n    organs = ['kidney', 'prostate', 'largeintestine', 'spleen', 'lung']\n    \nmeta = pd.read_csv(CFG.meta_path)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-26T22:50:38.927129Z","iopub.execute_input":"2022-08-26T22:50:38.927602Z","iopub.status.idle":"2022-08-26T22:50:39.266540Z","shell.execute_reply.started":"2022-08-26T22:50:38.927541Z","shell.execute_reply":"2022-08-26T22:50:39.265149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils","metadata":{}},{"cell_type":"code","source":"#Modified from https://www.kaggle.com/code/abhinand05/hubmap-extensive-eda-what-are-we-hacking\ndef read_tiff(path, scale=None, verbose=0): \n    image = tifffile.imread(path)\n    if len(image.shape) == 5:\n        image = image.squeeze().transpose(1, 2, 0)\n    \n    if verbose:\n        print(f\"[{path}] Image shape: {image.shape}\")\n    \n    if scale:\n        new_size = (image.shape[1] // scale, image.shape[0] // scale)\n        image = cv2.resize(image, new_size)\n        \n        if verbose:\n            print(f\"[{path}] Resized Image shape: {image.shape}\")\n        \n    mx = np.max(image)\n    image = image.astype(np.float32)\n    if mx:\n        image /= mx # scale image to [0, 1]\n    return image\n\n\n# https://www.kaggle.com/paulorzp/rle-functions-run-length-encode-decode\ndef mask2rle(img): # encoder\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels= img.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n \ndef rle2mask(mask_rle, shape=(1600,256)): # decoder\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (width,height) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T","metadata":{"execution":{"iopub.status.busy":"2022-08-26T22:50:39.268506Z","iopub.execute_input":"2022-08-26T22:50:39.269094Z","iopub.status.idle":"2022-08-26T22:50:39.284684Z","shell.execute_reply.started":"2022-08-26T22:50:39.269045Z","shell.execute_reply":"2022-08-26T22:50:39.283309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# FTU","metadata":{}},{"cell_type":"code","source":"class FTU:\n    def __init__(self, poly: Polygon, px_size: float) -> None:\n        self.poly = poly\n        self.px_size = px_size\n\n        self.area_units = self.poly.area\n        self.perimeter_units = self.poly.length\n        self.centroid = self.poly.centroid\n\n        self.area = self._compute_area_micrometers()\n        self.polsby_popper = self._compute_polsby_popper()\n\n    def _compute_area_micrometers(self) -> float:\n        area_per_unit = self.px_size**2\n        return float(self.area_units * area_per_unit)\n\n    def _compute_polsby_popper(self) -> float:\n        num = 4 * np.pi * self.area_units\n        den = self.perimeter_units**2\n        return num / den","metadata":{"execution":{"iopub.status.busy":"2022-08-26T22:50:39.287673Z","iopub.execute_input":"2022-08-26T22:50:39.288778Z","iopub.status.idle":"2022-08-26T22:50:39.301384Z","shell.execute_reply.started":"2022-08-26T22:50:39.288718Z","shell.execute_reply":"2022-08-26T22:50:39.300107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Organ","metadata":{}},{"cell_type":"code","source":"class Organ:\n    def __init__(self, img_id: int) -> None:\n        # vars\n        self.img_id = img_id\n        self.polys: List[FTU] = list()\n        self.sample = meta[meta.id == img_id]\n\n        self.organ_type = self.sample.organ.values[0]\n        self.pixel_size = self.sample.pixel_size.values[0]\n        self.age = self.sample.age.values[0]\n        self.sex = self.sample.sex.values[0]\n        self.data_source = self.sample.data_source.values[0]\n\n        self.img = read_tiff(f\"{CFG.img_dir}/{img_id}.tiff\")\n        self.img_pil = Image.open(f\"{CFG.img_dir}/{img_id}.tiff\")\n        self.mask = rle2mask(self.sample.rle.values[0], (self.img.shape[1], self.img.shape[0]))\n        \n        # fill polys with FTUs\n        f = open(f\"{CFG.ann_dir}/{img_id}.json\")\n        poly_json = json.load(f)\n        f.close()\n        for p in poly_json:\n            poly = Polygon(p)\n            self.polys.append(FTU(poly, self.pixel_size))\n\n    def __len__(self) -> int:\n        return len(self.polys)\n\n    def print_meta(self):\n        print(f\"organ_type: {self.organ_type}\")\n        print(f\"pixel_size: {self.pixel_size}\")\n        print(f\"age: {self.age}\")\n        print(f\"sex: {self.sex}\")\n        print(f\"data_source: {self.data_source}\")\n        \n    def compare_poly_to_true_area(self) -> float:\n        true_area = np.count_nonzero(self.mask)\n\n        poly_area = 0\n        for poly in self.polys:\n            poly_area += poly.area_units\n\n        return true_area - poly_area\n\n    def get_area_info(self) -> Dict[str, Any]:\n        areas = [poly.area for poly in self.polys]\n        avg_area = np.mean(areas)\n        return {\n            \"areas\": areas,\n            \"avg_area\" : avg_area\n        }\n\n    def get_polsby_popper_info(self) -> Dict[str, Any]:\n        pp = [poly.polsby_popper for poly in self.polys]\n        avg_pp = np.mean(pp)\n        return {\n            \"polsby_poppers\" : pp,\n            \"avg_polsby_popper\" : avg_pp\n        }\n\n    def display(self, info: str):\n        plt.figure(figsize=(20,10))\n        plt.imshow(self.img)\n\n        for ftu in self.polys:\n            x, y = ftu.poly.exterior.xy \n            plt.plot(x, y, c=\"blue\")\n            plt.fill(x, y, c=\"white\") \n            if info == \"area\":\n                plt.text(ftu.centroid.x, ftu.centroid.y, round(ftu.area), fontsize=10, horizontalalignment=\"center\")\n            elif info == \"polsby_popper\":\n                plt.text(ftu.centroid.x, ftu.centroid.y, round(ftu.polsby_popper, 3), fontsize=10, horizontalalignment=\"center\")\n\n        if info == \"area\":\n            plt.title(f\"Organ {self.img_id} - Area - Size\")\n        elif info == \"polsby_popper\":\n            plt.title(f\"Organ {self.img_id} - Polsby Popper - Compactness\")\n            \n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-26T23:04:36.586042Z","iopub.execute_input":"2022-08-26T23:04:36.586535Z","iopub.status.idle":"2022-08-26T23:04:36.607854Z","shell.execute_reply.started":"2022-08-26T23:04:36.586495Z","shell.execute_reply":"2022-08-26T23:04:36.606482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prostate","metadata":{}},{"cell_type":"code","source":"o = Organ(10274)\nprint(f\"num FTUs: {len(o)}\")\no.print_meta()\no.display(\"polsby_popper\")\no.display(\"area\")","metadata":{"execution":{"iopub.status.busy":"2022-08-26T23:05:16.757086Z","iopub.execute_input":"2022-08-26T23:05:16.757532Z","iopub.status.idle":"2022-08-26T23:05:21.925413Z","shell.execute_reply.started":"2022-08-26T23:05:16.757501Z","shell.execute_reply":"2022-08-26T23:05:21.923176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Spleen","metadata":{}},{"cell_type":"code","source":"o = Organ(10392)\nprint(f\"num FTUs: {len(o)}\")\no.print_meta()\no.display(\"polsby_popper\")\no.display(\"area\")","metadata":{"execution":{"iopub.status.busy":"2022-08-26T23:06:18.455359Z","iopub.execute_input":"2022-08-26T23:06:18.456783Z","iopub.status.idle":"2022-08-26T23:06:24.418142Z","shell.execute_reply.started":"2022-08-26T23:06:18.456725Z","shell.execute_reply":"2022-08-26T23:06:24.416895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Kidney","metadata":{}},{"cell_type":"code","source":"o = Organ(10611)\nprint(f\"num FTUs: {len(o)}\")\no.print_meta()\no.display(\"polsby_popper\")\no.display(\"area\")","metadata":{"execution":{"iopub.status.busy":"2022-08-26T23:06:55.937988Z","iopub.execute_input":"2022-08-26T23:06:55.938385Z","iopub.status.idle":"2022-08-26T23:07:01.566214Z","shell.execute_reply.started":"2022-08-26T23:06:55.938354Z","shell.execute_reply":"2022-08-26T23:07:01.564921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Lung","metadata":{}},{"cell_type":"code","source":"o = Organ(10488)\nprint(f\"num FTUs: {len(o)}\")\no.print_meta()\no.display(\"polsby_popper\")\no.display(\"area\")","metadata":{"execution":{"iopub.status.busy":"2022-08-26T23:07:58.772258Z","iopub.execute_input":"2022-08-26T23:07:58.772766Z","iopub.status.idle":"2022-08-26T23:08:04.575581Z","shell.execute_reply.started":"2022-08-26T23:07:58.772729Z","shell.execute_reply":"2022-08-26T23:08:04.574275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Large Intestine","metadata":{}},{"cell_type":"code","source":"o = Organ(10651)\nprint(f\"num FTUs: {len(o)}\")\no.print_meta()\no.display(\"polsby_popper\")\no.display(\"area\")","metadata":{"execution":{"iopub.status.busy":"2022-08-26T23:08:50.693561Z","iopub.execute_input":"2022-08-26T23:08:50.694064Z","iopub.status.idle":"2022-08-26T23:08:56.060759Z","shell.execute_reply.started":"2022-08-26T23:08:50.694013Z","shell.execute_reply":"2022-08-26T23:08:56.059798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate Dataset","metadata":{}},{"cell_type":"code","source":"def generate_dataset():\n    ds = list()\n\n    for img_id in meta.id:\n        organ = Organ(img_id)\n        for ftu in organ.polys:\n            ds.append([img_id, organ.organ_type, organ.age, organ.sex, ftu.area, ftu.polsby_popper])\n\n    return pd.DataFrame(ds, columns=[\"id\", \"organ\", \"age\", \"sex\", \"area\", \"polsby_popper\"])","metadata":{"execution":{"iopub.status.busy":"2022-08-26T23:11:27.711452Z","iopub.execute_input":"2022-08-26T23:11:27.711945Z","iopub.status.idle":"2022-08-26T23:11:27.719035Z","shell.execute_reply.started":"2022-08-26T23:11:27.711912Z","shell.execute_reply":"2022-08-26T23:11:27.717943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = generate_dataset()","metadata":{"execution":{"iopub.status.busy":"2022-08-26T23:11:38.950498Z","iopub.execute_input":"2022-08-26T23:11:38.951028Z","iopub.status.idle":"2022-08-26T23:13:36.447491Z","shell.execute_reply.started":"2022-08-26T23:11:38.950985Z","shell.execute_reply":"2022-08-26T23:13:36.446493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-26T23:13:48.574781Z","iopub.execute_input":"2022-08-26T23:13:48.575178Z","iopub.status.idle":"2022-08-26T23:13:48.592406Z","shell.execute_reply.started":"2022-08-26T23:13:48.575147Z","shell.execute_reply":"2022-08-26T23:13:48.591609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_distrubutions(df):\n    age = df.groupby(\"age\")[[\"area\", \"polsby_popper\"]].mean().reset_index()\n    fig = px.bar(age.age, age.area)\n    fig.update_xaxes(title=\"Mean Area (µm)\")\n    fig.update_yaxes(title=\"Age\")\n    fig.show()\n    \n    fig = px.bar(age.age, age.polsby_popper)\n    fig.update_xaxes(title=\"Mean Polsby Popper (Compactness)\")\n    fig.update_yaxes(title=\"Age\")\n    fig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-27T00:02:33.788933Z","iopub.execute_input":"2022-08-27T00:02:33.789358Z","iopub.status.idle":"2022-08-27T00:02:33.796293Z","shell.execute_reply.started":"2022-08-27T00:02:33.789322Z","shell.execute_reply":"2022-08-27T00:02:33.794914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_distrubutions(df)","metadata":{"execution":{"iopub.status.busy":"2022-08-27T00:02:34.303220Z","iopub.execute_input":"2022-08-27T00:02:34.303658Z","iopub.status.idle":"2022-08-27T00:02:34.412325Z","shell.execute_reply.started":"2022-08-27T00:02:34.303621Z","shell.execute_reply":"2022-08-27T00:02:34.411615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}