{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Imports","metadata":{}},{"cell_type":"code","source":"%%time\n%autosave 60\n%matplotlib inline\n%config InlineBackend.figure_format = 'retina'\n\nimport gc\ngc.enable();\nimport glob\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.auto import tqdm\n\nimport plotly\nfrom plotly import tools\nimport plotly.express as px\nimport plotly.offline as pyo\nimport plotly.io as pio\nimport plotly.graph_objects as go\npio.templates.default = 'plotly_white'\n\nimport cv2\nimport tifffile\n\nimport warnings\nwarnings.simplefilter(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-22T20:35:56.016125Z","iopub.execute_input":"2022-06-22T20:35:56.016599Z","iopub.status.idle":"2022-06-22T20:35:56.066823Z","shell.execute_reply.started":"2022-06-22T20:35:56.016561Z","shell.execute_reply":"2022-06-22T20:35:56.064607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Paths","metadata":{}},{"cell_type":"code","source":"ROOT = Path('../input/hubmap-organ-segmentation/')\nTRAIN_CSV_PATH = ROOT / \"train.csv\"\nTRAIN_IMAGES_PATH = ROOT / \"train_images/\"\nTRAIN_ANNOTATIONS_PATH = ROOT / \"train_annotations/\"\n\n# print([x for x in list(ROOT.iterdir())])\n\ndataframe = pd.read_csv(TRAIN_CSV_PATH, low_memory=False, squeeze=True)\n\n# assert len(list(TRAIN_IMAGES_PATH.glob(\"*\"))) == dataframe.shape[0]\n\ndataframe.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-22T19:56:43.293177Z","iopub.execute_input":"2022-06-22T19:56:43.293602Z","iopub.status.idle":"2022-06-22T19:56:43.496645Z","shell.execute_reply.started":"2022-06-22T19:56:43.293569Z","shell.execute_reply":"2022-06-22T19:56:43.495073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Utils","metadata":{}},{"cell_type":"code","source":"def rle2mask(mask_rle, shape):\n    s = mask_rle.split()\n    starts, lengths = [\n        np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])\n    ]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo : hi] = 1\n    return img.reshape(shape).T\n\n\ndef read_image(image_id):\n    image = tifffile.imread(str(TRAIN_IMAGES_PATH / f\"{image_id}.tiff\"))\n    \n    if len(image.shape) == 5:\n        image = image.squeeze().transpose(1, 2, 0)\n    \n    mask = rle2mask(\n        dataframe[dataframe[\"id\"] == image_id][\"rle\"].values[0], \n        (image.shape[1], image.shape[0])\n    )\n        \n    return image, mask\n\n\ndef plot_image_and_mask(image, mask, image_id):\n    plt.figure(figsize=(16, 10))\n    \n    plt.subplot(1, 3, 1)\n    plt.imshow(image)\n    plt.title(f\"Image {image_id}\", fontsize=18)\n    \n    plt.subplot(1, 3, 2)\n    plt.imshow(image)\n    plt.imshow(mask, cmap=\"hot\", alpha=0.5)\n    plt.title(f\"Image {image_id} + mask\", fontsize=18)    \n    \n    plt.subplot(1, 3, 3)\n    plt.imshow(mask, cmap=\"hot\")\n    plt.title(f\"Mask\", fontsize=18)    \n    \n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-22T20:04:45.212822Z","iopub.execute_input":"2022-06-22T20:04:45.213443Z","iopub.status.idle":"2022-06-22T20:04:45.238603Z","shell.execute_reply.started":"2022-06-22T20:04:45.213388Z","shell.execute_reply":"2022-06-22T20:04:45.236415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load images and masks","metadata":{}},{"cell_type":"code","source":"image_ids, images, masks = [[] for _ in range(3)]\n\nfor item in tqdm(range(9), total=9, position=0, desc=\"Reading image and mask\"):\n    idx = np.random.randint(0, high=len(dataframe), size=(1,))[0]\n    image_id = dataframe.iloc[idx].id\n    image, mask = read_image(image_id)\n    \n    images.append(image)\n    masks.append(mask)\n    image_ids.append(image_id)","metadata":{"execution":{"iopub.status.busy":"2022-06-22T20:08:54.675260Z","iopub.execute_input":"2022-06-22T20:08:54.675741Z","iopub.status.idle":"2022-06-22T20:08:57.348860Z","shell.execute_reply.started":"2022-06-22T20:08:54.675704Z","shell.execute_reply":"2022-06-22T20:08:57.347063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Plot images","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(16, 16))\nfor ind, (image_id, image) in enumerate(zip(image_ids, images)):\n    plt.subplot(3, 3, ind + 1)\n    plt.imshow(image)\n    plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2022-06-22T20:08:57.350606Z","iopub.execute_input":"2022-06-22T20:08:57.351058Z","iopub.status.idle":"2022-06-22T20:09:07.910030Z","shell.execute_reply.started":"2022-06-22T20:08:57.351024Z","shell.execute_reply":"2022-06-22T20:09:07.909004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Plot images with masks","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(16, 16))\nfor ind, (image_id, image, mask) in enumerate(zip(image_ids, images, masks)):\n    plt.subplot(3, 3, ind + 1)\n    plt.imshow(image)\n    plt.imshow(mask, cmap=\"hot\", alpha=0.5)\n    plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2022-06-22T20:17:44.720005Z","iopub.execute_input":"2022-06-22T20:17:44.720413Z","iopub.status.idle":"2022-06-22T20:18:04.555195Z","shell.execute_reply.started":"2022-06-22T20:17:44.720382Z","shell.execute_reply":"2022-06-22T20:18:04.553761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Single Sample Analysis","metadata":{}},{"cell_type":"markdown","source":"#### Sample 1","metadata":{}},{"cell_type":"code","source":"plot_image_and_mask(images[0], masks[0], image_ids[0])","metadata":{"execution":{"iopub.status.busy":"2022-06-22T20:19:28.261918Z","iopub.execute_input":"2022-06-22T20:19:28.263116Z","iopub.status.idle":"2022-06-22T20:19:33.117902Z","shell.execute_reply.started":"2022-06-22T20:19:28.263069Z","shell.execute_reply":"2022-06-22T20:19:33.116628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Sample 2","metadata":{}},{"cell_type":"code","source":"plot_image_and_mask(images[1], masks[1], image_ids[2])","metadata":{"execution":{"iopub.status.busy":"2022-06-22T20:19:45.093335Z","iopub.execute_input":"2022-06-22T20:19:45.094287Z","iopub.status.idle":"2022-06-22T20:19:49.968223Z","shell.execute_reply.started":"2022-06-22T20:19:45.094243Z","shell.execute_reply":"2022-06-22T20:19:49.966936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Metadata Analysis","metadata":{}},{"cell_type":"code","source":"def bar_hor(df, col, title, color, w=None, h=None, lm=0, limit=100, return_trace=False, rev=False, xlb = False):\n    cnt_srs = df[col].value_counts()\n    yy = cnt_srs.head(limit).index[::-1] \n    xx = cnt_srs.head(limit).values[::-1] \n    if rev:\n        yy = cnt_srs.tail(limit).index[::-1] \n        xx = cnt_srs.tail(limit).values[::-1] \n    if xlb:\n        trace = go.Bar(y=xlb, x=xx, orientation = 'h', marker=dict(color=color))\n    else:\n        trace = go.Bar(y=yy, x=xx, orientation = 'h', marker=dict(color=color))\n    if return_trace:\n        return trace \n    layout = dict(title=title, margin=dict(l=lm), width=w, height=h)\n    data = [trace]\n    fig = go.Figure(data=data, layout=layout)\n    fig.show()\n\ndef bar_hor_noagg(x, y, title, color, w=None, h=None, lm=0, limit=100, rt=False):\n    trace = go.Bar(y=x, x=y, orientation = 'h', marker=dict(color=color))\n    if rt:\n        return trace\n    layout = dict(title=title, margin=dict(l=lm), width=w, height=h)\n    data = [trace]\n    fig = go.Figure(data=data, layout=layout)\n    fig.show()\n\n\ndef bar_ver_noagg(x, y, title, color, w=None, h=None, lm=0, rt = False):\n    trace = go.Bar(y=y, x=x, marker=dict(color=color))\n    if rt:\n        return trace\n    layout = dict(title=title, margin=dict(l=lm), width=w, height=h)\n    data = [trace]\n    fig = go.Figure(data=data, layout=layout)\n    fig.show()\n    \ndef gp(df, col, title):\n    df1 = df[df[\"data_source\"] == \"HPA\"]\n    df0 = df[df[\"data_source\"] == \"HuBMAP\"]\n    a1 = df1[col].value_counts()\n    b1 = df0[col].value_counts()\n    \n    total = dict(df[col].value_counts())\n    x0 = a1.index\n    x1 = b1.index\n    \n    y0 = [float(x)*100 / total[x0[i]] for i,x in enumerate(a1.values)]\n    y1 = [float(x)*100 / total[x1[i]] for i,x in enumerate(b1.values)]\n\n    trace1 = go.Bar(x=a1.index, y=y0, name='Target : 1', marker=dict(color=\"#96D38C\"))\n    trace2 = go.Bar(x=b1.index, y=y1, name='Target : 0', marker=dict(color=\"#FEBFB3\"))\n    return trace1, trace2 ","metadata":{"execution":{"iopub.status.busy":"2022-06-22T20:29:24.435452Z","iopub.execute_input":"2022-06-22T20:29:24.435891Z","iopub.status.idle":"2022-06-22T20:29:24.458248Z","shell.execute_reply.started":"2022-06-22T20:29:24.435860Z","shell.execute_reply":"2022-06-22T20:29:24.457075Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Source Distribution","metadata":{}},{"cell_type":"code","source":"bar_hor(dataframe, 'data_source', 'Data Source Distribution', [\"#F1948A\", '#ABEBC6'], h=350, w=600, lm=200, xlb = ['Source : HPA','Target : HuBMAP'])","metadata":{"execution":{"iopub.status.busy":"2022-06-22T20:29:24.772903Z","iopub.execute_input":"2022-06-22T20:29:24.773638Z","iopub.status.idle":"2022-06-22T20:29:24.786964Z","shell.execute_reply.started":"2022-06-22T20:29:24.773596Z","shell.execute_reply":"2022-06-22T20:29:24.785980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Gender Distribution","metadata":{}},{"cell_type":"code","source":"tr0 = bar_hor(dataframe, \"sex\", \"Distribution of Gender Variable\" ,\"#f975ae\", w=700, lm=100, return_trace=True)\ntr1, tr2 = gp(dataframe, 'sex', 'Distribution of Source with Gender')\n\nfig = tools.make_subplots(rows=1, cols=3, print_grid=False, subplot_titles = [\"Gender Distribution\" , \"Gender, Data Source=HMAP\" ,\"Gender, Data Source=HuBMAP\"])\nfig.append_trace(tr0, 1, 1);\nfig.append_trace(tr1, 1, 2);\nfig.append_trace(tr2, 1, 3);\nfig['layout'].update(height=350, showlegend=False, margin=dict(l=50));\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-22T20:29:25.097563Z","iopub.execute_input":"2022-06-22T20:29:25.098316Z","iopub.status.idle":"2022-06-22T20:29:25.291106Z","shell.execute_reply.started":"2022-06-22T20:29:25.098281Z","shell.execute_reply":"2022-06-22T20:29:25.289876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Organ Type Distribution","metadata":{}},{"cell_type":"code","source":"bar_hor(dataframe, \"organ\", \"Organ Type Distribution\" ,\"#f975ae\", w=700, lm=100, return_trace= False)","metadata":{"execution":{"iopub.status.busy":"2022-06-22T20:32:30.335129Z","iopub.execute_input":"2022-06-22T20:32:30.335730Z","iopub.status.idle":"2022-06-22T20:32:30.351579Z","shell.execute_reply.started":"2022-06-22T20:32:30.335684Z","shell.execute_reply":"2022-06-22T20:32:30.350711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Gender Distribution","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12,5))\nplt.title(\"Distribution of Age\")\nax = sns.distplot((dataframe[\"age\"]))","metadata":{"execution":{"iopub.status.busy":"2022-06-22T20:36:23.347260Z","iopub.execute_input":"2022-06-22T20:36:23.347749Z","iopub.status.idle":"2022-06-22T20:36:23.643272Z","shell.execute_reply.started":"2022-06-22T20:36:23.347712Z","shell.execute_reply":"2022-06-22T20:36:23.641643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Age Distribution of Gender's","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12,5));\nplt.style.use('default')\nsns.kdeplot(dataframe.loc[dataframe['sex'] == \"Male\", 'age'], label = 'Gender == Male')\nsns.kdeplot(dataframe.loc[dataframe['sex'] == \"Female\", 'age'], label = 'Gender == Female')\nplt.xlabel('Age (years)'); plt.ylabel('Density'); plt.title('Distribution of Ages'); plt.legend()","metadata":{"execution":{"iopub.status.busy":"2022-06-22T20:38:51.367036Z","iopub.execute_input":"2022-06-22T20:38:51.367533Z","iopub.status.idle":"2022-06-22T20:38:51.757866Z","shell.execute_reply.started":"2022-06-22T20:38:51.367482Z","shell.execute_reply":"2022-06-22T20:38:51.756481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## WORK IN PROGRESS ...","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}