{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# FTUs⚕️Segm: EDA🔎 & viewer","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"!pip wheel -q \"https://github.com/Borda/kaggle_image-segm/archive/refs/heads/main.zip\" --wheel-dir frozen_packages\n!pip wheel -q \"https://github.com/PyTorchLightning/lightning-flash/archive/refs/heads/segm/multi-label.zip\" --wheel-dir frozen_packages\n!rm frozen_packages/torch*\n!ls -l frozen_packages | grep -e kaggle -e lightning\n!pip install -q 'kaggle-image-segmentation' --find-links frozen_packages --no-index","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-12T22:09:18.888360Z","iopub.execute_input":"2022-08-12T22:09:18.888737Z","iopub.status.idle":"2022-08-12T22:11:02.204191Z","shell.execute_reply.started":"2022-08-12T22:09:18.888662Z","shell.execute_reply":"2022-08-12T22:11:02.203404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, glob\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nDATASET_FOLDER = \"/kaggle/input/hubmap-organ-segmentation\"\npath_csv = os.path.join(DATASET_FOLDER, \"train.csv\")\ndf_train = pd.read_csv(path_csv)\ndisplay(df_train.head())","metadata":{"execution":{"iopub.status.busy":"2022-08-12T22:11:02.205896Z","iopub.execute_input":"2022-08-12T22:11:02.206135Z","iopub.status.idle":"2022-08-12T22:11:02.553510Z","shell.execute_reply.started":"2022-08-12T22:11:02.206111Z","shell.execute_reply":"2022-08-12T22:11:02.552545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Explore metadata","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(12, 5))\nfor i, col in enumerate([\"organ\", \"sex\"]):\n    _= df_train[[col]].value_counts().plot.pie(ax=axes[i], autopct='%1.1f%%', ylabel=col)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T22:11:02.554706Z","iopub.execute_input":"2022-08-12T22:11:02.555312Z","iopub.status.idle":"2022-08-12T22:11:02.814107Z","shell.execute_reply.started":"2022-08-12T22:11:02.555277Z","shell.execute_reply":"2022-08-12T22:11:02.813318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(nrows=1, ncols=3, figsize=(12, 4))\nfor i, col in enumerate([\"data_source\", \"tissue_thickness\", \"pixel_size\"]):\n    _= df_train[[col]].value_counts().plot.pie(ax=axes[i], autopct='%1.1f%%', ylabel=col)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T22:11:02.816436Z","iopub.execute_input":"2022-08-12T22:11:02.816711Z","iopub.status.idle":"2022-08-12T22:11:03.005451Z","shell.execute_reply.started":"2022-08-12T22:11:02.816683Z","shell.execute_reply":"2022-08-12T22:11:03.004727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_= df_train[[\"age\"]].hist(bins=35, figsize=(8, 4))","metadata":{"execution":{"iopub.status.busy":"2022-08-12T22:11:03.006610Z","iopub.execute_input":"2022-08-12T22:11:03.007034Z","iopub.status.idle":"2022-08-12T22:11:03.203198Z","shell.execute_reply.started":"2022-08-12T22:11:03.007005Z","shell.execute_reply":"2022-08-12T22:11:03.202091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\n\nLABELS = list(df_train[\"organ\"].unique())\nwith open(\"labels.json\", \"w\") as fp:\n    json.dump(LABELS, fp)\n    \n! cat labels.json","metadata":{"execution":{"iopub.status.busy":"2022-08-12T22:11:03.204385Z","iopub.execute_input":"2022-08-12T22:11:03.204711Z","iopub.status.idle":"2022-08-12T22:11:03.527055Z","shell.execute_reply.started":"2022-08-12T22:11:03.204678Z","shell.execute_reply":"2022-08-12T22:11:03.526279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Image sizes histograms","metadata":{}},{"cell_type":"code","source":"for _, row in df_train.iterrows():\n    if row[\"img_height\"] != row[\"img_width\"]:\n        print(dict(row))\ndf_train[\"img_size\"] = df_train[\"img_width\"]","metadata":{"execution":{"iopub.status.busy":"2022-08-12T22:11:03.528064Z","iopub.execute_input":"2022-08-12T22:11:03.529125Z","iopub.status.idle":"2022-08-12T22:11:03.554146Z","shell.execute_reply.started":"2022-08-12T22:11:03.529099Z","shell.execute_reply":"2022-08-12T22:11:03.553336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax = df_train[\"img_size\"].hist(bins=35)\nax.set_yscale('log')","metadata":{"execution":{"iopub.status.busy":"2022-08-12T22:11:03.555394Z","iopub.execute_input":"2022-08-12T22:11:03.555917Z","iopub.status.idle":"2022-08-12T22:11:04.146964Z","shell.execute_reply.started":"2022-08-12T22:11:03.555865Z","shell.execute_reply":"2022-08-12T22:11:04.145930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.groupby([\"img_height\", \"img_width\"]).size()","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-12T22:11:04.148087Z","iopub.execute_input":"2022-08-12T22:11:04.149404Z","iopub.status.idle":"2022-08-12T22:11:04.159897Z","shell.execute_reply.started":"2022-08-12T22:11:04.149353Z","shell.execute_reply":"2022-08-12T22:11:04.158859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Show some images","metadata":{}},{"cell_type":"code","source":"import numpy as np\n\ndef rle_decode(mask_rle: str, img_shape: tuple = None) -> np.ndarray:\n    seq = mask_rle.split()\n    starts = np.array(list(map(int, seq[0::2])))\n    lengths = np.array(list(map(int, seq[1::2])))\n    assert len(starts) == len(lengths)\n    ends = starts + lengths\n    img = np.zeros((np.product(img_shape),), dtype=np.uint8)\n    for begin, end in zip(starts, ends):\n        img[begin:end] = 1\n    # https://stackoverflow.com/a/46574906/4521646\n    img.shape = img_shape\n    return img","metadata":{"execution":{"iopub.status.busy":"2022-08-12T22:11:04.162842Z","iopub.execute_input":"2022-08-12T22:11:04.163432Z","iopub.status.idle":"2022-08-12T22:11:04.171117Z","shell.execute_reply.started":"2022-08-12T22:11:04.163405Z","shell.execute_reply":"2022-08-12T22:11:04.169660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom skimage import color\nfrom kaggle_imsegm.mask import rle_encode\n\nfig, axes = plt.subplots(nrows=7, ncols=2, figsize=(9, 30))\nfor i, (_, row) in enumerate(df_train.sample(14).iterrows()):\n    img = plt.imread(os.path.join(DATASET_FOLDER, \"train_images\", f\"{row['id']}.tiff\"))\n    mask = rle_decode(row['rle'], img_shape=(row[\"img_height\"], row[\"img_width\"])).T\n    axes[i // 2, i % 2].imshow(color.label2rgb(mask, img, bg_label=0, bg_color=(1.,1.,1.), alpha=0.25))\n    axes[i // 2, i % 2].set_axis_off()\n    rle = rle_encode(mask.T)\n    assert row['rle'] == rle[1]\nfig.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T22:11:04.172503Z","iopub.execute_input":"2022-08-12T22:11:04.172731Z","iopub.status.idle":"2022-08-12T22:11:51.760895Z","shell.execute_reply.started":"2022-08-12T22:11:04.172708Z","shell.execute_reply":"2022-08-12T22:11:51.759930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Export masks","metadata":{}},{"cell_type":"code","source":"! mkdir -p ./train_images\n! mkdir -p ./train_binary_masks\n! mkdir -p ./train_mclass_masks\n\nfrom PIL import Image\nfrom tqdm.auto import tqdm\n\nfor _, row in tqdm(df_train.iterrows(), total=len(df_train)):\n    img = Image.open(os.path.join(DATASET_FOLDER, \"train_images\", f\"{row['id']}.tiff\"))\n    img.save(os.path.join(\"train_images\", f\"{row['id']}.png\"))\n    del img\n    mask = rle_decode(row['rle'], img_shape=(row[\"img_height\"], row[\"img_width\"])).T\n    Image.fromarray(mask).save(os.path.join(\"train_binary_masks\", f\"{row['id']}.png\"))\n    mask = mask * (LABELS.index(row[\"organ\"]) + 1)\n    Image.fromarray(mask).save(os.path.join(\"train_mclass_masks\", f\"{row['id']}.png\"))\n    del mask","metadata":{"execution":{"iopub.status.busy":"2022-08-12T22:11:51.761885Z","iopub.execute_input":"2022-08-12T22:11:51.762144Z","iopub.status.idle":"2022-08-12T22:23:32.768995Z","shell.execute_reply.started":"2022-08-12T22:11:51.762120Z","shell.execute_reply":"2022-08-12T22:23:32.767517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\n\nfig, axes = plt.subplots(nrows=8, ncols=3, figsize=(9, 18))\nfor i, row in df_train.sample(8).reset_index().iterrows():\n    img = plt.imread(os.path.join(\"train_images\", f\"{row['id']}.png\"))\n    mask = np.array(Image.open(os.path.join(\"train_binary_masks\", f\"{row['id']}.png\")))\n    segm = np.array(Image.open(os.path.join(\"train_mclass_masks\", f\"{row['id']}.png\")))\n    contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)\n    for j, _ in enumerate(contours):\n        cv2.drawContours(img, contours, j, color=(0, 255, 0), thickness=15)\n    axes[i, 0].imshow(img)\n    show_args = dict(vmin=0, interpolation='antialiased', interpolation_stage='rgba') \n    axes[i, 1].imshow(mask, vmax=2, **show_args)\n    im = axes[i, 2].imshow(segm, vmax=len(LABELS), **show_args)\n    plt.colorbar(im, ax=axes[i, 2])\n    print(np.unique(mask[:]), np.unique(segm[:]))\n    # axes[i // 2, i % 2].imshow(color.label2rgb(mask.T, img, bg_label=0, bg_color=(1.,1.,1.), alpha=0.25))\n    for j in range(3):\n        axes[i, j].set_axis_off()\nfig.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T22:24:27.078690Z","iopub.execute_input":"2022-08-12T22:24:27.079026Z","iopub.status.idle":"2022-08-12T22:24:39.691333Z","shell.execute_reply.started":"2022-08-12T22:24:27.079001Z","shell.execute_reply":"2022-08-12T22:24:39.690152Z"},"trusted":true},"execution_count":null,"outputs":[]}]}