{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\n\n
\n\n

🫁🫀 Explore Basic Submissions – HuBMAP+HPA – Organ Segmentation 🫀🫁

\n
CREATED BY: DARIEN SCHETTLER
\n\n
\n\n---\n\n
\n\n
\n 🛑   WARNING:

THIS IS A WORK IN PROGRESS
\n
\n\n\n
\n 👏   IF YOU FORK THIS OR FIND THIS HELPFUL   👏

PLEASE UPVOTE!

This was a lot of work for me and while it may seem silly, it makes me feel appreciated when others like my work. 😅\n
\n\n\n","metadata":{}},{"cell_type":"code","source":"print(\"\\n... IMPORTS STARTING ...\\n\")\n\nprint(\"\\n\\tVERSION INFORMATION\")\n# Machine Learning and Data Science Imports\nimport tensorflow as tf; print(f\"\\t\\t– TENSORFLOW VERSION: {tf.__version__}\");\nimport tensorflow_hub as tfhub; print(f\"\\t\\t– TENSORFLOW HUB VERSION: {tfhub.__version__}\");\nimport tensorflow_addons as tfa; print(f\"\\t\\t– TENSORFLOW ADDONS VERSION: {tfa.__version__}\");\nimport pandas as pd; pd.options.mode.chained_assignment = None;\nimport numpy as np; print(f\"\\t\\t– NUMPY VERSION: {np.__version__}\");\nimport sklearn; print(f\"\\t\\t– SKLEARN VERSION: {sklearn.__version__}\");\nfrom sklearn.preprocessing import RobustScaler, PolynomialFeatures\nfrom pandarallel import pandarallel; pandarallel.initialize();\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nfrom scipy.spatial import cKDTree\n\n# Built In Imports\nfrom kaggle_datasets import KaggleDatasets\nfrom collections import Counter\nfrom datetime import datetime\nfrom glob import glob\nimport warnings\nimport requests\nimport hashlib\nimport imageio\nimport IPython\nimport sklearn\nimport urllib\nimport zipfile\nimport pickle\nimport random\nimport shutil\nimport string\nimport json\nimport math\nimport time\nimport gzip\nimport ast\nimport sys\nimport io\nimport os\nimport gc\nimport re\n\n# Visualization Imports\nfrom matplotlib.colors import ListedColormap\nfrom matplotlib.patches import Rectangle\nimport matplotlib.patches as patches\nimport plotly.graph_objects as go\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm; tqdm.pandas();\nimport plotly.express as px\nimport seaborn as sns\nfrom PIL import Image, ImageEnhance\nimport matplotlib; print(f\"\\t\\t– MATPLOTLIB VERSION: {matplotlib.__version__}\");\nfrom matplotlib import animation, rc; rc('animation', html='jshtml')\nimport plotly\nimport PIL\nimport cv2\n \nprint(\"\\n\\n... IMPORTS COMPLETE ...\\n\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-27T19:47:59.551214Z","iopub.execute_input":"2022-06-27T19:47:59.552187Z","iopub.status.idle":"2022-06-27T19:48:13.448999Z","shell.execute_reply.started":"2022-06-27T19:47:59.552075Z","shell.execute_reply":"2022-06-27T19:48:13.44784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"\\n... BASIC DATA SETUP STARTING ...\\n\\n\")\n\n# Open the training dataframe and display the initial dataframe\nDATA_DIR = \"/kaggle/input/hubmap-organ-segmentation\"\n\n# Open the training dataframe and display the initial dataframe\nTRAIN_IMAGES_DIR = os.path.join(DATA_DIR, \"train_images\")\nTRAIN_LABELS_DIR = os.path.join(DATA_DIR, \"train_annotations\")\nTRAIN_CSV = os.path.join(DATA_DIR, \"train.csv\")\ntrain_df = pd.read_csv(TRAIN_CSV)\n\n# Get all training images\nall_train_images = glob(os.path.join(TRAIN_IMAGES_DIR, \"*.tiff\"), recursive=True)\nall_train_labels = glob(os.path.join(TRAIN_LABELS_DIR, \"*.json\"), recursive=True)\n\nprint(\"\\n... ORIGINAL TRAINING DATAFRAME... \\n\")\ndisplay(train_df)\n\nTEST_IMAGES_DIR = os.path.join(DATA_DIR, \"test_images\")\nTEST_CSV = os.path.join(DATA_DIR, \"test.csv\")\ntest_df = pd.read_csv(TEST_CSV)\n\nSS_CSV = os.path.join(DATA_DIR, \"sample_submission.csv\")\nss_df = pd.read_csv(SS_CSV)\n\n# Get all testing images if there are any\nall_test_images = glob(os.path.join(TEST_IMAGES_DIR, \"*.tiff\"), recursive=True)\n\nprint(\"\\n\\n\\n... ORIGINAL SUBMISSION DATAFRAME... \\n\")\ndisplay(ss_df)\n\nprint(\"\\n... BASIC DATA SETUP FINISHED ...\\n\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-06-27T19:48:13.451392Z","iopub.execute_input":"2022-06-27T19:48:13.451826Z","iopub.status.idle":"2022-06-27T19:48:16.056745Z","shell.execute_reply.started":"2022-06-27T19:48:13.451782Z","shell.execute_reply":"2022-06-27T19:48:16.055147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ref: https://www.kaggle.com/paulorzp/run-length-encode-and-decode\n# modified from: https://www.kaggle.com/inversion/run-length-decoding-quick-start\ndef rle_decode(mask_rle, shape, color=1):\n \"\"\" TBD\n \n Args:\n mask_rle (str): run-length as string formated (start length)\n shape (tuple of ints): (height,width) of array to return \n \n Returns: \n Mask (np.array)\n - 1 indicating mask\n - 0 indicating background\n\n \"\"\"\n # Split the string by space, then convert it into a integer array\n s = np.array(mask_rle.split(), dtype=int)\n\n # Every even value is the start, every odd value is the \"run\" length\n starts = s[0::2] - 1\n lengths = s[1::2]\n ends = starts + lengths\n\n # The image image is actually flattened since RLE is a 1D \"run\"\n if len(shape)==3:\n h, w, d = shape\n img = np.zeros((h * w, d), dtype=np.float32)\n else:\n h, w = shape\n img = np.zeros((h * w,), dtype=np.float32)\n\n # The color here is actually just any integer you want!\n for lo, hi in zip(starts, ends):\n img[lo : hi] = color\n \n # Don't forget to change the image back to the original shape\n return img.reshape(shape).T\n\n# https://www.kaggle.com/namgalielei/which-reshape-is-used-in-rle\ndef rle_decode_top_to_bot_first(mask_rle, shape):\n \"\"\" TBD\n \n Args:\n mask_rle (str): run-length as string formated (start length)\n shape (tuple of ints): (height,width) of array to return \n \n Returns:\n Mask (np.array)\n - 1 indicating mask\n - 0 indicating background\n\n \"\"\"\n s = mask_rle.split()\n starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n starts -= 1\n ends = starts + lengths\n img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n for lo, hi in zip(starts, ends):\n img[lo:hi] = 1\n return img.reshape((shape[1], shape[0]), order='F').T # Reshape from top -> bottom first\n\n# ref.: https://www.kaggle.com/stainsby/fast-tested-rle\ndef rle_encode(img):\n \"\"\" TBD\n \n Args:\n img (np.array): \n - 1 indicating mask\n - 0 indicating background\n \n Returns: \n run length as string formated\n \"\"\"\n \n img = img.T\n pixels = img.flatten()\n pixels = np.concatenate([[0], pixels, [0]])\n runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n runs[1::2] -= runs[::2]\n return ' '.join(str(x) for x in runs)\n\ndef flatten_l_o_l(nested_list):\n \"\"\" Flatten a list of lists \"\"\"\n return [item for sublist in nested_list for item in sublist]\n\ndef load_json_to_dict(json_path):\n \"\"\" tbd \"\"\"\n with open(json_path) as json_file:\n data = json.load(json_file)\n return data\n\ndef tf_decode_tiff(img_path, to_numpy=False, to_rgb=False):\n img = tf.io.read_file(img_path)\n img = tfio.experimental.image.decode_tiff(img)\n \n # Optionals\n if to_rgb: img = tfio.experimental.color.rgba_to_rgb(img)\n if to_numpy: img = img.numpy()\n \n return img","metadata":{"execution":{"iopub.status.busy":"2022-06-27T19:48:16.058563Z","iopub.execute_input":"2022-06-27T19:48:16.058936Z","iopub.status.idle":"2022-06-27T19:48:16.080626Z","shell.execute_reply.started":"2022-06-27T19:48:16.05889Z","shell.execute_reply":"2022-06-27T19:48:16.077844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Probing\n# if (test_df.pixel_size>1.0).sum()>0:\n# raise ValueError()","metadata":{"execution":{"iopub.status.busy":"2022-06-27T19:49:29.307806Z","iopub.execute_input":"2022-06-27T19:49:29.30826Z","iopub.status.idle":"2022-06-27T19:49:29.314731Z","shell.execute_reply.started":"2022-06-27T19:49:29.308226Z","shell.execute_reply":"2022-06-27T19:49:29.313483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SUBMISSION_STYLE = \"smart_circle_ones\" # [all_zeros', 'all_ones', 'all_random', 'smart_circle_ones']\nSMART_CIRCLE_FRAC = 0.8\n\nif SUBMISSION_STYLE==\"all_zeros\":\n ss_df[\"rle\"] = \"\"\nelif SUBMISSION_STYLE==\"all_ones\":\n ss_df[\"rle\"] = \"1 \"+(test_df[\"img_width\"]*test_df[\"img_height\"]).astype(str)\nelif SUBMISSION_STYLE==\"all_random\":\n ss_df[\"rle\"] = test_df.apply(lambda row: rle_encode(np.where(np.random.random((row[\"img_width\"], row[\"img_height\"]))>0.5, 1.0, 0.0)), axis=1)\n\n# All ones in interior circle all zeros outside (mimic tissue sample)\n# - To construct our circle:\n# - we imagine a square (img_width by img_height)\n# - we then draw a circle originating at the image centre (img_width//2, img_height//2) w/ a radius of some % (let's say 80)\nelif SUBMISSION_STYLE==\"smart_circle_ones\":\n rles = []\n for i, (img_w, img_h) in enumerate(zip(test_df[\"img_width\"], test_df[\"img_height\"])):\n tmp_img = np.zeros((img_w, img_h))\n tmp_img = cv2.circle(tmp_img, (int(np.round(img_w/2)), int(np.round(img_h/2))), int(np.round((img_w/2)*SMART_CIRCLE_FRAC)), 1, -1)\n rles.append(rle_encode(tmp_img))\n if i==0:\n plt.figure(figsize=(10,10))\n plt.imshow(rle_decode(rles[0], (img_w, img_h)))\n plt.title(\"Example of Smart Circle Mask\", fontweight=\"bold\")\n plt.show()\n ss_df[\"rle\"] = rles\nelse:\n raise NotImplementedError()","metadata":{"execution":{"iopub.status.busy":"2022-06-27T19:48:16.184526Z","iopub.execute_input":"2022-06-27T19:48:16.185013Z","iopub.status.idle":"2022-06-27T19:48:17.114872Z","shell.execute_reply.started":"2022-06-27T19:48:16.184977Z","shell.execute_reply":"2022-06-27T19:48:17.113694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# else:\n# # 1. Remove broken masks\n# remove_ids = [\"case7_day0\", \"case81_day30\"]\n# for _id in remove_ids:\n# train_df = train_df[~train_df.id.str.contains(_id)].reset_index(drop=True)\n\n# # 2. Get existing training data mappings to align with known metadata\n# slice_px_map = {}\n# for _, row in train_df.groupby([\"slice_h\", \"slice_w\", \"px_spacing_h\", \"px_spacing_w\", \"slice_id\"])[[\"lb_seg_rle\", \"sb_seg_rle\", \"st_seg_rle\"]].first().reset_index().iterrows():\n# slice_px_map[f\"{row['slice_h']}-{row['slice_w']}-{row['px_spacing_h']}-{row['px_spacing_w']}-{row['slice_id']}-large_bowel\"] = row[\"lb_seg_rle\"]\n# slice_px_map[f\"{row['slice_h']}-{row['slice_w']}-{row['px_spacing_h']}-{row['px_spacing_w']}-{row['slice_id']}-small_bowel\"] = row[\"sb_seg_rle\"]\n# slice_px_map[f\"{row['slice_h']}-{row['slice_w']}-{row['px_spacing_h']}-{row['px_spacing_w']}-{row['slice_id']}-stomach\"] = row[\"st_seg_rle\"]\n \n# # 3. Create a lookup and assign the prediction RLE\n# ss_df[\"ident\"] = ss_df['slice_h'].astype(str)+\"-\"+ss_df['slice_w'].astype(str)+\"-\"+ss_df['px_spacing_h'].astype(str)+\"-\"+ss_df['px_spacing_w'].astype(str)+\"-\"+ss_df['slice_id'].astype(str)+\"-\"+ss_df[\"class\"].astype(str)\n# ss_df[\"predicted\"] = ss_df[\"ident\"].map(slice_px_map)\n# ss_df[\"predicted\"] = ss_df[\"predicted\"].apply(lambda x: \"\" if x==None else x)\n\n \n# # 4. If we want some additional post processing we do that here\n# if SUBMISSION_STYLE==\"smart_random\":\n \n# # 4a. Remove RLE masks from areas where no mask should exist\n# ss_df = ss_df.apply(fix_empty_slices, axis=1)\n \n# # 4b. Remove non-contiguous masks that do not align with known patterns\n# ss_df[\"prev_predicted\"] = ss_df.shift(3, fill_value=\"\")[\"predicted\"]\n# ss_df[\"next_predicted\"] = ss_df.shift(-3, fill_value=\"\")[\"predicted\"]\n# ss_df[\"seg_isolated\"] = ss_df.apply(is_isolated, axis=1)\n# ss_df = ss_df.apply(fix_nc_slices, axis=1)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-06-26T20:36:12.429804Z","iopub.execute_input":"2022-06-26T20:36:12.430131Z","iopub.status.idle":"2022-06-26T20:36:12.436139Z","shell.execute_reply.started":"2022-06-26T20:36:12.430101Z","shell.execute_reply":"2022-06-26T20:36:12.434843Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 5. Reduce number of columns (just in case) and save\nss_df = ss_df[[\"id\", \"rle\"]]\nss_df.to_csv(\"submission.csv\", index=False)\n\ndisplay(ss_df)","metadata":{"execution":{"iopub.status.busy":"2022-06-26T20:36:12.437693Z","iopub.execute_input":"2022-06-26T20:36:12.438169Z","iopub.status.idle":"2022-06-26T20:36:12.461703Z","shell.execute_reply.started":"2022-06-26T20:36:12.438135Z","shell.execute_reply":"2022-06-26T20:36:12.460341Z"},"trusted":true},"execution_count":null,"outputs":[]}]}