{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\n\n![](\"https://rs1.chemie.de/images//128537-76.jpg\")
\n\nCell Instance Segmentation Challenge
\nCREATED BY: DARIEN SCHETTLER
\n\n
\n\n---\n\n
\n\n\n π WARNING:
THIS IS A WORK IN PROGRESS
\n
\n\n\n\n π IF YOU FORK THIS OR FIND THIS HELPFUL π
PLEASE UPVOTE!
This was a lot of work for me and while it may seem silly, it makes me feel appreciated when others like my work. π
\n
\n\n\n","metadata":{}},{"cell_type":"markdown","source":"\n\n
\n\nTABLE OF CONTENTS
\n\n---\n\n\n\n---\n\n\n\n---\n\n\n\n---\n\n\n\n---\n\n\n\n---\n\n\n\n---","metadata":{}},{"cell_type":"markdown","source":"
\n\n\n\n0 IMPORTS ⤒
","metadata":{}},{"cell_type":"code","source":"print(\"\\n... IMPORTS STARTING ...\\n\")\n\nprint(\"\\n... PIP/APT INSTALLS AND DOWNLOADS/ZIP STARTING ...\")\n# Try to skip and disable so we can submit w/o internet\n!pip install -q ../input/tensorflow-model-optimization/numpy-1.21.3-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl\n!pip install -q ../input/tensorflow-model-optimization/dm_tree-0.1.6-cp37-cp37m-manylinux_2_24_x86_64.whl\n!pip install -q ../input/tensorflow-model-optimization/six-1.16.0-py2.py3-none-any.whl\n!pip install -q ../input/tensorflow-model-optimization/tensorflow_model_optimization-0.7.0-py2.py3-none-any.whl\n!pip install ../input/neural-structued-learning/neural_structured_learning-1.3.1-py2.py3-none-any.whl\n# !pip install -q --upgrade tensorflow_datasets\n# !pip install -q neural-structured-learning\nprint(\"... PIP/APT INSTALLS COMPLETE ...\\n\")\n\nprint(\"\\n\\tVERSION INFORMATION\")\n# Machine Learning and Data Science Imports\nimport tensorflow as tf; print(f\"\\t\\tβ TENSORFLOW VERSION: {tf.__version__}\");\nimport tensorflow_addons as tfa; print(f\"\\t\\tβ TENSORFLOW ADDONS VERSION: {tfa.__version__}\");\nimport pandas as pd; pd.options.mode.chained_assignment = None;\nimport numpy as np; print(f\"\\t\\tβ NUMPY VERSION: {np.__version__}\");\nimport sklearn; print(f\"\\t\\tβ SKLEARN VERSION: {sklearn.__version__}\");\nfrom sklearn.preprocessing import RobustScaler, PolynomialFeatures\nfrom pandarallel import pandarallel; pandarallel.initialize();\nfrom sklearn.model_selection import GroupKFold;\n\n# Built In Imports\nfrom kaggle_datasets import KaggleDatasets\nfrom collections import Counter\nfrom datetime import datetime\nfrom glob import glob\nimport warnings\nimport requests\nimport hashlib\nimport imageio\nimport IPython\nimport sklearn\nimport urllib\nimport zipfile\nimport pickle\nimport random\nimport shutil\nimport string\nimport json\nimport math\nimport time\nimport gzip\nimport ast\nimport sys\nimport io\nimport os\nimport gc\nimport re\n\n# Visualization Imports\nfrom matplotlib.colors import ListedColormap\nimport matplotlib.patches as patches\nimport plotly.graph_objects as go\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm; tqdm.pandas();\nimport plotly.express as px\nimport seaborn as sns\nfrom PIL import Image, ImageEnhance\nimport matplotlib; print(f\"\\t\\tβ MATPLOTLIB VERSION: {matplotlib.__version__}\");\nimport plotly\nimport PIL\nimport cv2\n\n\ndef seed_it_all(seed=7):\n \"\"\" Attempt to be Reproducible \"\"\"\n os.environ['PYTHONHASHSEED'] = str(seed)\n random.seed(seed)\n np.random.seed(seed)\n tf.random.set_seed(seed)\n\n \nprint(\"\\n\\n... IMPORTS COMPLETE ...\\n\")\n \nprint(\"\\n... EFFICIENTDET SETUP STARTING ...\")\n\n# SET LIBRARY DIRECTORY\nLIB_DIR = \"/kaggle/input/google-automl-efficientdetefficientnet-oct-2021\"\n\n# To give access to automl files\nsys.path.insert(0, LIB_DIR)\nsys.path.insert(0, os.path.join(LIB_DIR, \"automl-master\"))\nsys.path.insert(0, os.path.join(LIB_DIR, \"automl-master\", \"efficientdet\"))\nsys.path.insert(0, os.path.join(LIB_DIR, \"automl-master\", \"efficientdet\", \"tf2\"))\n \n# EfficientDET Module Imports\nimport hparams_config\nfrom tf2 import efficientdet_keras\nfrom tf2 import train_lib\nfrom tf2 import anchors\nfrom tf2 import efficientdet_keras\nfrom tf2 import label_util\nfrom tf2 import postprocess\nfrom tf2 import util_keras\nfrom tf2.train import setup_model\nfrom efficientdet import dataloader\nfrom visualize import vis_utils\nfrom inference import visualize_image\nprint(\"... EFFICIENTDET SETUP COMPLETE ...\\n\")\n\nprint(\"\\n... SEEDING FOR DETERMINISTIC BEHAVIOUR ...\\n\")\nseed_it_all()","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:20:46.013389Z","iopub.execute_input":"2021-11-06T20:20:46.014432Z","iopub.status.idle":"2021-11-06T20:23:16.361307Z","shell.execute_reply.started":"2021-11-06T20:20:46.014318Z","shell.execute_reply":"2021-11-06T20:23:16.360491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n\n\n---\n","metadata":{}},{"cell_type":"markdown","source":"1.1 BASIC COMPETITION INFORMATION
\n\n---\n\n
PRIMARY TASK DESCRIPTION\n\nThis is a placeholder copy/paste from the OVERVIEW>DESCRIPTION page of the Kaggle competition\n\n\nNeurological disorders, including neurodegenerative diseases such as Alzheimer's and brain tumors, are a leading cause of death and disability across the globe. However, it is hard to quantify how well these deadly disorders respond to treatment. One accepted method is to review neuronal cells via light microscopy, which is both accessible and non-invasive. Unfortunately, segmenting individual neuronal cells in microscopic images can be challenging and time-intensive. Accurate instance segmentation of these cellsβwith the help of computer visionβcould lead to new and effective drug discoveries to treat the millions of people with these disorders.\n\nCurrent solutions have limited accuracy for neuronal cells in particular. In internal studies to develop cell instance segmentation models, the neuroblastoma cell line SH-SY5Y consistently exhibits the lowest precision scores out of eight different cancer cell types tested. This could be because neuronal cells have a very unique, irregular and concave morphology associated with them, making them challenging to segment with commonly used mask heads.\n\nSartorius is a partner of the life science research and the biopharmaceutical industry. They empower scientists and engineers to simplify and accelerate progress in life science and bioprocessing, enabling the development of new and better therapies and more affordable medicine. They're a magnet and dynamic platform for pioneers and leading experts in the field. They bring creative minds together for a common goal: technological breakthroughs that lead to better health for more people.\n\n
\n\n---\n\nIn this competition, youβll detect and delineate distinct objects of interest in biological images depicting neuronal cell types commonly used in the study of neurological disorders. More specifically, you'll use phase contrast microscopy images to train and test your model for instance segmentation of neuronal cells. Successful models will do this with a high level of accuracy.\n\n---\n\n
\n\nIf successful, you'll help further research in neurobiology thanks to the collection of robust quantitative data. Researchers may be able to use this to more easily measure the effects of disease and treatment conditions on neuronal cells. As a result, new drugs could be discovered to treat the millions of people with these leading causes of death and disability.\n","metadata":{}},{"cell_type":"markdown","source":"1.2 COMPETITION EVALUATION
\n\n---\n\n
GENERAL EVALUATION INFORMATION\n\n\nThis competition is evaluated on the mean average precision at different intersection over union πΌππ thresholds. The IoU of a proposed set of object pixels and a set of true object pixels is calculated as:\n\n
$IoU(A,B) = \\frac{A \\cap B}{ A \\cup B}$
\n\nThe metric sweeps over a range of **IoU** thresholds, at each point calculating an average precision value. The threshold values range from 0.5 to 0.95 with a step size of 0.05: \n* i.e. 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95\n* In other words, at a threshold of 0.5, a predicted object is considered a \"hit\" if its intersection over union with a ground truth object is greater than 0.5\n\nAt each threshold value **π‘**, a precision value is calculated based on the number of true positives **ππ**, false negatives **πΉπ**, and false positives **πΉπ** resulting from comparing the predicted object to all ground truth objects:\n\n
$\\frac{TP(t)}{TP(t) + FP(t) + FN(t)}$
\n\n* A true positive is counted when a single predicted object matches a ground truth object with an IoU above the threshold. \n* A false positive indicates a predicted object had no associated ground truth object. A false negative indicates a ground truth object had no associated predicted object. \n* The average precision of a single image is then calculated as the mean of the above precision values at each \n\n
IoU threshold:\n\n
$\\frac{1}{|thresholds|} \\sum_t \\frac{TP(t)}{TP(t) + FP(t) + FN(t)}$
\n\nLastly, the score returned by the competition metric is the mean taken over the individual average precisions of each image in the test dataset.\n\n
SUBMISSION FILE INFORMATION\n\nIn order to reduce the submission file size, our metric uses run-length encoding (RLE) on the pixel values. Instead of submitting an exhaustive list of indices for your segmentation, you will submit pairs of values that contain a start position and a run length. \n* E.g. '1 3' implies starting at pixel 1 and running a total of 3 pixels 1,2,3.\n\nThe competition format requires a space delimited list of pairs. For example, `'1 3 10 5'` implies pixels `1,2,3,10,11,12,13,14` are to be included in the mask. The pixels are one-indexed\nand numbered from top to bottom, then left to right: \n* 1 is pixel 1,1\n* 2 is pixel 2,1\n* etc.\n\nThe metric checks that the pairs are sorted, positive, and the decoded pixel values are not duplicated. It also checks that no two predicted masks for the same image are overlapping.\n\nThe file should contain a header and have the following format. Each row in your submission represents a single predicted nucleus segmentation for the given **`ImageId`**.\n\n```\nImageId,EncodedPixels \n0114f484a16c152baa2d82fdd43740880a762c93f436c8988ac461c5c9dbe7d5,1 1 \n0999dab07b11bc85fb8464fc36c947fbd8b5d6ec49817361cb780659ca805eac,1 1 \n0999dab07b11bc85fb8464fc36c947fbd8b5d6ec49817361cb780659ca805eac,2 3 8 9 \netc...\n```\n\nSubmission files may take several minutes to process due to the size.\n\n
\n\n
IS THIS A CODE COMPETITION?\n\nYES\n\n
","metadata":{}},{"cell_type":"markdown","source":"1.3 DATASET OVERVIEW
\n\n---\n\n
GENERAL INFORMATION\n\nIn this competition we are segmenting neuronal cells in images. The training annotations are provided as run length encoded masks, and the images are in PNG format. The number of images is small, but the number of annotated objects is quite high. The hidden test set is roughly 240 images.\n\n
FILES\n\n**`train.csv`** \n- IDs and masks for all training objects. None of this metadata is provided for the test set.\n\n**`id`** \n- unique identifier for object\n\n**`annotation`**\n- run length encoded pixels for the identified neuronal cell\n\n**`width`** \n- source image width\n\n**`height`** \n- source image height\n\n**`cell_type`** \n- the cell line\n\n**`plate_time`** \n- time plate was created\n\n**`sample_date`** \n- date sample was created\n\n**`sample_id`** \n- sample identifier\n\n**`elapsed_timedelta`**\n- time since first image taken of sample\n\n**`sample_submission.csv`** \n- a sample submission file in the correct format\n\n**`train`** \n- train images in PNG format\n\n**`test`** \n- test images in PNG format. Only a few test set images are available for download; the remainder can only be accessed by your notebooks when you submit.\n\n**`train_semi_supervised`** \n- unlabeled images offered in case you want to use additional data for a semi-supervised approach.\n\n**`LIVECell_dataset_2021`** \n- A mirror of the data from the LIVECell dataset. \n- LIVECell is the predecessor dataset to this competition. \n- You will find extra data for the SH-SHY5Y cell line, plus several other cell lines not covered in the competition dataset that may be of interest for transfer learning.\n","metadata":{}},{"cell_type":"markdown","source":"
\n\n\n\n2 SETUP ⤒
\n\n---\n","metadata":{}},{"cell_type":"markdown","source":"2.1 ACCELERATOR DETECTION
\n\n---\n\nIn order to use **`TPU`**, we use **`TPUClusterResolver`** for the initialization which is necessary to connect to the remote cluster and initialize cloud TPUs. Let's go over two important points\n\n1. When using TPU on Kaggle, you don't need to specify arguments for **`TPUClusterResolver`**\n2. However, on **G**oogle **C**ompute **E**ngine (**GCE**), you will need to do the following:\n\n
\n\n```python\n# The name you gave to the TPU to use\nTPU_WORKER = 'my-tpu-name'\n\n# or you can also specify the grpc path directly\n# TPU_WORKER = 'grpc://xxx.xxx.xxx.xxx:8470'\n\n# The zone you chose when you created the TPU to use on GCP.\nZONE = 'us-east1-b'\n\n# The name of the GCP project where you created the TPU to use on GCP.\nPROJECT = 'my-tpu-project'\n\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver(tpu=TPU_WORKER, zone=ZONE, project=PROJECT)\n```\n\n\n π WARNING:
- Although the Tensorflow documentation says it is the project name that should be provided for the argument `project`, it is actually the Project ID, that you should provide. This can be found on the GCP project dashboard page.
\n
\n\n","metadata":{}},{"cell_type":"code","source":"print(f\"\\n... ACCELERATOR SETUP STARTING ...\\n\")\n\n# Detect hardware, return appropriate distribution strategy\ntry:\n # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n TPU = tf.distribute.cluster_resolver.TPUClusterResolver() \nexcept ValueError:\n TPU = None\n\nif TPU:\n print(f\"\\n... RUNNING ON TPU - {TPU.master()}...\")\n tf.config.experimental_connect_to_cluster(TPU)\n tf.tpu.experimental.initialize_tpu_system(TPU)\n strategy = tf.distribute.experimental.TPUStrategy(TPU)\nelse:\n print(f\"\\n... RUNNING ON CPU/GPU ...\")\n # Yield the default distribution strategy in Tensorflow\n # --> Works on CPU and single GPU.\n strategy = tf.distribute.get_strategy() \n\n# What Is a Replica?\n# --> A single Cloud TPU device consists of FOUR chips, each of which has TWO TPU cores. \n# --> Therefore, for efficient utilization of Cloud TPU, a program should make use of each of the EIGHT (4x2) cores. \n# --> Each replica is essentially a copy of the training graph that is run on each core and \n# trains a mini-batch containing 1/8th of the overall batch size\nN_REPLICAS = strategy.num_replicas_in_sync\n \nprint(f\"... # OF REPLICAS: {N_REPLICAS} ...\\n\")\n\nprint(f\"\\n... ACCELERATOR SETUP COMPLTED ...\\n\")","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:23:16.364262Z","iopub.execute_input":"2021-11-06T20:23:16.364468Z","iopub.status.idle":"2021-11-06T20:23:16.381149Z","shell.execute_reply.started":"2021-11-06T20:23:16.364441Z","shell.execute_reply":"2021-11-06T20:23:16.380412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"2.2 COMPETITION DATA ACCESS
\n\n---\n\nTPUs read data must be read directly from **G**oogle **C**loud **S**torage **(GCS)**. Kaggle provides a utility library βΒ **`KaggleDatasets`** β which has a utility function **`.get_gcs_path`** that will allow us to access the location of our input datasets within **GCS**.
\n\n\n π TIPS:
- If you have multiple datasets attached to the notebook, you should pass the name of a specific dataset to the `get_gcs_path()` function. In our case, the name of the dataset is the name of the directory the dataset is mounted within.
\n
","metadata":{}},{"cell_type":"code","source":"print(\"\\n... DATA ACCESS SETUP STARTED ...\\n\")\n\nif TPU:\n # Google Cloud Dataset path to training and validation images\n DATA_DIR = KaggleDatasets().get_gcs_path('sartorius-cell-instance-segmentation')\n save_locally = tf.saved_model.SaveOptions(experimental_io_device='/job:localhost')\nelse:\n # Local path to training and validation images\n DATA_DIR = \"/kaggle/input/sartorius-cell-instance-segmentation\"\n save_locally = None\n \nprint(f\"\\n... DATA DIRECTORY PATH IS:\\n\\t--> {DATA_DIR}\")\n\nprint(f\"\\n... IMMEDIATE CONTENTS OF DATA DIRECTORY IS:\")\nfor file in tf.io.gfile.glob(os.path.join(DATA_DIR, \"*\")): print(f\"\\t--> {file}\")\n\n \nprint(\"\\n\\n... DATA ACCESS SETUP COMPLETED ...\\n\")","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:23:16.382182Z","iopub.execute_input":"2021-11-06T20:23:16.382664Z","iopub.status.idle":"2021-11-06T20:23:16.398197Z","shell.execute_reply.started":"2021-11-06T20:23:16.382629Z","shell.execute_reply":"2021-11-06T20:23:16.397461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"2.3 LEVERAGING XLA OPTIMIZATIONS
\n\n---\n\n\n**XLA** (Accelerated Linear Algebra) is a domain-specific compiler for linear algebra that can accelerate TensorFlow models with potentially no source code changes. **The results are improvements in speed and memory usage**.\n\n
\n\nWhen a TensorFlow program is run, all of the operations are executed individually by the TensorFlow executor. Each TensorFlow operation has a precompiled GPU/TPU kernel implementation that the executor dispatches to.\n\nXLA provides us with an alternative mode of running models: it compiles the TensorFlow graph into a sequence of computation kernels generated specifically for the given model. Because these kernels are unique to the model, they can exploit model-specific information for optimization.
\n\n\n π WARNING:
- XLA can not currently compile functions where dimensions are not inferrable: that is, if it's not possible to infer the dimensions of all tensors without running the entire computation
\n
\n\n\n π NOTE:
- XLA compilation is only applied to code that is compiled into a graph (in TF2 that's only a code inside tf.function).
- The jit_compile API has must-compile semantics, i.e. either the entire function is compiled with XLA, or an errors.InvalidArgumentError exception is thrown)\n
\n\n","metadata":{}},{"cell_type":"code","source":"print(f\"\\n... XLA OPTIMIZATIONS STARTING ...\\n\")\n\nprint(f\"\\n... CONFIGURE JIT (JUST IN TIME) COMPILATION ...\\n\")\n# enable XLA optmizations (10% speedup when using @tf.function calls)\ntf.config.optimizer.set_jit(True)\n\nprint(f\"\\n... XLA OPTIMIZATIONS COMPLETED ...\\n\")","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:23:16.400041Z","iopub.execute_input":"2021-11-06T20:23:16.400407Z","iopub.status.idle":"2021-11-06T20:23:16.406106Z","shell.execute_reply.started":"2021-11-06T20:23:16.400371Z","shell.execute_reply":"2021-11-06T20:23:16.405360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"2.4 BASIC DATA DEFINITIONS & INITIALIZATIONS
\n\n---\n","metadata":{}},{"cell_type":"code","source":"print(\"\\n... BASIC DATA SETUP STARTING ...\\n\\n\")\n\nprint(\"\\n... SET PATH INFORMATION ..\\n\")\nSEG_DIR = \"/kaggle/input/sartorius-segmentation-train-mask-dataset-npz\"\nLC_DIR = os.path.join(DATA_DIR, \"LIVECell_dataset_2021\")\nLC_ANN_DIR = os.path.join(LC_DIR, \"annotations\")\nLC_IMG_DIR = os.path.join(LC_DIR, \"images\")\nTRAIN_DIR = os.path.join(DATA_DIR, \"train\")\nTEST_DIR = os.path.join(DATA_DIR, \"test\")\nSEMI_DIR = os.path.join(DATA_DIR, \"train_semi_supervised\")\n\nprint(\"\\n... TRAIN DATAFRAME ...\\n\")\n\n# FIX THE TRAIN DATAFRAME (GROUP THE RLEs TOGETHER)\nTRAIN_CSV = os.path.join(DATA_DIR, \"train.csv\")\ntrain_df = pd.read_csv(TRAIN_CSV)\ndisplay(train_df)\n\nprint(\"\\n... SS DATAFRAME ..\\n\")\nSS_CSV = os.path.join(DATA_DIR, \"sample_submission.csv\")\nss_df = pd.read_csv(SS_CSV)\nss_df[\"img_path\"] = ss_df[\"id\"].apply(lambda x: os.path.join(TEST_DIR, x+\".png\")) # Capture Image Path As Well\ndisplay(ss_df)\n\nCELL_TYPES = list(train_df.cell_type.unique())\nFIRST_SHSY5Y_IDX = 0\nFIRST_ASTRO_IDX = 1\nFIRST_CORT_IDX = 2\n\n# This is required for plotting so that the smaller distributions get plotted on top\nARB_SORT_MAP = {\"astro\":0, \"shsy5y\":1, \"cort\":2}\n\nprint(\"\\n... CELL TYPES ..\")\nfor x in CELL_TYPES: print(f\"\\t--> {x}\")\n \nprint(\"\\n\\n... BASIC DATA SETUP FINISHING ...\\n\")","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:23:16.407473Z","iopub.execute_input":"2021-11-06T20:23:16.408338Z","iopub.status.idle":"2021-11-06T20:23:16.952353Z","shell.execute_reply.started":"2021-11-06T20:23:16.408302Z","shell.execute_reply":"2021-11-06T20:23:16.951398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n\n\n\n 3 HELPER FUNCTION & CLASSES ⤒\n
\n\n---","metadata":{}},{"cell_type":"code","source":"# ref: https://www.kaggle.com/paulorzp/run-length-encode-and-decode\n# modified from: https://www.kaggle.com/inversion/run-length-decoding-quick-start\ndef rle_decode(mask_rle, shape, color=1):\n \"\"\" TBD\n \n Args:\n mask_rle (str): run-length as string formated (start length)\n shape (tuple of ints): (height,width) of array to return \n \n Returns: \n Mask (np.array)\n - 1 indicating mask\n - 0 indicating background\n\n \"\"\"\n # Split the string by space, then convert it into a integer array\n s = np.array(mask_rle.split(), dtype=int)\n\n # Every even value is the start, every odd value is the \"run\" length\n starts = s[0::2] - 1\n lengths = s[1::2]\n ends = starts + lengths\n\n # The image image is actually flattened since RLE is a 1D \"run\"\n if len(shape)==3:\n h, w, d = shape\n img = np.zeros((h * w, d), dtype=np.float32)\n else:\n h, w = shape\n img = np.zeros((h * w,), dtype=np.float32)\n\n # The color here is actually just any integer you want!\n for lo, hi in zip(starts, ends):\n img[lo : hi] = color\n \n # Don't forget to change the image back to the original shape\n return img.reshape(shape)\n\n# https://www.kaggle.com/namgalielei/which-reshape-is-used-in-rle\ndef rle_decode_top_to_bot_first(mask_rle, shape):\n \"\"\" TBD\n \n Args:\n mask_rle (str): run-length as string formated (start length)\n shape (tuple of ints): (height,width) of array to return \n \n Returns:\n Mask (np.array)\n - 1 indicating mask\n - 0 indicating background\n\n \"\"\"\n s = mask_rle.split()\n starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n starts -= 1\n ends = starts + lengths\n img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n for lo, hi in zip(starts, ends):\n img[lo:hi] = 1\n return img.reshape((shape[1], shape[0]), order='F').T # Reshape from top -> bottom first\n\n# ref.: https://www.kaggle.com/stainsby/fast-tested-rle\ndef rle_encode(img):\n \"\"\" TBD\n \n Args:\n img (np.array): \n - 1 indicating mask\n - 0 indicating background\n \n Returns: \n run length as string formated\n \"\"\"\n \n pixels = img.flatten()\n pixels = np.concatenate([[0], pixels, [0]])\n runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n runs[1::2] -= runs[::2]\n return ' '.join(str(x) for x in runs)\n\n\ndef flatten_l_o_l(nested_list):\n \"\"\" Flatten a list of lists \"\"\"\n return [item for sublist in nested_list for item in sublist]\n\n\ndef load_json_to_dict(json_path):\n \"\"\" tbd \"\"\"\n with open(json_path) as json_file:\n data = json.load(json_file)\n return data\n\n# https://github.com/PyImageSearch/imutils/blob/master/imutils/convenience.py\ndef grab_contours(cnts):\n \"\"\" TBD \"\"\"\n \n # if the length the contours tuple returned by cv2.findContours\n # is '2' then we are using either OpenCV v2.4, v4-beta, or\n # v4-official\n if len(cnts) == 2:\n cnts = cnts[0]\n\n # if the length of the contours tuple is '3' then we are using\n # either OpenCV v3, v4-pre, or v4-alpha\n elif len(cnts) == 3:\n cnts = cnts[1]\n\n # otherwise OpenCV has changed their cv2.findContours return\n # signature yet again and I have no idea WTH is going on\n else:\n raise Exception((\"Contours tuple must have length 2 or 3, \"\n \"otherwise OpenCV changed their cv2.findContours return \"\n \"signature yet again. Refer to OpenCV's documentation \"\n \"in that case\"))\n\n # return the actual contours array\n return cnts\n\ndef get_contour_bbox(msk):\n \"\"\" Function to return the bounding box (tl, br) for a given mask \"\"\"\n \n # Get contour(s) --> There should be only one\n cnts = cv2.findContours(msk.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n\n contour = grab_contours(cnts)\n \n if len(contour)==0:\n return None\n else:\n contour = contour[0]\n \n # Get extreme coordinates\n tl = (tuple(contour[contour[:, :, 0].argmin()][0])[0], \n tuple(contour[contour[:, :, 1].argmin()][0])[1])\n br = (tuple(contour[contour[:, :, 0].argmax()][0])[0], \n tuple(contour[contour[:, :, 1].argmax()][0])[1])\n return tl, br\n\ndef tf_load_png(img_path):\n return tf.image.decode_png(tf.io.read_file(img_path), channels=3)\n\ndef get_img_and_mask(img_path, annotation, width, height, mask_only=False, rle_fn=rle_decode):\n \"\"\" Capture the relevant image array as well as the image mask \"\"\"\n img_mask = np.zeros((height, width), dtype=np.uint8)\n for i, annot in enumerate(annotation): \n img_mask = np.where(rle_fn(annot, (height, width))!=0, i, img_mask)\n \n # Early Exit\n if mask_only:\n return img_mask\n \n # Else Return images\n img = tf_load_png(img_path)[..., 0]\n return img, img_mask\n\ndef plot_img_and_mask(img, mask, bboxes=None, invert_img=True, boost_contrast=True):\n \"\"\" Function to take an image and the corresponding mask and plot\n \n Args:\n img (np.arr): 1 channel np arr representing the image of cellular structures\n mask (np.arr): 1 channel np arr representing the instance masks (incrementing by one)\n bboxes (list of tuples, optional): (tl, br) coordinates of enclosing bboxes\n invert_img (bool, optional): Whether or not to invert the base image\n boost_contrast (bool, optional): Whether or not to boost contrast of the base image\n \n Returns:\n None; Plots the two arrays and overlays them to create a merged image\n \"\"\"\n plt.figure(figsize=(20,10))\n \n plt.subplot(1,3,1)\n _img = np.tile(np.expand_dims(img, axis=-1), 3)\n \n # Flip black-->white ... white-->black\n if invert_img:\n _img = _img.max()-_img\n \n if boost_contrast:\n _img = np.asarray(ImageEnhance.Contrast(Image.fromarray(_img)).enhance(16))\n \n if bboxes:\n for i, bbox in enumerate(bboxes):\n mask = cv2.rectangle(mask, bbox[0], bbox[1], (i+1, 0, 0), thickness=2)\n \n plt.imshow(_img)\n plt.axis(False)\n plt.title(\"Cell Image\", fontweight=\"bold\")\n \n plt.subplot(1,3,2)\n _mask = np.zeros_like(_img)\n _mask[..., 0] = mask\n plt.imshow(mask, cmap=\"inferno\")\n plt.axis(False)\n plt.title(\"Instance Segmentation Mask\", fontweight=\"bold\")\n \n merged = cv2.addWeighted(_img, 0.75, np.clip(_mask, 0, 1)*255, 0.25, 0.0,)\n plt.subplot(1,3,3)\n plt.imshow(merged)\n plt.axis(False)\n plt.title(\"Cell Image w/ Instance Segmentation Mask Overlay\", fontweight=\"bold\")\n \n plt.tight_layout()\n plt.show()\n \ndef pd_get_bboxes(row):\n \"\"\" Get all bboxes for a given row/cell-image \"\"\"\n mask = get_img_and_mask(row.img_path, row.annotation, row.width, row.height, mask_only=True)\n return [get_contour_bbox(np.where(mask==i, 1, 0).astype(np.uint8)) for i in range(1, mask.max()+1)]\n\ndef get_bbox_stats(bbox_list, style=\"area\"): \n \"\"\" TBD \n \n Args:\n bbox_list(): TBD\n style (str, optional): TBD\n Returns:\n TBD\n \"\"\"\n bbox_stats = []\n for box in bbox_list:\n try:\n if style==\"area\":\n bbox_stats.append(float((box[1][0]-box[0][0])*(box[1][1]-box[0][1])))\n elif style==\"width\":\n bbox_stats.append(float(box[1][0]-box[0][0]))\n else:\n bbox_stats.append(float(box[1][1]-box[0][1]))\n except:\n bbox_stats.append(0.0)\n return bbox_stats","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:23:16.953745Z","iopub.execute_input":"2021-11-06T20:23:16.954165Z","iopub.status.idle":"2021-11-06T20:23:16.989159Z","shell.execute_reply.started":"2021-11-06T20:23:16.954125Z","shell.execute_reply":"2021-11-06T20:23:16.988484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n\n\n\n 4 DATASET CREATION AND EXPLORATION ⤒\n
\n\n---","metadata":{}},{"cell_type":"markdown","source":"4.0 UPDATE THE TRAIN DATAFRAME
\n\n---\n\nWe need to change a few things with the train dataframe\n* Aggregate under **`id`**\n* Add in certain columns\n","metadata":{}},{"cell_type":"code","source":"# Aggregate under training \ntrain_df[\"img_path\"] = train_df[\"id\"].apply(lambda x: os.path.join(TRAIN_DIR, x+\".png\")) # Capture Image Path As Well\ntmp_df = train_df.drop_duplicates(subset=[\"id\", \"img_path\"]).reset_index(drop=True)\ntmp_df[\"annotation\"] = train_df.groupby(\"id\")[\"annotation\"].agg(list).reset_index(drop=True)\ntrain_df = tmp_df.copy()\ntrain_df[\"seg_path\"] = train_df.id.apply(lambda x: os.path.join(SEG_DIR, f\"{x}.npz\"))\ndisplay(train_df)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:23:16.990229Z","iopub.execute_input":"2021-11-06T20:23:16.990874Z","iopub.status.idle":"2021-11-06T20:23:17.235411Z","shell.execute_reply.started":"2021-11-06T20:23:16.990832Z","shell.execute_reply":"2021-11-06T20:23:17.234650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4.1 VISUALIZE THE TRAIN DATA
\n\n---\n\nLet's create a function to take a single example (row of the dataframe) and plot the resulting information\n","metadata":{}},{"cell_type":"code","source":"for i in range(2, 70, 8):\n print(f\"\\n\\n\\n\\n... RELEVANT DATAFRAME ROW - INDEX={i} ...\\n\")\n display(train_df.iloc[i:i+1])\n img, msk = get_img_and_mask(**train_df[[\"img_path\", \"annotation\", \"width\", \"height\"]].iloc[i].to_dict())\n plot_img_and_mask(img, msk)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:23:17.236782Z","iopub.execute_input":"2021-11-06T20:23:17.237032Z","iopub.status.idle":"2021-11-06T20:23:25.560674Z","shell.execute_reply.started":"2021-11-06T20:23:17.236998Z","shell.execute_reply":"2021-11-06T20:23:25.560022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"QUICK DOUBLECHECK ON RLE DECODE FUNCTION\n* https://www.kaggle.com/namgalielei/which-reshape-is-used-in-rle","metadata":{}},{"cell_type":"code","source":"x1 = rle_decode_top_to_bot_first(train_df.iloc[0].annotation[0], (train_df.iloc[0].height, train_df.iloc[0].width))\nx2 = rle_decode(train_df.iloc[0].annotation[0], (train_df.iloc[0].height, train_df.iloc[0].width))\n\nplt.figure(figsize=(15,6))\nplt.subplot(1,2,1)\nplt.imshow(x1, cmap=\"inferno\")\nplt.axis(False)\nplt.title(\"NamGalielei RLE Decode Function\", fontweight=\"bold\")\nplt.subplot(1,2,2)\nplt.imshow(x2, cmap=\"inferno\")\nplt.axis(False)\nplt.title(\"Original RLE Decode Function\", fontweight=\"bold\")\nplt.tight_layout()\nplt.show()\nprint(f\"\\n... THERE ARE {(x1!=x2).sum()} PIXELS IN DISAGREEMENT WHEN USING THE TWO FUNCTIONS ON A SINGLE CELL...\\n\")\n\nimg1, msk1 = get_img_and_mask(**train_df[[\"img_path\", \"annotation\", \"width\", \"height\"]].iloc[0].to_dict())\nimg2, msk2 = get_img_and_mask(**train_df[[\"img_path\", \"annotation\", \"width\", \"height\"]].iloc[0].to_dict(), rle_fn=rle_decode_top_to_bot_first)\n\nplot_img_and_mask(img1, msk1)\nplot_img_and_mask(img2, msk2)\n\nprint(f\"\\n... THERE ARE {(msk2!=msk1).sum()} PIXELS IN DISAGREEMENT WHEN USING THE TWO FUNCTIONS ON ALL CELL MASK ...\\n\")","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:23:25.561734Z","iopub.execute_input":"2021-11-06T20:23:25.562092Z","iopub.status.idle":"2021-11-06T20:23:27.352876Z","shell.execute_reply.started":"2021-11-06T20:23:25.562058Z","shell.execute_reply":"2021-11-06T20:23:27.352173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4.2 INVESTIGATE THE TRAIN DATAFRAME
\n\n---\n","metadata":{}},{"cell_type":"code","source":"print(\"\\n\\n... WIDTH VALUE COUNTS ...\")\nfor k,v in train_df.width.value_counts().items():\n print(f\"\\t--> There are {v} images with WIDTH={k}\")\n\nprint(\"\\n\\n... HEIGHT VALUE COUNTS ...\")\nfor k,v in train_df.height.value_counts().items():\n print(f\"\\t--> There are {v} images with HEIGHT={k}\")\n\nprint(\"\\n\\n... AREA COUNTS ...\")\nfor k,v in (train_df.width*train_df.height).value_counts().items():\n print(f\"\\t--> There are {v} images with AREA={k}\")\n\nprint(\"\\n\\n... NOTE: ALL THE IMAGES ARE THE SAME SIZE ...\\n\")\n\nprint(\"\\n\\n... PLATE TIME VALUE COUNTS ...\")\nfor k,v in train_df.plate_time.value_counts().items():\n print(f\"\\t--> There are {v} images with PLATE_TIME={k}\")\nfig = px.histogram(train_df, x=\"plate_time\", color=\"cell_type\", title=\"Plate Time Histogram\")\nfig.show()\n\nprint(\"\\n\\n... SAMPLE DATE VALUE COUNTS ...\")\nfor k,v in train_df.sample_date.value_counts().items():\n print(f\"\\t--> There are {v} images with SAMPLE_DATE={k}\")\nfig = px.histogram(train_df, train_df.sample_date.apply(lambda x: x.replace(\"-\", \"_\")), color=\"cell_type\", title=\"Sample Date Value Histogram\")\nfig.show()\n\nprint(\"\\n\\n... ELAPSED TIME DELTA VALUE COUNTS ...\")\nfor k,v in train_df.elapsed_timedelta.value_counts().items():\n print(f\"\\t--> There are {v} images with SAMPLE_DATE={k}\")\nfig = px.histogram(train_df, \"elapsed_timedelta\", color=\"cell_type\", title=\"Elapsed Time Delta Value Histogram\")\nfig.show()\n \nprint(\"\\n\\n... SAMPLE ID VALUE COUNTS (>1) ...\")\nprint(f\"\\t--> There are {len(train_df[train_df.sample_id.isin([x for x,v in train_df.sample_id.value_counts().items() if v>1])])} SAMPLE_IDs with more than one image\\n\")\nfor k,v in train_df[train_df.sample_id.isin([x for x,v in train_df.sample_id.value_counts().items() if v>1])].reset_index()[\"sample_id\"].value_counts().items():\n print(f\"\\t--> There are {v} images with SAMPLE_ID={k}\")\nfig = px.histogram(train_df[train_df.sample_id.isin([x for x,v in train_df.sample_id.value_counts().items() if v>1])].reset_index(), \"sample_id\", color=\"cell_type\", title=\"Sample ID Value Histogram\")\nfig.show()\n\n \nprint(\"\\n\\n... CELL TYPE VALUE COUNTS ...\")\nfor k,v in train_df.cell_type.value_counts().items():\n print(f\"\\t--> There are {v} images with CELL_TYPE={k}\")\n \nfig = px.histogram(train_df, x=\"cell_type\", title=\"Cell Type Histogram\")\nfig.show()\n\nfor ct in CELL_TYPES:\n print(f\"\\n\\n... SHOWING THREE EXAMPLES OF CELL TYPE {ct.upper()} ...\\n\")\n for i in range(3):\n img, msk = get_img_and_mask(**train_df[train_df.cell_type==ct][[\"img_path\", \"annotation\", \"width\", \"height\"]].sample(3).reset_index(drop=True).iloc[i].to_dict())\n plot_img_and_mask(img, msk)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:23:27.356138Z","iopub.execute_input":"2021-11-06T20:23:27.356571Z","iopub.status.idle":"2021-11-06T20:23:34.237115Z","shell.execute_reply.started":"2021-11-06T20:23:27.356535Z","shell.execute_reply":"2021-11-06T20:23:34.236475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4.3 VISUALIZE & INVESTIGATE THE LIVECell DATA
\n\n---\n\nLet's review the LIVECell dataset\n","metadata":{}},{"cell_type":"code","source":"DEFER = True\n\nif not DEFER:\n LC_CELL_TYPES = os.listdir(os.path.join(LC_ANN_DIR, \"LIVECell_single_cells\"))\n\n print(\"\\n... LOADING TRAIN COCO JSON ...\\n\")\n LC_COCO_TRAIN = os.path.join(LC_ANN_DIR, \"LIVECell\", \"livecell_coco_train.json\")\n\n print(\"\\n... LOADING VALIDATION COCO JSON ...\\n\")\n LC_COCO_VAL = os.path.join(LC_ANN_DIR, \"LIVECell\", \"livecell_coco_val.json\")\n\n print(\"\\n... LOADING TEST COCO JSON ...\\n\")\n LC_COCO_TEST = os.path.join(LC_ANN_DIR, \"LIVECell\", \"livecell_coco_test.json\")\n\n LC_SC_TRAIN = {\n lc_ct:os.path.join(LC_ANN_DIR, \"LIVECell_single_cells\", lc_ct, f\"livecell_{lc_ct}_train.json\") \\\n for lc_ct in LC_CELL_TYPES\n }\n LC_SC_VAL = {\n lc_ct:os.path.join(LC_ANN_DIR, \"LIVECell_single_cells\", lc_ct, f\"livecell_{lc_ct}_val.json\") \\\n for lc_ct in LC_CELL_TYPES\n }\n LC_SC_TEST = {\n lc_ct:os.path.join(LC_ANN_DIR, \"LIVECell_single_cells\", lc_ct, f\"livecell_{lc_ct}_test.json\") \\\n for lc_ct in LC_CELL_TYPES\n }\n\n print(LC_SC_TRAIN)\n print(LC_SC_VAL)\n print(LC_SC_TEST)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:23:34.238300Z","iopub.execute_input":"2021-11-06T20:23:34.238630Z","iopub.status.idle":"2021-11-06T20:23:34.247819Z","shell.execute_reply.started":"2021-11-06T20:23:34.238599Z","shell.execute_reply":"2021-11-06T20:23:34.247226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4.4 VISUALIZE & INVESTIGATE THE SEMI-SUPERVISED TRAIN DATA
\n\n---\n\nLet's review the semi-supervised training data\n","metadata":{}},{"cell_type":"code","source":"semi_df = pd.DataFrame()\n\nsemi_df[\"cell_type\"] = [x.split(\"[\", 1)[0] for x in tf.io.gfile.listdir(SEMI_DIR)]\nsemi_df[\"compound\"] = [x.split(\"]\", 1)[0].split(\"[\", 1)[-1] for x in tf.io.gfile.listdir(SEMI_DIR)]\nsemi_df[\"img_path\"] = tf.io.gfile.glob(os.path.join(SEMI_DIR, \"**\"))\n\nfig = px.histogram(semi_df, \"cell_type\", color=\"compound\")\nfig.show()\n\nfig = px.histogram(semi_df, \"compound\", color=\"cell_type\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:23:34.249491Z","iopub.execute_input":"2021-11-06T20:23:34.250066Z","iopub.status.idle":"2021-11-06T20:23:35.012206Z","shell.execute_reply.started":"2021-11-06T20:23:34.250025Z","shell.execute_reply":"2021-11-06T20:23:35.010783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20,26))\nfor i, img_path in zip(range(15), semi_df.img_path.to_list()):\n plt.subplot(5,3,i+1)\n plt.imshow((255-np.asarray(ImageEnhance.Contrast(Image.fromarray(tf_load_png(img_path).numpy())).enhance(16))), cmap=\"inferno\")\n plt.axis(False)\n plt.title(img_path.rsplit(\"/\", 1)[-1].rsplit(\".\", 1)[0], fontweight=\"bold\")\n \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:23:35.013522Z","iopub.execute_input":"2021-11-06T20:23:35.013997Z","iopub.status.idle":"2021-11-06T20:23:38.636376Z","shell.execute_reply.started":"2021-11-06T20:23:35.013955Z","shell.execute_reply":"2021-11-06T20:23:38.634859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4.5 VISUALIZE & INVESTIGATE INDIVIDUAL CELLS
\n\n---\n\nLet's review a single image and then look into the cells that make up that image\n","metadata":{}},{"cell_type":"code","source":"DEMO_IDX = 11\nimg, msk = get_img_and_mask(**train_df[[\"img_path\", \"annotation\", \"width\", \"height\"]].iloc[DEMO_IDX].to_dict())\nplot_img_and_mask(img, msk)\n\nplt.figure(figsize=(20, min(80, msk.max()//2)))\nfor i in range(1, msk.max()+1):\n plt.subplot(10,10,i)\n tl, br = get_contour_bbox(np.where(msk==i, 1, 0).astype(np.uint8))\n plt.imshow(np.asarray(ImageEnhance.Contrast(Image.fromarray(255-img.numpy())).enhance(16))[tl[1]:br[1], tl[0]:br[0]], cmap=\"magma\")\n plt.axis(False)\n plt.title(f\"{i}\", fontweight=\"bold\")\n if i==100:\n break\n \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:23:38.637964Z","iopub.execute_input":"2021-11-06T20:23:38.638262Z","iopub.status.idle":"2021-11-06T20:23:45.077285Z","shell.execute_reply.started":"2021-11-06T20:23:38.638222Z","shell.execute_reply":"2021-11-06T20:23:45.076639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4.6 ADD BBOXES TO DATAFRAME
\n\n---\n\nLet's iterate over the images and add the bounding box coordinates to the dataframe in order (matching the RLE)\n\n* We also add the 0-1 normalized version of the bboxes to the dataframe for later training purposes\n","metadata":{}},{"cell_type":"code","source":"# Takes about 1 minute\nprint(\"\\n... CREATE FULL SCALE BBOXES ...\\n\")\ntrain_df[\"bboxes\"] = train_df.parallel_apply(pd_get_bboxes, axis=1)\ndisplay(train_df.head())\n\nprint(\"\\n... CREATE SCALED DOWN (0-1) BBOXES ...\\n\")\nIMG_O_W, IMG_O_H = train_df.iloc[0].width, train_df.iloc[0].height\ntrain_df[\"scaled_bboxes\"] = train_df.bboxes.progress_apply(lambda box_list: [((box[0][0]/IMG_O_W, box[0][1]/IMG_O_H), (box[1][0]/IMG_O_W,box[1][1]/IMG_O_H)) if box else None for box in box_list])\n\n# CORT\nimg, msk = get_img_and_mask(**train_df[[\"img_path\", \"annotation\", \"width\", \"height\"]].iloc[FIRST_SHSY5Y_IDX].to_dict())\nplot_img_and_mask(img, msk, bboxes=train_df.iloc[FIRST_SHSY5Y_IDX].bboxes)\n\n# ASTRO\nimg, msk = get_img_and_mask(**train_df[[\"img_path\", \"annotation\", \"width\", \"height\"]].iloc[FIRST_ASTRO_IDX].to_dict())\nplot_img_and_mask(img, msk, bboxes=train_df.iloc[FIRST_ASTRO_IDX].bboxes)\n\n# SHSY5Y\nimg, msk = get_img_and_mask(**train_df[[\"img_path\", \"annotation\", \"width\", \"height\"]].iloc[FIRST_CORT_IDX].to_dict())\nplot_img_and_mask(img, msk, bboxes=train_df.iloc[FIRST_CORT_IDX].bboxes)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:23:45.078709Z","iopub.execute_input":"2021-11-06T20:23:45.079201Z","iopub.status.idle":"2021-11-06T20:25:08.311881Z","shell.execute_reply.started":"2021-11-06T20:23:45.079165Z","shell.execute_reply":"2021-11-06T20:25:08.311230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4.7 STATS REGARDING BOUNDING BOXES
\n\n---","metadata":{}},{"cell_type":"code","source":"train_df[\"bbox_widths\"] = train_df.bboxes.apply(lambda x: get_bbox_stats(x, style=\"width\"))\ntrain_df[\"bbox_heights\"] = train_df.bboxes.apply(lambda x: get_bbox_stats(x, style=\"height\"))\ntrain_df[\"bbox_areas\"] = train_df.bboxes.apply(lambda x: get_bbox_stats(x, style=\"area\"))\n\ntrain_df[\"scaled_bbox_widths\"] = train_df.scaled_bboxes.apply(lambda x: get_bbox_stats(x, style=\"width\"))\ntrain_df[\"scaled_bbox_heights\"] = train_df.scaled_bboxes.apply(lambda x: get_bbox_stats(x, style=\"height\"))\ntrain_df[\"scaled_bbox_areas\"] = train_df.scaled_bboxes.apply(lambda x: get_bbox_stats(x, style=\"area\"))\n\ndisplay(train_df.head())\n\n# Plot\npx.scatter(train_df.sort_values(by=\"cell_type\", key=lambda x: x.map(ARB_SORT_MAP))[[\"cell_type\", \"bbox_widths\", \"bbox_heights\", \"bbox_areas\"]].explode(column=[\"bbox_widths\",\"bbox_heights\", \"bbox_areas\"]), x=\"bbox_widths\", y=\"bbox_heights\", color=\"cell_type\", title=\"Cell Bounding Box Sizes (WxH)\")","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:25:08.313040Z","iopub.execute_input":"2021-11-06T20:25:08.314068Z","iopub.status.idle":"2021-11-06T20:25:10.320368Z","shell.execute_reply.started":"2021-11-06T20:25:08.314028Z","shell.execute_reply":"2021-11-06T20:25:10.319717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n\n\n\n 5 MODELLING ⤒\n
\n\n---\n\nEfficientDET for Object Detection, Classification, and Segmentation","metadata":{}},{"cell_type":"markdown","source":"5.0 EFFICIENTDET UTILITY FUNCTIONS & CONSTANTS
\n\n---\n","metadata":{}},{"cell_type":"code","source":"IMAGE_SHAPE = (train_df.iloc[0].height, train_df.iloc[0].width, 3)\nINPUT_SHAPE = (640,640,3)\nSEG_SHAPE = (INPUT_SHAPE[0]//4, INPUT_SHAPE[1]//4, 1)\nMODEL_LEVEL = \"d1\"\nMODEL_NAME = f\"efficientdet-{MODEL_LEVEL}\"\nBATCH_SIZE = 8\nN_EVAL = 50\nN_TRAIN = len(train_df)-N_EVAL\nN_TEST = len(ss_df)\nDEBUG = N_TEST==3\nN_EPOCH = 40\nN_EX_PER_REC = 280\nCLASS_LABELS = list(train_df.cell_type.unique())\nN_CLASSES_OD = len(CLASS_LABELS)+1 # Background + 3 Cell Types\nN_CLASSES_SEG = 2 # Background + Foreground (Cells)\nMAX_N_INSTANCES = int(100*np.ceil(train_df.bboxes.apply(len).max()/100))\n\n# Whether or not we train from scratch or load\nDO_TRAIN=False\nPRETRAINED_MODEL_DIR=\"/kaggle/input/model-weights-40-epoch-efficientdet-d1-640\"\n\nprint(\"\\n ... HYPERPARAMETER CONSTANTS ...\")\nprint(f\"\\t--> MODEL NAME : {MODEL_NAME}\")\nprint(f\"\\t--> BATCH SIZE : {BATCH_SIZE}\")\nprint(f\"\\t--> IMAGE SHAPE : {IMAGE_SHAPE}\")\nprint(f\"\\t--> INPUT SHAPE : {INPUT_SHAPE}\")\nprint(f\"\\t--> SEGMENTATION SHAPE : {SEG_SHAPE}\")","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:25:10.321861Z","iopub.execute_input":"2021-11-06T20:25:10.322372Z","iopub.status.idle":"2021-11-06T20:25:10.336945Z","shell.execute_reply.started":"2021-11-06T20:25:10.322333Z","shell.execute_reply":"2021-11-06T20:25:10.336236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5.1 LOAD EFFICIENTDET MODEL AND INITIALIZE
\n\n---\n","metadata":{}},{"cell_type":"code","source":"config = hparams_config.get_efficientdet_config(MODEL_NAME)\nKEY_CONFIGS = [\n \"name\", \"image_size\", \"num_classes\", \"seg_num_classes\", \"heads\", \"train_file_pattern\",\n \"val_file_pattern\", \"model_name\", \"model_dir\", \"pretrained_ckpt\", \"batch_size\", \"eval_samples\",\n \"num_examples_per_epoch\", \"num_epochs\", \"steps_per_execution\", \"steps_per_epoch\", \n \"profile\", \"val_json_file\", \"max_instances_per_image\", \"mixed_precision\", \n \"learning_rate\", \"lr_warmup_init\", \"mean_rgb\", \"stddev_rgb\",\"scale_range\",\n ]\n\nfor k in config.keys():\n if k==\"model_optimizations\":\n continue\n elif k==\"nms_configs\":\n for _k, _v in dict(config[k]).items():\n print(f\"PARAMETER: {' ' if _k not in KEY_CONFIGS else ' *** '}nms_config_{_k: <16} ----> VALUE: {_v}\")\n \n else:\n print(f\"PARAMETER: {' ' if k not in KEY_CONFIGS else ' *** '}{k: <27} ----> VALUE: {config[k]}\")","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:25:10.338512Z","iopub.execute_input":"2021-11-06T20:25:10.339001Z","iopub.status.idle":"2021-11-06T20:25:10.360922Z","shell.execute_reply.started":"2021-11-06T20:25:10.338966Z","shell.execute_reply":"2021-11-06T20:25:10.360246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DO_ADV_PROP=True\nMODEL_DIR = f\"/kaggle/working/{MODEL_NAME}-finetune\"\n\nif TPU:\n TFRECORD_DIR = os.path.join(KaggleDatasets().get_gcs_path('effdet-d5-dataset-sartorius'), \"tfrecords\")\nelse:\n TFRECORD_DIR = \"/kaggle/working/tfrecords\"\n\nos.makedirs(MODEL_DIR, exist_ok=True)\nconfig = hparams_config.get_efficientdet_config(MODEL_NAME)\noverrides = dict(\n train_file_pattern=os.path.join(TFRECORD_DIR, \"train\", \"*.tfrec\"),\n val_file_pattern=os.path.join(TFRECORD_DIR, \"val\", \"*.tfrec\"),\n test_file_pattern=os.path.join(TFRECORD_DIR, \"test\", \"*.tfrec\"),\n model_name=MODEL_NAME,\n model_dir=MODEL_DIR,\n pretrained_ckpt=MODEL_NAME,\n batch_size=BATCH_SIZE,\n eval_samples=N_EVAL,\n num_examples_per_epoch=N_TRAIN,\n num_epochs=N_EPOCH,\n steps_per_execution=1,\n steps_per_epoch=N_TRAIN//BATCH_SIZE,\n profile=None, val_json_file=None,\n heads = ['object_detection', 'segmentation'],\n image_size = INPUT_SHAPE[:-1],\n num_classes = N_CLASSES_OD,\n seg_num_classes = N_CLASSES_SEG,\n max_instances_per_image = MAX_N_INSTANCES,\n input_rand_hflip=False, jitter_min=0.99, jitter_max=1.01,\n skip_crowd_during_training=False,\n )\nconfig.override(overrides, True)\nconfig.nms_configs.max_output_size = MAX_N_INSTANCES\n\n# Change how input preprocessing is done\nif DO_ADV_PROP:\n config.override(dict(mean_rgb=0.0, stddev_rgb=1.0, scale_range=True), True)\n\n\ntf.keras.backend.clear_session()\n\nmodel = efficientdet_keras.EfficientDetModel(config=config)\nmodel.build((1,*INPUT_SHAPE))\n\nprint(\"\\n... MODEL PREDICTIONS ...\\n\")\npreds = model.predict(np.zeros((1,*INPUT_SHAPE)))\nfor i, name in enumerate([\"bboxes\", \"confidences\", \"classes\", \"valid_len\", \"segmentation map\"]):\n print(name)\n print(preds[i].shape)\n try:\n if preds[i].shape[-2]==64:\n print(preds[i][0, 0, 0, :5])\n else:\n print(preds[i][0, :5])\n \n except:\n print(preds[i][0])\n print()","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:25:10.362181Z","iopub.execute_input":"2021-11-06T20:25:10.362422Z","iopub.status.idle":"2021-11-06T20:25:53.007435Z","shell.execute_reply.started":"2021-11-06T20:25:10.362390Z","shell.execute_reply":"2021-11-06T20:25:53.006431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5.2 CREATE A DATASET WITH THE CORRECT STRUCTURE
\n\n---\n\n* **INPUT**\n * Raw Image (256x256x3)\n\n\n* **OUTPUT/TARGET**\n * Bounding Boxes\n * Instance Classes\n * Segmented Image (64x64x3)\n \n---\n\nWe first create the tfrecord datasets... then we instantiate a dataloader ","metadata":{}},{"cell_type":"code","source":"def create_id_to_iloc_map(df):\n \"\"\"\n Create mapping to allow for numeric file-names\n --> index in original df --> id\n \"\"\"\n return {v:k for k,v in df.id.to_dict().items()}\n\nTRAIN_ID_2_ILOC = create_id_to_iloc_map(train_df)\nTEST_ID_2_ILOC = create_id_to_iloc_map(ss_df)\n\n\ndef tf_load_image(path, resize_to=INPUT_SHAPE):\n \"\"\" Load an image with the correct shape using only TF\n \n Args:\n path (tf.string): Path to the image to be loaded\n resize_to (tuple, optional): Size to reshape image\n \n Returns:\n 3 channel tf.Constant image ready for training/inference\n \n \"\"\"\n \n img_bytes = tf.io.read_file(path)\n img = tf.image.decode_png(img_bytes, channels=resize_to[-1])\n img = tf.image.resize(img, resize_to[:-1])\n img = tf.cast(img, tf.uint8)\n \n return img\n\ndef load_npz(path, resize_to=SEG_SHAPE, to_binary=True):\n np_arr = np.load(path)[\"arr_0\"]\n if to_binary:\n return np.where(cv2.resize(np_arr, resize_to[:-1])>0, 1, 0).reshape(resize_to).astype(np.uint8)\n else:\n return cv2.resize(np_arr, resize_to[:-1]).reshape(resize_to).astype(np.int32)\n\ndef image_preprocess(image, image_size, mean_rgb=config.mean_rgb, stddev_rgb=config.stddev_rgb):\n \"\"\"Preprocess image for inference.\n Args:\n image: input image, can be a tensor or a numpy arary.\n image_size: single integer of image size for square image or tuple of two\n integers, in the format of (image_height, image_width).\n mean_rgb: Mean value of RGB, can be a list of float or a float value.\n stddev_rgb: Standard deviation of RGB, can be a list of float or a float\n value.\n Returns:\n (image, scale): a tuple of processed image and its scale.\n \"\"\"\n input_processor = dataloader.DetectionInputProcessor(image, image_size)\n input_processor.normalize_image(mean_rgb, stddev_rgb)\n input_processor.set_scale_factors_to_output_size()\n image = input_processor.resize_and_crop_image()\n image_scale = input_processor.image_scale_to_original\n return image, image_scale\n\n\ndef _bytes_feature(value, is_list=False):\n \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n if isinstance(value, type(tf.constant(0))):\n value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n \n if not is_list:\n value = [value]\n \n return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))\n\ndef _float_feature(value, is_list=False):\n \"\"\"Returns a float_list from a float / double.\"\"\"\n \n if not is_list:\n value = [value]\n \n return tf.train.Feature(float_list=tf.train.FloatList(value=value))\n\ndef _int64_feature(value, is_list=False):\n \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n \n if not is_list:\n value = [value]\n \n return tf.train.Feature(int64_list=tf.train.Int64List(value=value))\n\ndef serialize_raw(example_data):\n \"\"\"\n Creates a tf.Example message ready to be written to a file from 4 features.\n\n Args:\n example_data: Everything from pandas row\n style (str, optional): Which subset to do... [train|val]\n [test] will be processed through a different function\n \n Returns:\n A tf.Example Message ready to be written to file\n \"\"\"\n \n image_object_mask = tf.io.encode_png(load_npz(example_data[\"seg_path\"]))\n \n image_height = INPUT_SHAPE[0]\n image_width = INPUT_SHAPE[1]\n image_source_id = image_filename = f\"{TRAIN_ID_2_ILOC[example_data['id']]:>05}\".encode('utf8')\n \n image_encoded = tf.io.encode_png(tf_load_image(example_data[\"img_path\"]))\n image_key_sha256 = hashlib.sha256(image_encoded).hexdigest().encode('utf8')\n image_format = example_data[\"img_path\"][-4:].encode('utf8') #png\n \n image_object_bbox_xmins, image_object_bbox_xmaxs = [], []\n image_object_bbox_ymins, image_object_bbox_ymaxs = [], []\n image_object_class_text, image_object_class_label = [], []\n image_object_is_crowd, image_object_area = [], []\n for i, box in enumerate(example_data[\"scaled_bboxes\"]):\n if box and example_data[\"bbox_areas\"][i]>0.0:\n image_object_bbox_xmins.append(box[0][0])\n image_object_bbox_xmaxs.append(box[1][0])\n image_object_bbox_ymins.append(box[0][1])\n image_object_bbox_ymaxs.append(box[1][1])\n image_object_class_text.append(example_data[\"cell_type\"].encode('utf8'))\n image_object_class_label.append(ARB_SORT_MAP[example_data[\"cell_type\"]])\n image_object_is_crowd.append(0)\n image_object_area.append(example_data[\"scaled_bbox_areas\"][i])\n \n # Create a dictionary mapping the feature name to the \n # tf.Example-compatible data type.\n feature_dict = {\n 'image/height': _int64_feature(image_height),\n 'image/width': _int64_feature(image_width),\n 'image/filename': _bytes_feature(image_filename),\n 'image/source_id': _bytes_feature(image_source_id),\n 'image/key/sha256': _bytes_feature(image_key_sha256),\n 'image/encoded': _bytes_feature(image_encoded),\n 'image/format': _bytes_feature(image_format),\n 'image/object/bbox/xmin': _float_feature(image_object_bbox_xmins, is_list=True),\n 'image/object/bbox/xmax': _float_feature(image_object_bbox_xmaxs, is_list=True),\n 'image/object/bbox/ymin': _float_feature(image_object_bbox_ymins, is_list=True),\n 'image/object/bbox/ymax': _float_feature(image_object_bbox_ymaxs, is_list=True),\n 'image/object/class/text': _bytes_feature(image_object_class_text, is_list=True),\n 'image/object/class/label': _int64_feature(image_object_class_label, is_list=True),\n 'image/object/is_crowd': _int64_feature(image_object_is_crowd, is_list=True),\n 'image/object/area': _float_feature(image_object_area, is_list=True),\n 'image/object/mask': _bytes_feature(image_object_mask),\n }\n \n # Create a Features message using tf.train.Example.\n example_proto = tf.train.Example(features=tf.train.Features(feature=feature_dict))\n return example_proto.SerializeToString()\n\n\ndef serialize_test_raw(example_data):\n \"\"\"\n Creates a tf.Example message ready to be written to a file\n\n Args:\n example_data: Everything from pandas row\n \n Returns:\n A tf.Example Message ready to be written to file\n \"\"\"\n \n image_height = INPUT_SHAPE[0]\n image_width = INPUT_SHAPE[1]\n image_source_id = image_filename = f\"{TEST_ID_2_ILOC[example_data['id']]:>05}\".encode('utf8')\n \n image_encoded = tf.io.encode_png(tf_load_image(example_data[\"img_path\"]))\n image_key_sha256 = hashlib.sha256(image_encoded).hexdigest().encode('utf8')\n image_format = example_data[\"img_path\"][-4:].encode('utf8') #png\n \n # Create a dictionary mapping the feature name to the \n # tf.Example-compatible data type.\n feature_dict = {\n 'image/height': _int64_feature(image_height),\n 'image/width': _int64_feature(image_width),\n 'image/filename': _bytes_feature(image_filename),\n 'image/source_id': _bytes_feature(image_source_id),\n 'image/key/sha256': _bytes_feature(image_key_sha256),\n 'image/encoded': _bytes_feature(image_encoded),\n 'image/format': _bytes_feature(image_format),\n }\n \n # Create a Features message using tf.train.Example.\n example_proto = tf.train.Example(features=tf.train.Features(feature=feature_dict))\n return example_proto.SerializeToString()\n\n\ndef write_tfrecords(df, n_ex, n_ex_per_rec=50, serialize_fn=serialize_raw, out_dir=\"/kaggle/working/tfrecords\", ds_type=\"train\"):\n \"\"\"\"\"\"\n n_recs = int(np.ceil(n_ex/n_ex_per_rec))\n \n # Make dataframe iterable\n iter_df = df.iterrows()\n \n out_dir = os.path.join(out_dir, ds_type)\n # Create folder\n if not os.path.isdir(out_dir):\n os.makedirs(out_dir, exist_ok=True)\n \n # Create tfrecords\n for i in tqdm(range(n_recs), total=n_recs):\n print(f\"\\n... Writing {ds_type.title()} TFRecord {i+1} of {n_recs} ...\\n\")\n tfrec_path = os.path.join(out_dir, f\"{ds_type}__{(i+1):02}_{n_recs:02}.tfrec\")\n \n # This makes the tfrecord\n with tf.io.TFRecordWriter(tfrec_path) as writer:\n for ex in tqdm(range(n_ex_per_rec), total=n_ex_per_rec):\n try:\n example = serialize_fn(next(iter_df)[1])\n writer.write(example)\n except:\n break\n\n# TRAIN\nwrite_tfrecords(train_df.iloc[:-N_EVAL], N_TRAIN, n_ex_per_rec=N_EX_PER_REC, serialize_fn=serialize_raw, out_dir=TFRECORD_DIR, ds_type=\"train\")\n \n# VAL\nwrite_tfrecords(train_df[-N_EVAL:], N_EVAL, n_ex_per_rec=N_EX_PER_REC, serialize_fn=serialize_raw, out_dir=TFRECORD_DIR, ds_type=\"val\")\n\n# VAL\nwrite_tfrecords(ss_df, N_TEST, n_ex_per_rec=N_EX_PER_REC, serialize_fn=serialize_test_raw, out_dir=TFRECORD_DIR, ds_type=\"test\")","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:25:53.008999Z","iopub.execute_input":"2021-11-06T20:25:53.009400Z","iopub.status.idle":"2021-11-06T20:27:35.392388Z","shell.execute_reply.started":"2021-11-06T20:25:53.009359Z","shell.execute_reply":"2021-11-06T20:27:35.391688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5.3 INSTANIATE OUR DATALOADER
\n\n---\n\nAugmentations are breaking the masks... so disablled for now","metadata":{}},{"cell_type":"code","source":"train_dl = dataloader.InputReader(\n file_pattern=config.train_file_pattern,\n is_training=\"train\" in config.train_file_pattern,\n max_instances_per_image=config.max_instances_per_image\n)(config.as_dict())\n\nval_dl = dataloader.InputReader(\n file_pattern=config.val_file_pattern,\n is_training=\"train\" in config.val_file_pattern,\n max_instances_per_image=config.max_instances_per_image\n)(config.as_dict())\n\ntest_dl = dataloader.InputReader(\n file_pattern=config.test_file_pattern,\n is_training=\"train\" in config.test_file_pattern,\n max_instances_per_image=config.max_instances_per_image\n)(config.as_dict(), batch_size=1)\n\n\nprint(\"\\n... TRAIN DATALOADER ...\\n\")\nprint(train_dl)\n\nprint(\"\\n\\n... VALIDATION DATALOADER ...\\n\")\nprint(val_dl)\n\nprint(\"\\n\\n... TEST DATALOADER ...\\n\")\nprint(test_dl)\n\nprint(\"\\n\\n\\n\\n LETS SEE AN EXAMPLE FROM OUR TRAIN DATALOADER ...\\n\\n\")\n\nx = next(iter(train_dl))\n\nprint(int(x[1][\"source_ids\"][0]))\nimg, msk = get_img_and_mask(**train_df[[\"img_path\", \"annotation\", \"width\", \"height\"]].iloc[int(x[1][\"source_ids\"][0])].to_dict(), )\nplot_img_and_mask(img, msk)\n\nplt.figure(figsize=(20,10))\n\nplt.subplot(1,3,1)\nplt.imshow(x[0][0])\nplt.axis(False)\nplt.title(\"Cell Image\", fontweight=\"bold\")\n\nplt.subplot(1,3,2)\nplt.imshow(x[1][\"image_masks\"][0][0])\nplt.axis(False)\nplt.title(\"Segmentation Mask Overlay\", fontweight=\"bold\")\n\nmerged = cv2.addWeighted(np.array(x[0][0]), 0.75, np.clip(cv2.resize(np.tile(np.expand_dims(x[1][\"image_masks\"][0][0], axis=-1), 3), INPUT_SHAPE[:-1]), 0, 1)*255, 0.25, 0.0,)\nplt.subplot(1,3,3)\nplt.imshow(merged)\nplt.axis(False)\nplt.title(\"Cell Image w/ Instance Segmentation Mask Overlay\", fontweight=\"bold\")\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:27:35.393808Z","iopub.execute_input":"2021-11-06T20:27:35.394211Z","iopub.status.idle":"2021-11-06T20:27:42.484409Z","shell.execute_reply.started":"2021-11-06T20:27:35.394172Z","shell.execute_reply":"2021-11-06T20:27:42.483622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5.4 CREATE MODEL AND LOAD PRETRAINED WEIGHTS
\n\n---\n\nCOCO weights","metadata":{}},{"cell_type":"code","source":"if DO_TRAIN:\n if not os.path.isdir(MODEL_NAME):\n if DO_ADV_PROP:\n !wget https://storage.googleapis.com/cloud-tpu-checkpoints/efficientdet/advprop/{MODEL_NAME}.tar.gz\n else:\n !wget https://storage.googleapis.com/cloud-tpu-checkpoints/efficientdet/coco2/{MODEL_NAME}.tar.gz\n !tar -zxf {MODEL_NAME}.tar.gz\n !rm -rf {MODEL_NAME}.tar.gz\n \nwith strategy.scope():\n model = train_lib.EfficientDetNetTrain(config=config)\n model = setup_model(model, config)\n \n if DO_TRAIN:\n util_keras.restore_ckpt(\n model=model,\n ckpt_path_or_file=tf.train.latest_checkpoint(MODEL_NAME),\n ema_decay=config.moving_average_decay,\n exclude_layers=['class_net']\n )\n ckpt_cb = tf.keras.callbacks.ModelCheckpoint(\n os.path.join(MODEL_DIR, 'ckpt-{epoch:d}'),\n verbose=1, save_freq=\"epoch\", save_weights_only=True)\n else:\n model.load_weights(os.path.join(PRETRAINED_MODEL_DIR, \"ckpt\"))\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:27:42.485819Z","iopub.execute_input":"2021-11-06T20:27:42.486244Z","iopub.status.idle":"2021-11-06T20:27:51.244941Z","shell.execute_reply.started":"2021-11-06T20:27:42.486209Z","shell.execute_reply":"2021-11-06T20:27:51.244222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5.5 TRAIN THE MODEL
\n\n---\n\nSince I trained the model in a previous version I will be loading the trained model from that previously trained checkpoint. Please see a previous version of this notebook ([**Version 33**](https://www.kaggle.com/dschettler8845/train-sartorius-segmentation-eda-effdet-tf?scriptVersionId=78096161)) to see the training run.","metadata":{}},{"cell_type":"code","source":"if DO_TRAIN:\n history = model.fit(\n train_dl,\n epochs=config.num_epochs,\n steps_per_epoch=config.steps_per_epoch,\n callbacks=[ckpt_cb,],\n validation_data=val_dl,\n validation_steps=N_EVAL//BATCH_SIZE\n )\nelse:\n print(model.evaluate(train_dl, steps=config.steps_per_epoch))\n print(model.evaluate(val_dl, steps=N_EVAL//BATCH_SIZE))","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:27:51.246426Z","iopub.execute_input":"2021-11-06T20:27:51.246683Z","iopub.status.idle":"2021-11-06T20:31:32.452338Z","shell.execute_reply.started":"2021-11-06T20:27:51.246648Z","shell.execute_reply":"2021-11-06T20:31:32.451468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5.6 VALIDATE THE MODEL IS LEARNING
\n\n---\n","metadata":{}},{"cell_type":"code","source":"def plot_gt(_image, _gt_classes, _gt_boxes, _gt_mask):\n img_class = int(_gt_classes.numpy()[0])\n img_boxes = _gt_boxes.numpy().astype(np.int32)[np.where(_gt_classes!=-1)[0]] \n _image = _image.numpy()\n _gt_dummy_mask = np.zeros_like(_image)\n _gt_dummy_mask[..., img_class] = cv2.resize(np.expand_dims(_gt_mask, axis=-1), INPUT_SHAPE[:-1])\n _gt_mask = _gt_dummy_mask\n \n \n plt.figure(figsize=(20,7))\n \n plt.subplot(1,3,1)\n plt.imshow(_image, cmap=\"inferno\")\n plt.axis(False)\n plt.title(\"Original Image After Preprocessing\", fontweight=\"bold\")\n \n mask_merged = cv2.addWeighted(_image, 0.55, _gt_mask, 1.25, 0.0)\n plt.subplot(1,3,2)\n plt.imshow(mask_merged)\n plt.axis(False)\n plt.title(f\"Original Image Mask (CLASS={img_class})\", fontweight=\"bold\")\n \n plt.subplot(1,3,3)\n box_image = np.zeros_like(_image)\n for box in img_boxes:\n ymin, xmin, ymax, xmax = box\n box_image = cv2.rectangle(img=box_image, thickness=1, pt1=(xmin, ymin), pt2=(xmax, ymax), \n color=[0 if i!=img_class else 255 for i in range(3)])\n \n box_merged = cv2.addWeighted(_image, 0.55, box_image, 1.25 if img_class==2 else 0.45, 0.0,)\n plt.imshow(box_merged)\n plt.axis(False)\n plt.title(f\"Original Image Bounding Boxes (CLASS={img_class})\", fontweight=\"bold\")\n\n plt.tight_layout()\n plt.show()\n \ndef plot_pred(_image, _pred_boxes, _pred_scores, _pred_classes, _pred_mask, conf_thresh=0.25, iou_thresh=0.0001):\n \"\"\"\"\"\"\n \n if iou_thresh is not None:\n _indices, _pred_scores = tf.image.non_max_suppression_with_scores(\n _pred_boxes, _pred_scores, 800, iou_threshold=iou_thresh,\n score_threshold=conf_thresh/5, soft_nms_sigma=0.0\n )\n _pred_boxes = tf.gather(_pred_boxes, _indices)\n\n \n above_thresh_idx = np.where(_pred_scores.numpy()>conf_thresh)[0]\n if len(above_thresh_idx)==0:\n print(\"\\n... NO PREDS OVER CONF THRESH... SAMPLING UP-TO FIFTY SAMPLES ...\\n\")\n above_thresh_idx = np.arange(min(50, len(_pred_scores)))\n\n _image = _image.numpy()\n _pred_class = int(np.round(_pred_classes.numpy()[above_thresh_idx].mean()))\n\n _pred_scores = _pred_scores.numpy()[above_thresh_idx]\n _pred_boxes = _pred_boxes.numpy().astype(np.int32)[above_thresh_idx]\n _pred_mask = np.where(_pred_mask[..., 1]>_pred_mask[..., 0], 1.0, 0.0)\n _dummy_mask = np.zeros_like(_image)\n _dummy_mask[..., _pred_class] = cv2.resize(np.expand_dims(_pred_mask, axis=-1), INPUT_SHAPE[:-1])\n _pred_mask = _dummy_mask\n \n \n plt.figure(figsize=(20,7))\n \n plt.subplot(1,3,1)\n plt.imshow(_image, cmap=\"inferno\")\n plt.axis(False)\n plt.title(\"Original Image After Preprocessing\", fontweight=\"bold\")\n \n mask_merged = cv2.addWeighted(_image, 0.55, _pred_mask, 1.25, 0.0,)\n plt.subplot(1,3,2)\n plt.imshow(mask_merged)\n plt.axis(False)\n plt.title(f\"Predicted Image Mask (CLASS={_pred_class})\", fontweight=\"bold\")\n \n plt.subplot(1,3,3)\n box_image = np.zeros_like(_image)\n for box in _pred_boxes:\n ymin, xmin, ymax, xmax = box\n box_image = cv2.rectangle(img=box_image, thickness=1, pt1=(xmin, ymin), pt2=(xmax, ymax), \n color=[0 if i!=_pred_class else 255 for i in range(3)])\n \n box_merged = cv2.addWeighted(_image, 0.55, box_image, 1.25 if _pred_class==2 else 0.45, 0.0,)\n plt.imshow(box_merged)\n plt.axis(False)\n plt.title(f\"Predicted Image Bounding Boxes (CLASS={_pred_class})\", fontweight=\"bold\")\n\n plt.tight_layout()\n plt.show()\n\ndef plot_diff(_image, _gt_classes, _gt_boxes, _gt_mask, _pred_boxes, _pred_scores, _pred_classes, _pred_mask, conf_thresh=0.25, iou_thresh=0.0001):\n \"\"\"\"\"\"\n \n if iou_thresh is not None:\n _indices, _pred_scores = tf.image.non_max_suppression_with_scores(\n _pred_boxes, _pred_scores, 800, iou_threshold=iou_thresh,\n score_threshold=conf_thresh/5, soft_nms_sigma=0.0\n )\n _pred_boxes = tf.gather(_pred_boxes, _indices)\n \n _image = _image.numpy()\n \n above_thresh_idx = np.where(_pred_scores.numpy()>conf_thresh)[0]\n gt_idxs = np.where(_gt_classes!=-1)[0]\n \n if len(above_thresh_idx)==0:\n print(\"\\n... NO PREDS OVER CONF THRESH... SAMPLING UP-TO FIFTY SAMPLES ...\\n\")\n above_thresh_idx = np.arange(min(50, len(_pred_scores)))\n \n _img_class = int(_gt_classes.numpy()[0])\n _pred_class = int(np.round(_pred_classes.numpy()[above_thresh_idx].mean()))\n \n img_boxes = _gt_boxes.numpy().astype(np.int32)[gt_idxs]\n _pred_boxes = _pred_boxes.numpy().astype(np.int32)[above_thresh_idx]\n \n _pred_scores = _pred_scores.numpy()[above_thresh_idx]\n \n _combo_mask = np.zeros_like(_image)\n _combo_mask[..., 0] = cv2.resize(np.expand_dims(_gt_mask, axis=-1), INPUT_SHAPE[:-1]) \n _pred_mask = np.where(_pred_mask[..., -1]>_pred_mask[..., 0], 1.0, 0.0)\n _combo_mask[..., 1] = cv2.resize(np.expand_dims(_pred_mask, axis=-1), INPUT_SHAPE[:-1])\n \n plt.figure(figsize=(20,7))\n \n plt.subplot(1,3,1)\n plt.imshow(_image, cmap=\"inferno\")\n plt.axis(False)\n plt.title(\"Original Image After Preprocessing\", fontweight=\"bold\")\n \n mask_merged = cv2.addWeighted(_image, 0.55, _combo_mask, 1.25, 0.0,)\n plt.subplot(1,3,2)\n plt.imshow(mask_merged)\n plt.axis(False)\n plt.title(f\"Combo Image Mask\\n(RED=GT, GREEN=PRED, YELLOW=CONSENSUS)\", fontweight=\"bold\")\n \n plt.subplot(1,3,3)\n box_image = np.zeros_like(_image)\n for box in img_boxes:\n ymin, xmin, ymax, xmax = box\n box_image = cv2.rectangle(img=box_image, thickness=1, pt1=(xmin, ymin), pt2=(xmax, ymax), \n color=(255,0,0))\n for box in _pred_boxes:\n ymin, xmin, ymax, xmax = box\n box_image = cv2.rectangle(img=box_image, thickness=1, pt1=(xmin, ymin), pt2=(xmax, ymax), \n color=(0,255,0))\n \n box_merged = cv2.addWeighted(_image, 0.55, box_image, 1.25, 0.0)\n plt.imshow(box_merged)\n plt.axis(False)\n plt.title(f\"Predicted Image Bounding Boxes\\n(RED=GT, GREEN=PRED)\", fontweight=\"bold\")\n\n plt.tight_layout()\n plt.show()\n \n \ndef compute_iou(labels, y_pred):\n \"\"\"\n Computes the IoU for instance labels and predictions.\n\n Args:\n labels (np array): Labels.\n y_pred (np array): predictions\n\n Returns:\n np array: IoU matrix, of size true_objects x pred_objects.\n \"\"\"\n\n true_objects = len(np.unique(labels))\n pred_objects = len(np.unique(y_pred))\n\n # Compute intersection between all objects\n intersection = np.histogram2d(\n labels.flatten(), y_pred.flatten(), bins=(true_objects, pred_objects)\n )[0]\n\n # Compute areas (needed for finding the union between all objects)\n area_true = np.histogram(labels, bins=true_objects)[0]\n area_pred = np.histogram(y_pred, bins=pred_objects)[0]\n area_true = np.expand_dims(area_true, -1)\n area_pred = np.expand_dims(area_pred, 0)\n\n # Compute union\n union = area_true + area_pred - intersection\n iou = intersection / union\n \n return iou[1:, 1:] # exclude background\n\n\ndef precision_at(threshold, iou):\n \"\"\"\n Computes the precision at a given threshold.\n\n Args:\n threshold (float): Threshold.\n iou (np array): IoU matrix.\n\n Returns:\n int: Number of true positives,\n int: Number of false positives,\n int: Number of false negatives.\n \"\"\"\n matches = iou > threshold\n true_positives = np.sum(matches, axis=1) == 1 # Correct objects\n false_positives = np.sum(matches, axis=0) == 0 # Missed objects\n false_negatives = np.sum(matches, axis=1) == 0 # Extra objects\n tp, fp, fn = (\n np.sum(true_positives),\n np.sum(false_positives),\n np.sum(false_negatives),\n )\n return tp, fp, fn\n\n\ndef iou_map(truths, preds, verbose=1):\n \"\"\"\n Computes the metric for the competition.\n Masks contain the segmented pixels where each object has one value associated,\n and 0 is the background.\n\n Args:\n truths (list of masks): Ground truths.\n preds (list of masks): Predictions.\n verbose (int, optional): Whether to print infos. Defaults to 0.\n\n Returns:\n float: mAP.\n \"\"\"\n ious = [compute_iou(truth, pred) for truth, pred in zip(truths, preds)]\n\n if verbose:\n print(\"Thresh\\tTP\\tFP\\tFN\\tPrec.\")\n\n prec = []\n for t in np.arange(0.5, 1.0, 0.05):\n tps, fps, fns = 0, 0, 0\n for iou in ious:\n tp, fp, fn = precision_at(t, iou)\n tps += tp\n fps += fp\n fns += fn\n\n p = tps / (tps + fps + fns)\n prec.append(p)\n\n if verbose:\n print(\"{:1.3f}\\t{}\\t{}\\t{}\\t{:1.3f}\".format(t, tps, fps, fns, p))\n\n if verbose:\n print(\"AP\\t-\\t-\\t-\\t{:1.3f}\".format(np.mean(prec)))\n\n return np.mean(prec)\n\n\ndef get_pred_instance_mask(_pred_boxes, _pred_scores, _pred_mask, iou_thresh=0.0, conf_thresh=0.25):\n _indices, _pred_scores = tf.image.non_max_suppression_with_scores(\n _pred_boxes, _pred_scores, 800, iou_threshold=iou_thresh,\n score_threshold=conf_thresh/5, soft_nms_sigma=0.0\n )\n _pred_boxes = tf.gather(_pred_boxes, _indices)\n \n above_thresh_idx = np.where(_pred_scores.numpy()>conf_thresh)[0]\n if len(above_thresh_idx)==0:\n above_thresh_idx = np.arange(min(50, len(_pred_scores)))\n\n _pred_scores = _pred_scores.numpy()[above_thresh_idx]\n _pred_boxes = _pred_boxes.numpy().astype(np.int32)[above_thresh_idx]\n _pred_mask = cv2.resize(_pred_mask, INPUT_SHAPE[:-1], interpolation=cv2.INTER_NEAREST)\n _pred_mask = np.where(_pred_mask[..., 1]>_pred_mask[..., 0], 1.0, 0.0)\n _instance_mask = np.zeros_like(_pred_mask)\n for i, _box in enumerate(_pred_boxes):\n _instance_mask[_box[0]:_box[2], _box[1]:_box[3]] = (i+1)*_pred_mask[_box[0]:_box[2], _box[1]:_box[3]]\n _instance_mask = cv2.resize(_instance_mask, IMAGE_SHAPE[-2::-1], interpolation=cv2.INTER_NEAREST)\n return _instance_mask","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:31:32.454164Z","iopub.execute_input":"2021-11-06T20:31:32.454439Z","iopub.status.idle":"2021-11-06T20:31:32.506086Z","shell.execute_reply.started":"2021-11-06T20:31:32.454402Z","shell.execute_reply":"2021-11-06T20:31:32.505330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5.7 IOU ON VALIDATION DATASET
\n\n---\n","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5.8 CONFUSION MATRIX FOR VALIDATION DATASET
\n\n---\n","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**TRAINING DATA FIRST BATCH**","metadata":{}},{"cell_type":"code","source":"# We want... tbd --> wrap in fn\n# GT:\n# - Bounding Boxes\n# - Confidence Scores\n# - Segmentation Mask\n# PRED:\n# - Bounding Boxes\n# - Confidence Scores\n# - Instance Classes\n# - Segmentation Mask\nfor _image_batch, _label_batch in train_dl.take(1):\n gt_mask = _label_batch[\"image_masks\"][:, 0]\n gt_boxes = _label_batch[\"groundtruth_data\"][..., :4]\n gt_is_crowds = _label_batch[\"groundtruth_data\"][..., 4]\n gt_areas = _label_batch[\"groundtruth_data\"][..., 5]\n gt_classes = _label_batch[\"groundtruth_data\"][..., 6]\n \n pred_classes, pred_boxes, pred_mask = model(_image_batch, training=False)\n pred_boxes, pred_scores, pred_classes, valid_len = postprocess.postprocess_global(config, pred_classes, pred_boxes)\n gt_instance_masks, pred_instance_masks = [], []\n for i in range(BATCH_SIZE):\n print(\"\\n\\n... ORIGINAL DISPLAY PLOT ...\\n\")\n _img, _mask = get_img_and_mask(**train_df.iloc[int(_label_batch[\"source_ids\"][i])][[\"img_path\", \"annotation\", \"width\", \"height\"]])\n plot_img_and_mask(_img, _mask)\n gt_instance_masks.append(_mask)\n\n print(\"\\n... GROUND TRUTH PLOT ...\\n\")\n plot_gt(_image_batch[i], gt_classes[i], gt_boxes[i], gt_mask[i])\n\n print(f\"\\n... PREDICTION PLOT (NMS={'yes' if i<4 else 'no'}) ...\\n\")\n plot_pred(_image_batch[i], pred_boxes[i], pred_scores[i], pred_classes[i], pred_mask[i], iou_thresh=0.0 if i<4 else None)\n\n print(f\"\\n... GROUND TRUTH VS. PREDICTION PLOT (NMS={'yes' if i<4 else 'no'}) ...\\n\")\n plot_diff(_image_batch[i], gt_classes[i], gt_boxes[i], gt_mask[i], pred_boxes[i], pred_scores[i], pred_classes[i], pred_mask[i], iou_thresh=0.0 if i<4 else None)\n \n pred_instance_masks.append(get_pred_instance_mask(pred_boxes[i], pred_scores[i], pred_mask[i].numpy(), iou_thresh=0.0, conf_thresh=0.1))\n \n print(\"\\n\\n\\n\\n\")\n print(\"-\"*50)\n print(\"\\n\\n\")\n \n print(\"\\nBATCH_EVAL:\\n\")\n iou_map(gt_instance_masks, pred_instance_masks)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:31:32.508873Z","iopub.execute_input":"2021-11-06T20:31:32.509079Z","iopub.status.idle":"2021-11-06T20:32:08.489165Z","shell.execute_reply.started":"2021-11-06T20:31:32.509047Z","shell.execute_reply":"2021-11-06T20:32:08.488417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**VALIDATION DATA FIRST BATCH**","metadata":{}},{"cell_type":"code","source":"for _image_batch, _label_batch in val_dl.take(1):\n gt_mask = _label_batch[\"image_masks\"][:, 0]\n gt_boxes = _label_batch[\"groundtruth_data\"][..., :4]\n gt_is_crowds = _label_batch[\"groundtruth_data\"][..., 4]\n gt_areas = _label_batch[\"groundtruth_data\"][..., 5]\n gt_classes = _label_batch[\"groundtruth_data\"][..., 6]\n \n pred_classes, pred_boxes, pred_mask = model(_image_batch, training=False)\n pred_boxes, pred_scores, pred_classes, valid_len = postprocess.postprocess_global(config, pred_classes, pred_boxes)\n gt_instance_masks, pred_instance_masks = [], []\n \n for i in range(BATCH_SIZE):\n print(\"\\n\\n... Original Image Display Plots ...\\n\")\n _img, _mask = get_img_and_mask(**train_df.iloc[int(_label_batch[\"source_ids\"][i])][[\"img_path\", \"annotation\", \"width\", \"height\"]])\n plot_img_and_mask(_img, _mask)\n gt_instance_masks.append(_mask)\n\n print(\"\\n... GROUND TRUTH PLOT ...\\n\")\n plot_gt(_image_batch[i], gt_classes[i], gt_boxes[i], gt_mask[i])\n\n print(f\"\\n... PREDICTION PLOT (NMS={'yes' if i<4 else 'no'}) ...\\n\")\n plot_pred(_image_batch[i], pred_boxes[i], pred_scores[i], pred_classes[i], pred_mask[i], iou_thresh=0.0 if i<4 else None)\n\n print(f\"\\n... GROUND TRUTH VS. PREDICTION PLOT (NMS={'yes' if i<4 else 'no'}) ...\\n\")\n plot_diff(_image_batch[i], gt_classes[i], gt_boxes[i], gt_mask[i], pred_boxes[i], pred_scores[i], pred_classes[i], pred_mask[i], iou_thresh=0.0 if i<4 else None)\n \n pred_instance_masks.append(get_pred_instance_mask(pred_boxes[i], pred_scores[i], pred_mask[i].numpy(), iou_thresh=0.0, conf_thresh=0.1))\n \n print(\"\\n\\n\\n\\n\")\n print(\"-\"*50)\n print(\"\\n\\n\")\n \n print(\"\\nBATCH_EVAL:\\n\")\n iou_map(gt_instance_masks, pred_instance_masks)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:32:08.493011Z","iopub.execute_input":"2021-11-06T20:32:08.493221Z","iopub.status.idle":"2021-11-06T20:32:36.637192Z","shell.execute_reply.started":"2021-11-06T20:32:08.493194Z","shell.execute_reply":"2021-11-06T20:32:36.636417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_instance_masks = []\nfor _image_batch, _label_batch in test_dl:\n pred_classes, pred_boxes, pred_mask = model(_image_batch, training=False)\n pred_boxes, pred_scores, pred_classes, valid_len = postprocess.postprocess_global(config, pred_classes, pred_boxes)\n pred_instance_masks.append(get_pred_instance_mask(pred_boxes[0], pred_scores[0], pred_mask[0].numpy(), iou_thresh=0.0, conf_thresh=0.075))","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:32:36.640923Z","iopub.execute_input":"2021-11-06T20:32:36.641125Z","iopub.status.idle":"2021-11-06T20:32:38.124480Z","shell.execute_reply.started":"2021-11-06T20:32:36.641099Z","shell.execute_reply":"2021-11-06T20:32:38.123656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n\n\n\n 5 SUBMISSION ⤒\n
\n\n---\n\nWIP","metadata":{}},{"cell_type":"code","source":"submission_dfs = []\nfor i, _id in enumerate(ss_df.id.to_list()):\n tmp_df = pd.DataFrame([_id,]*int(pred_instance_masks[i].max()), columns=[\"id\"])\n tmp_df[\"predicted\"] = [rle_encode(np.where(pred_instance_masks[i]==j, 1.0, 0.0)) for j in range(1, int(pred_instance_masks[i].max()+1))]\n submission_dfs.append(tmp_df)\nsubmission_df = pd.concat(submission_dfs).reset_index(drop=True)\nsubmission_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T20:32:38.130171Z","iopub.execute_input":"2021-11-06T20:32:38.130378Z","iopub.status.idle":"2021-11-06T20:32:38.359228Z","shell.execute_reply.started":"2021-11-06T20:32:38.130353Z","shell.execute_reply":"2021-11-06T20:32:38.358502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}