{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5"},"cell_type":"markdown","source":"# Intro\nIn this notebook, I'd create a simple baseline. I'll build a classifier on top of image-level labels (multi-label classification), then use an explainability technique Guided-GRADCAM to extract regions responsible for particular class prediction, and then assign the segmented cells to particular classes based on the overlap with Grad-CAM outputs.\n\n**Credits**:\nTo segment cells offline, I'll use [this notebook by RDizzl3](https://www.kaggle.com/rdizzl3/hpa-segmentation-masks-no-internet), the corresponding datasets. I also checked out the batched version from [this notebook by Darek Kłeczek](https://www.kaggle.com/thedrcat/hpa-baseline-cell-segmentation).\n"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true},"cell_type":"markdown","source":"# Plan\n1. [Libraries](#Libraries)\n2. [Data Generators](#Data-Generators)\n  * [One-hot encoding classes](#One-hot-encoding-classes)\n  * [Stratified split into train/val](#Stratified-split-into-train/val)\n  * [Generator class](#Generator-class)\n3. [PR-AUC-based Callback](#PR-AUC-based-Callback)\n4. [Classifier](#Classifier)\n  * [Defining a model](#Defining-a-model)\n  * [Initial tuning of the added fully-connected layer](#Initial-tuning-of-the-added-fully-connected-layer)\n  * [Training the whole model](#Training-the-whole-model)\n  * [Visualizing train and val PR AUC](#Visualizing-train-and-val-PR-AUC)\n5. [Extracting Integrated gradients](#Extracting-Integrated-gradients)\n6. [Cell segmentation](#Cell-segmentation)\n7. [Cell level predictions](#Cell-level-predictions)\n8. [Submission](#Submission)"},{"metadata":{},"cell_type":"markdown","source":"# Libraries"},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!pip install \"../input/keras-application/Keras_Applications-1.0.8-py3-none-any.whl\"\n!pip install \"../input/efficientnet111/efficientnet-1.1.1-py3-none-any.whl\"\n!pip install \"../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl\"\n!pip install \"../input/hpapytorchzoozip/pytorch_zoo-master\"\n!pip install \"../input/hpacellsegmentatormaster/HPA-Cell-Segmentation-master\"\n!pip install \"../input/tfexplainforoffline/tf_explain-0.2.1-py3-none-any.whl\"","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"import os, glob\nimport tensorflow as tf\ngpus = tf.config.experimental.list_physical_devices('GPU')\nfor gpu in gpus:\n    tf.config.experimental.set_memory_growth(gpu, True)\n# tf.compat.v1.disable_eager_execution()\nimport random\nfrom sklearn.model_selection import train_test_split\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport multiprocessing\nfrom copy import deepcopy\nfrom sklearn.metrics import precision_recall_curve, auc\nimport keras\nimport keras.backend as K\nfrom keras.optimizers import Adam\nfrom keras.callbacks import Callback\n# please note, that locally I've trained a keras.efficientnet model, but using tensorflow.keras.applications.EfficientNetB0 should lead to the same results\nfrom efficientnet.keras import EfficientNetB0\nfrom keras.layers import Dense, Flatten\nfrom keras.models import Model, load_model\nfrom keras.utils import Sequence\nfrom albumentations import Compose, VerticalFlip, HorizontalFlip, Rotate, GridDistortion\nimport matplotlib.pyplot as plt\nfrom IPython.display import Image, display\nfrom numpy.random import seed\nseed(10)\nfrom tensorflow.python.framework import ops\nimport gc\nfrom numba import cuda \nimport hpacellseg.cellsegmentator as cellsegmentator\nfrom hpacellseg.utils import label_cell, label_nuclei\nfrom tqdm.auto import tqdm\nimport base64\nimport numpy as np\nfrom pycocotools import _mask as coco_mask\nimport typing as t\nimport zlib\nimport warnings\nfrom tf_explain.core.integrated_gradients import IntegratedGradients\nwarnings.filterwarnings('ignore')\n\ntf.random.set_seed(10)\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"TEST_IMGS_FOLDER = '../input/hpa-single-cell-image-classification/test/'\nTRAIN_IMGS_FOLDER = '../input/hpa-single-cell-image-classification/train/'\nIMG_HEIGHT = IMG_WIDTH = 512\nBATCH_SIZE = 8\n\nLOAD_PRETRAINED_MODEL = True\nTRAIN_MODEL = False\nFAST_PUBLIC_RUN = True\n\nCHECKPOINT_NAME = 'classifier_effnetb0_512.h5'\n\nnum_cores = multiprocessing.cpu_count()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Generators"},{"metadata":{},"cell_type":"markdown","source":"## One-hot encoding classes"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/hpa-single-cell-image-classification/train.csv')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from https://www.kaggle.com/c/hpa-single-cell-image-classification/data\n\nspecified_class_names = \"\"\"0. Nucleoplasm\n1. Nuclear membrane\n2. Nucleoli\n3. Nucleoli fibrillar center\n4. Nuclear speckles\n5. Nuclear bodies\n6. Endoplasmic reticulum\n7. Golgi apparatus\n8. Intermediate filaments\n9. Actin filaments \n10. Microtubules\n11. Mitotic spindle\n12. Centrosome\n13. Plasma membrane\n14. Mitochondria\n15. Aggresome\n16. Cytosol\n17. Vesicles and punctate cytosolic patterns\n18. Negative\"\"\"\n\nclass_names = [class_name.split('. ')[1] for class_name in specified_class_names.split('\\n')]\nclass_names","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['Label'] = train_df['Label'].map(lambda x: map(int, x.split('|'))).map(set)\nfor class_i, class_name in enumerate(class_names):\n    train_df[class_name] = train_df['Label'].map(lambda x: 1 if class_i in x else 0)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dictionary for fast access to ohe vectors\nid_2_ohe_vector = {img:vec for img, vec in zip(train_df['ID'], train_df.iloc[:, 2:-1].values)}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Stratified split into train/val\nLet's stratify based on combination of labels. The unique combinations will be put into train."},{"metadata":{"trusted":true},"cell_type":"code","source":"label_combinations = train_df['Label'].map(lambda x: str(sorted(list(x))))\nf'There are {sum(label_combinations.value_counts() == 1)} images with unique label combinations out of {len(label_combinations)}.'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_combinations_counts = label_combinations.value_counts()\nunique_label_combs = label_combinations_counts.index[(label_combinations_counts == 1).values]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ids_unique_combs = train_df['ID'].loc[label_combinations.map(lambda x: x in unique_label_combs)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"non_unique_combo_bool_idx = label_combinations.map(lambda x: x not in unique_label_combs)\ntrain_ids, val_ids = train_test_split(train_df['ID'].loc[non_unique_combo_bool_idx].values, \n                                        test_size=0.2, \n                                        stratify=label_combinations.loc[non_unique_combo_bool_idx], # sorting present classes in lexicographical order, just to be sure\n                                        random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ids = np.concatenate((train_ids, train_ids_unique_combs))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df = pd.read_csv('../input/hpa-single-cell-image-classification/sample_submission.csv')\ntest_ids = sub_df['ID'].values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generator class"},{"metadata":{},"cell_type":"markdown","source":"Using green filter, as \"the green filter should be used to predict the label, and the other filters are used as references.\" ([from the data page](https://www.kaggle.com/c/hpa-single-cell-image-classification/data))"},{"metadata":{"trusted":true},"cell_type":"code","source":"class DataGenenerator(Sequence):\n    def __init__(self, id_list, id_2_ohe_vector=None, folder_imgs=TRAIN_IMGS_FOLDER, \n                 batch_size=BATCH_SIZE, shuffle=True, augmentation=None,\n                 resized_height=IMG_HEIGHT, resized_width=IMG_WIDTH, num_channels=3):\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.augmentation = augmentation\n        self.id_list = deepcopy(id_list)\n        self.folder_imgs = folder_imgs\n        self.len = len(self.id_list) // self.batch_size\n        self.resized_height = resized_height\n        self.resized_width = resized_width\n        self.num_channels = num_channels\n        self.id_2_ohe_vector = id_2_ohe_vector\n        self.is_test = not 'train' in folder_imgs\n        if not self.is_test:       \n            self.num_classes = len(next(iter(id_2_ohe_vector.values())))\n        if not shuffle and not self.is_test:\n            self.labels = [id_2_ohe_vector[img] for img in self.id_list[:self.len*self.batch_size]]\n\n    def __len__(self):\n        return self.len\n    \n    def on_epoch_start(self):\n        if self.shuffle:\n            random.shuffle(self.id_list)\n\n    def __getitem__(self, idx):\n        current_batch = self.id_list[idx * self.batch_size: (idx + 1) * self.batch_size]\n        X = np.empty((self.batch_size, self.resized_height, self.resized_width, self.num_channels))\n\n        if not self.is_test:\n            y = np.empty((self.batch_size, self.num_classes))\n\n        for i, image_id in enumerate(current_batch):\n            path = os.path.join(self.folder_imgs, f'{image_id}_green.png')\n            img = cv2.resize(cv2.imread(path), (self.resized_height, self.resized_width)).astype(np.float32)\n            if not self.augmentation is None:\n                augmented = self.augmentation(image=img)\n                img = augmented['image']\n            X[i, :, :, :] = img/255.0\n            if not self.is_test:\n                y[i, :] = self.id_2_ohe_vector[image_id]\n        if not self.is_test:\n            return X, y\n        return X\n\n    def get_labels(self):\n        if self.shuffle:\n            images_current = self.id_list[:self.len*self.batch_size]\n            labels = [self.id_2_ohe_vector[img] for img in images_current]\n        else:\n            labels = self.labels\n        return np.array(labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"albumentations_train = Compose([\n    VerticalFlip(), HorizontalFlip(), Rotate(limit=50), GridDistortion()\n], p=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Generator instances"},{"metadata":{"trusted":true},"cell_type":"code","source":"is_public_test_run = len(sub_df)==559 and FAST_PUBLIC_RUN\nif is_public_test_run:\n    test_ids = test_ids[:10]\n# to speed up private set submission, if no training is expected\nif not TRAIN_MODEL:\n    train_ids = train_ids[:2000]\n    val_ids = val_ids[:1000]\ndata_generator_train = DataGenenerator(train_ids, id_2_ohe_vector, augmentation=albumentations_train)\ndata_generator_train_eval = DataGenenerator(train_ids, id_2_ohe_vector, shuffle=False)\ndata_generator_val = DataGenenerator(val_ids, id_2_ohe_vector, shuffle=False)\ndata_generator_test = DataGenenerator(test_ids, folder_imgs=TEST_IMGS_FOLDER, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# PR-AUC-based Callback"},{"metadata":{},"cell_type":"markdown","source":"The callback would be used:\n1. to estimate AUC under precision recall curve for each class,\n2. to early stop after 5 epochs of no improvement in mean PR AUC,\n3. save a model with the best PR AUC in validation,\n4. to reduce learning rate on PR AUC plateau."},{"metadata":{"trusted":true},"cell_type":"code","source":"class PrAucCallback(Callback):\n    def __init__(self, data_generator, class_names, num_workers=num_cores, \n                 early_stopping_patience=5, \n                 plateau_patience=3, reduction_rate=0.5,\n                 stage='train', checkpoints_path='checkpoints/', model_name='effnetb0'):\n        super(Callback, self).__init__()\n        self.data_generator = data_generator\n        self.num_workers = num_workers\n        self.class_names = class_names\n        self.history = [[] for _ in range(len(self.class_names) + 1)] # to store per each class and also mean PR AUC\n        self.early_stopping_patience = early_stopping_patience\n        self.plateau_patience = plateau_patience\n        self.reduction_rate = reduction_rate\n        self.stage = stage\n        self.best_pr_auc = -float('inf')\n        if not os.path.exists(checkpoints_path):\n            os.makedirs(checkpoints_path)\n        self.checkpoints_path = checkpoints_path\n        self.model_name = model_name\n        \n    def compute_pr_auc(self, y_true, y_pred):\n        pr_auc_mean = 0\n        print(f\"\\n{'#'*30}\\n\")\n        for class_i in range(len(self.class_names)):\n            precision, recall, _ = precision_recall_curve(y_true[:, class_i], y_pred[:, class_i])\n            pr_auc = auc(recall, precision)\n            pr_auc_mean += pr_auc/len(self.class_names)\n            print(f\"PR AUC {self.class_names[class_i]}, {self.stage}: {pr_auc:.3f}\\n\")\n            self.history[class_i].append(pr_auc)        \n        print(f\"\\n{'#'*20}\\n PR AUC mean, {self.stage}: {pr_auc_mean:.3f}\\n{'#'*20}\\n\")\n        self.history[-1].append(pr_auc_mean)\n        return pr_auc_mean\n              \n    def is_patience_lost(self, patience):\n        if len(self.history[-1]) > patience:\n            best_performance = max(self.history[-1][-(patience + 1):-1])\n            return best_performance == self.history[-1][-(patience + 1)] and best_performance >= self.history[-1][-1]    \n              \n    def early_stopping_check(self, pr_auc_mean):\n        if self.is_patience_lost(self.early_stopping_patience):\n            self.model.stop_training = True    \n              \n    def model_checkpoint(self, pr_auc_mean, epoch):\n        if pr_auc_mean > self.best_pr_auc:\n            # remove previous checkpoints to save space\n            for checkpoint in glob.glob(os.path.join(self.checkpoints_path, f'classifier_{self.model_name}_epoch_*')):\n                os.remove(checkpoint)\n            self.best_pr_auc = pr_auc_mean\n            self.model.save(os.path.join(self.checkpoints_path, f'classifier_{self.model_name}_epoch_{epoch}_val_pr_auc_{pr_auc_mean}.h5'))              \n            print(f\"\\n{'#'*20}\\nSaved new checkpoint\\n{'#'*20}\\n\")\n              \n    def reduce_lr_on_plateau(self):\n        if self.is_patience_lost(self.plateau_patience):\n            new_lr = float(keras.backend.get_value(self.model.optimizer.lr)) * self.reduction_rate\n            keras.backend.set_value(self.model.optimizer.lr, new_lr)\n            print(f\"\\n{'#'*20}\\nReduced learning rate to {new_lr}.\\n{'#'*20}\\n\")\n        \n    def on_epoch_end(self, epoch, logs={}):\n        y_pred = self.model.predict(self.data_generator, workers=self.num_workers)\n        y_true = self.data_generator.get_labels()\n        # estimate AUC under precision recall curve for each class\n        pr_auc_mean = self.compute_pr_auc(y_true, y_pred)\n              \n        if self.stage == 'val':\n            # early stop after early_stopping_patience=4 epochs of no improvement in mean PR AUC\n            self.early_stopping_check(pr_auc_mean)\n\n            # save a model with the best PR AUC in validation\n            self.model_checkpoint(pr_auc_mean, epoch)\n\n            # reduce learning rate on PR AUC plateau\n            self.reduce_lr_on_plateau()            \n        \n    def get_pr_auc_history(self):\n        return self.history","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Callback instances"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_metric_callback = PrAucCallback(data_generator_train_eval, class_names[:-1])\nval_callback = PrAucCallback(data_generator_val, class_names[:-1], stage='val')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Classifier"},{"metadata":{},"cell_type":"markdown","source":"## Defining a model"},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"def get_model(class_names):\n    K.clear_session()\n    base_model =  EfficientNetB0(weights='imagenet', include_top=False, pooling='avg', \n                                 input_shape=(IMG_HEIGHT, IMG_WIDTH, 3))\n    x = base_model.output\n    y_pred = Dense(len(class_names) - 1, activation='sigmoid')(x)\n    return Model(inputs=base_model.input, outputs=y_pred)\n\nif not LOAD_PRETRAINED_MODEL:\n    model = get_model(class_names)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Initial tuning of the added fully-connected layer"},{"metadata":{"trusted":true},"cell_type":"code","source":"if LOAD_PRETRAINED_MODEL:\n    model = load_model(f'../input/cell-models/{CHECKPOINT_NAME}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for base_layer in model.layers[:-1]:\n    base_layer.trainable = False\n    \nmodel.compile(optimizer=Adam(lr=1e-2),  loss='categorical_crossentropy')\nhistory_0 = model.fit(data_generator_train,\n                      validation_data=data_generator_val,\n                      epochs=1,\n                      callbacks=[train_metric_callback, val_callback],\n                      workers=num_cores,\n                      verbose=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training the whole model"},{"metadata":{"trusted":true},"cell_type":"code","source":"if TRAIN_MODEL:\n    for base_layer in model.layers:\n        base_layer.trainable = True\n\n    model.compile(optimizer=Adam(lr=3e-4),  loss='categorical_crossentropy')\n    history_1 = model.fit(data_generator_train,\n                          validation_data=data_generator_val,\n                          epochs=30,\n                          callbacks=[train_metric_callback, val_callback],\n                          workers=num_cores,\n                          verbose=1,\n                          initial_epoch=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualizing train and val PR AUC"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_with_dots(ax, np_array):\n    ax.scatter(list(range(1, len(np_array) + 1)), np_array, s=50)\n    ax.plot(list(range(1, len(np_array) + 1)), np_array)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if TRAIN_MODEL:\n    pr_auc_history_train = train_metric_callback.get_pr_auc_history()\n    pr_auc_history_val = val_callback.get_pr_auc_history()\n\n    plt.figure(figsize=(10, 7))\n    plot_with_dots(plt, pr_auc_history_train[-1])\n    plot_with_dots(plt, pr_auc_history_val[-1])\n\n    plt.xlabel('Epoch', fontsize=17)\n    plt.ylabel('Mean PR AUC', fontsize=17)\n    plt.legend(['Train', 'Val'])\n    plt.title('Training and Validation PR AUC', fontsize=22)\n    plt.savefig('pr_auc_hist.png')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I left the model to train longer on my local GPU. I then upload the best model and plots from the model training."},{"metadata":{"trusted":true},"cell_type":"code","source":"if LOAD_PRETRAINED_MODEL and not TRAIN_MODEL:\n    model = load_model(f'../input/cell-models/{CHECKPOINT_NAME}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#  (for the 260x260 effnetb0)\nif LOAD_PRETRAINED_MODEL:\n    display(Image(\"../input/cell-models/pr_auc_hist.png\"))    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Extracting Integrated gradients"},{"metadata":{},"cell_type":"markdown","source":"Using the awesome tf-explain library."},{"metadata":{"trusted":true},"cell_type":"code","source":"explainer = IntegratedGradients()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_id = test_ids[0]\npath = os.path.join(TEST_IMGS_FOLDER, f'{image_id}_green.png')\nimg = cv2.resize(cv2.imread(path), (IMG_HEIGHT, IMG_WIDTH)).astype(np.float32)/255.0\ndata = ([img], None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grid = explainer.explain(data, model, 1, n_steps=15)\nplt.imshow(grid)\nplt.title(grid.max())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Cell segmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_image_names(image_id: str, test: bool = False, height: int = 2048, width: int = 2048) -> list:\n    \n    def read_img(path: str, height: int = height, width: int = width):\n        return cv2.resize(cv2.cvtColor(cv2.imread(path), cv2.COLOR_BGR2GRAY), (height, width))\n    \n    # mt is the mitchondria\n    mt = f'../input/hpa-single-cell-image-classification/{\"test\" if test else \"train\"}/{image_id}_red.png'\n    # er is the endoplasmic reticulum\n    er = f'../input/hpa-single-cell-image-classification/{\"test\" if test else \"train\"}/{image_id}_yellow.png'\n    # nu is the nuclei\n    nu = f'../input/hpa-single-cell-image-classification/{\"test\" if test else \"train\"}/{image_id}_blue.png'\n    \n    mt_img = read_img(mt)\n    er_img = read_img(er)\n    nu_img = read_img(nu)\n    return mt_img, er_img, nu_img\n\nNUC_MODEL = '../input/hpacellsegmentatormodelweights/dpn_unet_nuclei_v1.pth'\nCELL_MODEL = '../input/hpacellsegmentatormodelweights/dpn_unet_cell_3ch_v1.pth'\n\nsegmentator = cellsegmentator.CellSegmentator(\n    NUC_MODEL,\n    CELL_MODEL,\n    scale_factor=0.25,\n    device='cuda',\n    padding=False,\n    multi_channel_model=True\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_masks(img_ids, test=True):\n    mt_imgs = []\n    er_imgs = []\n    nu_imgs = []\n    for img_id in img_ids:\n        mt_img, er_img, nu_img = build_image_names(image_id=img_id, test=test)\n        mt_imgs.append(mt_img)\n        er_imgs.append(er_img)\n        nu_imgs.append(nu_img)\n    images = [mt_imgs, er_imgs, nu_imgs]\n    \n    try:\n        nuc_segmentations = segmentator.pred_nuclei(images[2])\n        cell_segmentations = segmentator.pred_cells(images)\n        cell_masks = []\n        for i in tqdm(range(len(cell_segmentations)), desc='Labeling cells..'):\n            _, cell_mask = label_cell(nuc_segmentations[i], cell_segmentations[i])\n            cell_masks.append(cell_mask)\n        return cell_masks\n    except:\n        return None","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Cell-level predictions"},{"metadata":{},"cell_type":"markdown","source":"Let's use the following heuristics:\n\nwe'll use the 95th percentile of Integrated Gradients output to estimate value roughly corresponding to the image-level confidence. The image conf. level is the predicted class confidence from our classifier. \n\nNext, we'll focus on Integrated Gradients output on top of the cell region. As not the whole true-positive cell must be lightened up by Integrated Gradients, we'll use the 95th percentile of the cell's to compare it with the value corresponding to the global image conf. level. Using these two levels we'll estimate the cell-level confidence: the higher the Integrated Gradients output, the higher the confidence. \n\nFinally, we'll output only cells with confidence values reaching at least 60% of the image-level confidence."},{"metadata":{"trusted":true},"cell_type":"code","source":"max_cell_level_conf_2_image_level_conf=0.4\nquantile_level=0.95\n\n\ndef vis_masks_test(img_idx, conf_threshold=0.05, mask_height=2048, mask_width=2048, \n                   max_cell_level_conf_2_image_level_conf=max_cell_level_conf_2_image_level_conf, \n                   test_ids=test_ids, figsize=7, quantile_level=quantile_level):\n    image_id = test_ids[img_idx]\n    mask = get_masks([image_id])[0]\n    n_cells = mask.max()\n    cell_2_max_conf = dict()\n    \n    path = os.path.join(TEST_IMGS_FOLDER, f'{image_id}_green.png')\n    img = cv2.resize(cv2.imread(path), (IMG_HEIGHT, IMG_WIDTH)).astype(np.float32)/255.0\n    predictions_test = model.predict(np.expand_dims(img, 0))\n    \n    for class_i, class_name in enumerate(class_names[:-1]):\n        class_conf_score = predictions_test[0][class_i]\n        if class_conf_score > conf_threshold:\n            try:\n                explanation = explainer.explain(([img], None), model, class_i, n_steps=15)\n                explanation_img = cv2.resize(explanation, (mask_height, mask_width))\n                explanation_total_level = np.quantile(explanation_img.flatten(), quantile_level)\n            except:\n                continue\n\n            plt.figure(figsize=(figsize, figsize))\n            plt.imshow(mask)\n            plt.imshow(explanation_img, alpha=0.7)\n            plt.xticks([])\n            plt.yticks([])\n            plt.title(f'{test_ids[img_idx]}\\n{class_name} ({class_conf_score:.2f}): raw Grad-CAMs', fontsize=22)\n            plt.show()\n            \n            masks_all = np.zeros((mask_height, mask_width))\n            coord_2_conf = dict()\n            for cell_i in range(1, n_cells + 1):\n                cell_mask_bool = mask == cell_i\n                cell_explanation_perc = np.quantile(explanation_img[cell_mask_bool], quantile_level)\n                cell_conf = np.clip(cell_explanation_perc*class_conf_score/explanation_total_level, 0, class_conf_score)\n                if cell_conf/class_conf_score >= max_cell_level_conf_2_image_level_conf:\n                    masks_all[cell_mask_bool] = 1\n                    mask_pixels_x, mask_pixels_y = np.where(cell_mask_bool)\n                    coord_2_conf[(int(mask_pixels_y.mean()), int(mask_pixels_x.mean()))] = cell_conf\n                    if not cell_i in cell_2_max_conf:\n                        cell_2_max_conf[cell_i] = cell_conf\n                    else:\n                        cell_2_max_conf[cell_i] = max(cell_conf, cell_2_max_conf[cell_i])\n\n            plt.figure(figsize=(figsize, figsize)) \n            plt.imshow(masks_all)\n            for coords, conf in coord_2_conf.items():\n                conf_rounded = np.round(conf*100)/100\n                plt.scatter(*coords, s=700, color='red', marker=r\"$ {} $\".format(conf_rounded))\n            plt.xticks([])\n            plt.yticks([])\n            plt.title(f'{test_ids[img_idx]}\\n{class_name}: cell-level predictions', fontsize=22)\n            plt.show()\n            \n    # nothing interesting there\n\n    masks_all = np.zeros((mask_height, mask_width))\n    coord_2_conf = dict()\n    for cell_i in range(1, n_cells + 1):\n        if not cell_i in cell_2_max_conf:\n            cell_conf = 0.99\n        else:\n            cell_conf = 1 - cell_2_max_conf[cell_i]\n        if cell_conf >= conf_threshold:\n            cell_mask_bool = mask == cell_i\n            masks_all[cell_mask_bool] = 1\n            mask_pixels_x, mask_pixels_y = np.where(cell_mask_bool)\n            coord_2_conf[(int(mask_pixels_y.mean()), int(mask_pixels_x.mean()))] = cell_conf\n\n    plt.figure(figsize=(figsize, figsize)) \n    plt.imshow(masks_all)\n    for coords, conf in coord_2_conf.items():\n        conf_rounded = np.round(conf*100)/100\n        plt.scatter(*coords, s=700, color='red', marker=r\"$ {} $\".format(conf_rounded))\n    plt.xticks([])\n    plt.yticks([])\n    plt.title(f'{test_ids[img_idx]}\\n{class_names[-1]}: cell-level predictions', fontsize=22)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for test_img_id in range(10):\n    vis_masks_test(test_img_id)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"def encode_binary_mask(mask: np.ndarray) -> t.Text:\n    \"\"\"Converts a binary mask into OID challenge encoding ascii text.\"\"\"\n\n    # check input mask --\n    if mask.dtype != np.bool:\n        raise ValueError(\n            \"encode_binary_mask expects a binary mask, received dtype == %s\" %\n            mask.dtype)\n\n    mask = np.squeeze(mask)\n    if len(mask.shape) != 2:\n        raise ValueError(\n            \"encode_binary_mask expects a 2d mask, received shape == %s\" %\n            mask.shape)\n\n    # convert input mask to expected COCO API input --\n    mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)\n    mask_to_encode = mask_to_encode.astype(np.uint8)\n    mask_to_encode = np.asfortranarray(mask_to_encode)\n\n    # RLE encode mask --\n    encoded_mask = coco_mask.encode(mask_to_encode)[0][\"counts\"]\n\n    # compress and base64 encoding --\n    binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)\n    base64_str = base64.b64encode(binary_str)\n    return base64_str.decode()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_id_2_order_idx = {test_id: idx for idx, test_id in enumerate(test_ids)}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dummy_prediction = sub_df['PredictionString'].values[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_predictions_string(img_ids, mask_heights, mask_widths, conf_threshold=0.05, \n                           max_cell_level_conf_2_image_level_conf=max_cell_level_conf_2_image_level_conf, \n                           quantile_level=quantile_level, batch_size=8): \n    results_list = []\n    img_idx = 0\n    data_gen = DataGenenerator(test_ids, folder_imgs=TEST_IMGS_FOLDER, shuffle=False, batch_size=batch_size)\n    # for the case, when there's just a single incomplete batch\n    batch_i = -1\n    for batch_i in range(len(img_ids)//batch_size + 1):\n        assert batch_i*batch_size == len(results_list), f'Prev batch_i: {batch_i - 1}'\n        img_batch_i = 0\n\n        images_batch = data_gen.__getitem__(batch_i)[:len(img_ids) - batch_i*batch_size, :, :]\n\n        predictions_test = model.predict(images_batch)\n\n        img_batch_ids = img_ids[batch_i*batch_size:(batch_i + 1)*batch_size]\n        masks_batch = get_masks(img_batch_ids)\n        if masks_batch is None:\n            current_batch_size = images_batch.shape[0]\n            results_list.append([dummy_prediction for _ in range(current_batch_size)])\n            print('batch masks were None')\n            continue\n        for mask in masks_batch:\n            n_cells = mask.max()\n            cell_2_max_conf = dict()\n            results_list_img = []\n            mask_height, mask_width = mask_heights[img_idx], mask_widths[img_idx]\n            mask = cv2.resize(mask, (mask_height, mask_width))\n            for class_i, class_name in enumerate(class_names[:-1]):\n                class_conf_score = predictions_test[img_batch_i][class_i]\n                if class_conf_score > conf_threshold:\n                    try:\n                        explanation = explainer.explain(([images_batch[img_batch_i]], None), model, class_i, n_steps=15)\n                        explanation_img = cv2.resize(explanation, (mask_height, mask_width))\n                        explanation_total_level = np.quantile(explanation_img.flatten(), quantile_level)\n                    except:\n                        print('explanation issue')\n                        continue\n                    coord_2_conf = dict()\n                    for cell_i in range(1, n_cells + 1):\n                        cell_mask_bool = mask == cell_i\n                        cell_explanation_perc = np.quantile(explanation_img[cell_mask_bool], quantile_level)\n                        cell_conf = np.clip(cell_explanation_perc*class_conf_score/explanation_total_level, 0, class_conf_score)\n                        if cell_conf/class_conf_score >= max_cell_level_conf_2_image_level_conf:\n                            mask_rle = encode_binary_mask(cell_mask_bool)\n                            results_list_img.extend([class_i, cell_conf, mask_rle])\n                            if not cell_i in cell_2_max_conf:\n                                cell_2_max_conf[cell_i] = cell_conf\n                            else:\n                                cell_2_max_conf[cell_i] = max(cell_conf, cell_2_max_conf[cell_i])\n                                \n            # nothing interesting there\n            for cell_i in range(1, n_cells + 1):\n                if not cell_i in cell_2_max_conf:\n                    cell_conf = 0.99\n                else:\n                    cell_conf = 1 - cell_2_max_conf[cell_i]\n                if cell_conf >= conf_threshold:\n                    cell_mask_bool = mask == cell_i\n                    mask_rle = encode_binary_mask(cell_mask_bool)\n                    results_list_img.extend([len(class_names) - 1, cell_conf, mask_rle])\n\n            results_list.append(' '.join([str(item) for item in results_list_img]))\n            img_idx += 1\n            img_batch_i += 1\n\n    return results_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# to save the time for the public test set run\nif is_public_test_run:\n    sub_df.to_csv('submission.csv', index=None)\nelse:    \n    sub_df['PredictionString'] = get_predictions_string(sub_df['ID'].values, \n                                                        sub_df['ImageHeight'].values,\n                                                        sub_df['ImageWidth'].values)\n    sub_df.to_csv('submission.csv', index=None)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}