{"cells":[{"metadata":{},"cell_type":"markdown","source":"# **Libraries**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install \"../input/keras-application/Keras_Applications-1.0.8-py3-none-any.whl\"\n!pip install \"../input/efficientnet111/efficientnet-1.1.1-py3-none-any.whl\"\n!pip install \"../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl\"\n!pip install \"../input/hpapytorchzoozip/pytorch_zoo-master\"\n!pip install \"../input/hpacellsegmentatormaster/HPA-Cell-Segmentation-master\"\n!pip install \"../input/tfexplainforoffline/tf_explain-0.2.1-py3-none-any.whl\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os, glob\nimport tensorflow as tf\ngpus = tf.config.experimental.list_physical_devices('GPU')\nfor gpu in gpus:\n    tf.config.experimental.set_memory_growth(gpu, True)\n# tf.compat.v1.disable_eager_execution()\nimport random\nfrom sklearn.model_selection import train_test_split\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport multiprocessing\nfrom copy import deepcopy\nimport keras\nimport keras.backend as K\nfrom keras.optimizers import Adam\nfrom keras.callbacks import Callback\n# please note, that locally I've trained a keras.efficientnet model, but using tensorflow.keras.applications.EfficientNetB0 should lead to the same results\nfrom efficientnet.keras import EfficientNetB0\nfrom keras.layers import Dense, Flatten\nfrom keras.models import Model, load_model\nfrom keras.utils import Sequence\nfrom albumentations import Compose, VerticalFlip, HorizontalFlip, Rotate, GridDistortion\nimport matplotlib.pyplot as plt\nfrom IPython.display import Image, display\nfrom numpy.random import seed\nseed(10)\nfrom tensorflow.python.framework import ops\nimport gc\nfrom numba import cuda \nimport hpacellseg.cellsegmentator as cellsegmentator\nfrom hpacellseg.utils import label_cell, label_nuclei\nfrom tqdm.auto import tqdm\nimport base64\nimport numpy as np\nfrom pycocotools import _mask as coco_mask\nimport typing as t\nimport zlib\nimport warnings\nfrom tf_explain.core.integrated_gradients import IntegratedGradients\nwarnings.filterwarnings('ignore')\n\ntf.random.set_seed(10)\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **4-channel classifier init**"},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST_IMGS_FOLDER = '../input/hpa-single-cell-image-classification/test/'\nTRAIN_IMGS_FOLDER = '../input/hpa-single-cell-image-classification/train/'\nIMG_HEIGHT = IMG_WIDTH = 512\nBATCH_SIZE = 16\nFAST_PUBLIC_RUN = True\n\n\nCHECKPOINT_NAME = 'classifier_effnetb0_rgby_512.h5'\n\nnum_cores = multiprocessing.cpu_count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from https://www.kaggle.com/c/hpa-single-cell-image-classification/data\n\n# specified_class_names = \"\"\"0. Nucleoplasm\n# 1. Nuclear membrane\n# 2. Nucleoli\n# 3. Nucleoli fibrillar center\n# 4. Nuclear speckles\n# 5. Nuclear bodies\n# 6. Endoplasmic reticulum\n# 7. Golgi apparatus\n# 8. Intermediate filaments\n# 9. Actin filaments \n# 10. Microtubules\n# 11. Mitotic spindle\n# 12. Centrosome\n# 13. Plasma membrane\n# 14. Mitochondria\n# 15. Aggresome\n# 16. Cytosol\n# 17. Vesicles and punctate cytosolic patterns\n# 18. Negative\"\"\"\n\n# class_names = [class_name.split('. ')[1] for class_name in specified_class_names.split('\\n')]\n\nclass_names = ['Nucleoplasm',\n 'Nuclear membrane',\n 'Nucleoli',\n 'Nucleoli fibrillar center',\n 'Nuclear speckles',\n 'Nuclear bodies',\n 'Endoplasmic reticulum',\n 'Golgi apparatus',\n 'Intermediate filaments',\n 'Actin filaments ',\n 'Microtubules',\n 'Mitotic spindle',\n 'Centrosome',\n 'Plasma membrane',\n 'Mitochondria',\n 'Aggresome',\n 'Cytosol',\n 'Vesicles and punctate cytosolic patterns',\n 'Negative']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_names","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **A RGB model**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# you'll need an internet connection to download ImageNet weights,\n# for illustration I'm using a randomly generated RGB model\nweights_init = 'imagenet'\n\nimagenet_model = EfficientNetB0(weights=weights_init, include_top=False, pooling='avg',\n                               input_shape=(IMG_HEIGHT, IMG_WIDTH, 3))\nrgb_model_output = Dense(len(class_names) - 1, activation='sigmoid')(imagenet_model.output)\nmodel_rgb = Model(inputs=imagenet_model.input, outputs=rgb_model_output)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **A RGBY model**"},{"metadata":{"trusted":true},"cell_type":"code","source":"four_channel_effnet = EfficientNetB0(weights=None, include_top=False, pooling='avg', \n                                     input_shape=(IMG_HEIGHT, IMG_WIDTH, 4))\nmodel_rgby_output = Dense(len(class_names) - 1, activation='sigmoid')(four_channel_effnet.output)\nmodel_rgby = Model(inputs=four_channel_effnet.input, outputs=model_rgby_output)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Copying weights**"},{"metadata":{},"cell_type":"markdown","source":"The Stem layer of EffNet requires special care: we'll copy the blud-channel weights to the newly introduced yellow-channel."},{"metadata":{"trusted":true},"cell_type":"code","source":"for layer in tqdm(model_rgby.layers, desc='Copying the pre-trained net weights..'):\n    if 'input' in layer.name or 'dense' in layer.name:\n        continue\n    elif layer.name == 'stem_conv':\n#         with graph_green.as_default():\n        kernels = model_rgb.get_layer('stem_conv').get_weights()[0]\n        kernels_extra_channel = np.concatenate((kernels, kernels[:,:,-1:,:]), axis=-2)\n        layer.set_weights([kernels_extra_channel])\n    else:\n#         with graph_green.as_default():\n        weights_green = model_rgb.get_layer(layer.name).get_weights()\n        layer.set_weights(weights_green)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Loading the fine-tuned RGBY model."},{"metadata":{"trusted":true},"cell_type":"code","source":"if FAST_PUBLIC_RUN:\n    model_rgby = load_model(f'../input/cell-models/{CHECKPOINT_NAME}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Check using explainability**"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df = pd.read_csv('../input/hpa-single-cell-image-classification/sample_submission.csv')\ntest_ids = sub_df['ID'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DataGenenerator(Sequence):\n    def __init__(self, id_list, id_2_ohe_vector=None, folder_imgs=TRAIN_IMGS_FOLDER, \n                 batch_size=BATCH_SIZE, shuffle=True, augmentation=None, resize=False,\n                 resized_height=IMG_HEIGHT, resized_width=IMG_WIDTH, num_channels=4):\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.augmentation = augmentation\n        self.id_list = deepcopy(id_list)\n        self.folder_imgs = folder_imgs\n        self.len = len(self.id_list) // self.batch_size\n        self.resized_height = resized_height\n        self.resized_width = resized_width\n        self.num_channels = num_channels\n        self.id_2_ohe_vector = id_2_ohe_vector\n        self.is_test = not 'train' in folder_imgs\n        if not self.is_test:       \n            self.num_classes = len(next(iter(id_2_ohe_vector.values())))\n        if not shuffle and not self.is_test:\n            self.labels = [id_2_ohe_vector[img] for img in self.id_list[:self.len*self.batch_size]]\n        self.resize = resize\n\n    def __len__(self):\n        return self.len\n    \n    def on_epoch_start(self):\n        if self.shuffle:\n            random.shuffle(self.id_list)\n            \n    # open_rgby adapted from https://www.kaggle.com/iafoss/pretrained-resnet34-with-rgby-0-460-public-lb\n    def open_rgby(self, image_id): #a function that reads RGBY image\n        colors = ['red','green','blue','yellow']\n        img = [cv2.imread(os.path.join(self.folder_imgs, f'{image_id}_{color}.png'), cv2.IMREAD_GRAYSCALE)\n               for color in colors]\n        img = np.stack(img, axis=-1)\n        if img.shape[0] == self.resized_height and img.shape[1] == self.resized_width:\n            return img\n        img_resized = cv2.resize(img, (self.resized_height, self.resized_width))\n        return img_resized\n\n    def __getitem__(self, idx):\n        current_batch = self.id_list[idx * self.batch_size: (idx + 1) * self.batch_size]\n        X = np.empty((self.batch_size, self.resized_height, self.resized_width, self.num_channels))\n\n        if not self.is_test:\n            y = np.empty((self.batch_size, self.num_classes))\n\n        for i, image_id in enumerate(current_batch):\n            img = self.open_rgby(image_id)\n            if not self.augmentation is None:\n                augmented = self.augmentation(image=img)\n                img = augmented['image']\n            X[i, :, :, :] = img.astype(np.float32)/255.0\n            if not self.is_test:\n                y[i, :] = self.id_2_ohe_vector[image_id]\n        if not self.is_test:\n            return X, y\n        return X\n\n    def get_labels(self):\n        if self.shuffle:\n            images_current = self.id_list[:self.len*self.batch_size]\n            labels = [self.id_2_ohe_vector[img] for img in images_current]\n        else:\n            labels = self.labels\n        return np.array(labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"is_public_test_run = len(sub_df)==559 and FAST_PUBLIC_RUN\nif is_public_test_run:\n    test_ids = test_ids[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"is_public_test_run","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"explainer = IntegratedGradients()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Cell segmentation**"},{"metadata":{"trusted":true},"cell_type":"code","source":"NUC_MODEL = '../input/hpacellsegmentatormodelweights/dpn_unet_nuclei_v1.pth'\nCELL_MODEL = '../input/hpacellsegmentatormodelweights/dpn_unet_cell_3ch_v1.pth'\n\nsegmentator = cellsegmentator.CellSegmentator(\n    NUC_MODEL,\n    CELL_MODEL,\n    scale_factor=0.25,\n    device='cuda',\n    padding=False,\n    multi_channel_model=True\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUC_MODEL","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_masks(imgs, test=True):\n    try:\n        images = [[img[:, :, 0] for img in imgs], \n                  [img[:, :, 3] for img in imgs], \n                  [img[:, :, 2] for img in imgs]]\n    \n        nuc_segmentations = segmentator.pred_nuclei(images[2])\n        cell_segmentations = segmentator.pred_cells(images)\n        cell_masks = []\n        for i in tqdm(range(len(cell_segmentations)), desc='Labeling cells..'):\n            _, cell_mask = label_cell(nuc_segmentations[i], cell_segmentations[i])\n            cell_masks.append(cell_mask)\n        return cell_masks\n    except:\n        raise ValueError('Segmentation failed')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Cell-level predictions using explainability**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def vis_integrated_gradients_masks_test(img_idx, conf_threshold=0.01, mask_height=2048, mask_width=2048, \n                                        max_cell_level_conf_2_image_level_conf=0.005, test_ids=test_ids,\n                                        model=model_rgby, quantile_level=0.9, figsize=7):\n    image_id = test_ids[img_idx]\n    img = [cv2.resize(cv2.imread(os.path.join(TEST_IMGS_FOLDER, f'{image_id}_{color}.png'), cv2.IMREAD_GRAYSCALE),\n                      (mask_height, mask_width))\n           for color in ['red','green','blue','yellow']]\n    img = np.stack(img, axis=-1)\n    mask = get_masks([img])[0]\n    n_cells = mask.max()\n    cell_2_max_conf = dict()   \n    \n    img = cv2.resize(img, (IMG_HEIGHT, IMG_WIDTH)).astype(np.float32)/255.\n    predictions_test = model.predict(np.expand_dims(img, 0))\n    \n    for class_i, class_name in enumerate(class_names[:-1]):\n        class_conf_score = predictions_test[0][class_i]\n        if class_conf_score > conf_threshold:\n            try:\n                explanation = explainer.explain(([img], None), model, class_i, n_steps=15)\n                explanation_img = cv2.resize(explanation, (mask_height, mask_width))\n                explanation_total_level = np.quantile(explanation_img.flatten(), quantile_level)\n            except:\n                continue\n\n            plt.figure(figsize=(figsize, figsize))\n            plt.imshow(mask)\n            plt.imshow(explanation_img, alpha=0.7)\n            plt.xticks([])\n            plt.yticks([])\n            plt.title(f'{test_ids[img_idx]}\\n{class_name} ({class_conf_score:.2f}): raw Grad-CAMs', fontsize=22)\n            plt.show()\n\n            masks_all = np.zeros((mask_height, mask_width))\n            coord_2_conf = dict()\n            for cell_i in range(1, n_cells + 1):\n                cell_mask_bool = mask == cell_i\n                cell_explanation_perc = np.quantile(explanation_img[cell_mask_bool], quantile_level)\n                cell_conf = np.clip(cell_explanation_perc*class_conf_score/explanation_total_level, 0, class_conf_score)\n                if cell_conf/class_conf_score >= max_cell_level_conf_2_image_level_conf and cell_conf > 1e-3:\n                    masks_all[cell_mask_bool] = 1\n                    mask_pixels_x, mask_pixels_y = np.where(cell_mask_bool)\n                    coord_2_conf[(int(mask_pixels_y.mean()), int(mask_pixels_x.mean()))] = cell_conf\n                    if not cell_i in cell_2_max_conf:\n                        cell_2_max_conf[cell_i] = cell_conf\n                    else:\n                        cell_2_max_conf[cell_i] = max(cell_conf, cell_2_max_conf[cell_i])\n\n            plt.figure(figsize=(figsize, figsize)) \n            plt.imshow(masks_all)\n            for coords, conf in coord_2_conf.items():\n                conf_rounded = np.round(conf*100)/100\n                plt.scatter(*coords, s=700, color='red', marker=r\"$ {} $\".format(conf_rounded))\n            plt.xticks([])\n            plt.yticks([])\n            plt.title(f'{test_ids[img_idx]}\\n{class_name}: cell-level predictions', fontsize=22)\n            plt.show()\n\n    masks_all = np.zeros((mask_height, mask_width))\n    coord_2_conf = dict()\n    for cell_i in range(1, n_cells + 1):\n        if not cell_i in cell_2_max_conf:\n            cell_conf = 0.99\n        else:\n            cell_conf = 1 - cell_2_max_conf[cell_i]\n        if cell_conf >= conf_threshold:\n            cell_mask_bool = mask == cell_i\n            masks_all[cell_mask_bool] = 1\n            mask_pixels_x, mask_pixels_y = np.where(cell_mask_bool)\n            coord_2_conf[(int(mask_pixels_y.mean()), int(mask_pixels_x.mean()))] = cell_conf\n\n    plt.figure(figsize=(9, 9)) \n    plt.imshow(masks_all)\n    for coords, conf in coord_2_conf.items():\n        conf_rounded = np.round(conf*100)/100\n        plt.scatter(*coords, s=700, color='red', marker=r\"$ {} $\".format(conf_rounded))\n    plt.xticks([])\n    plt.yticks([])\n    plt.title(f'{test_ids[img_idx]}\\n{class_names[-1]}: cell-level predictions', fontsize=22)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if is_public_test_run:\n    for test_img_id in range(10):\n        vis_integrated_gradients_masks_test(test_img_id)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Submission routines**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def encode_binary_mask(mask: np.ndarray) -> t.Text:\n    \"\"\"Converts a binary mask into OID challenge encoding ascii text.\"\"\"\n\n    # check input mask --\n    if mask.dtype != np.bool:\n        raise ValueError(\n            \"encode_binary_mask expects a binary mask, received dtype == %s\" %\n            mask.dtype)\n\n    mask = np.squeeze(mask)\n    if len(mask.shape) != 2:\n        raise ValueError(\n            \"encode_binary_mask expects a 2d mask, received shape == %s\" %\n            mask.shape)\n\n    # convert input mask to expected COCO API input --\n    mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)\n    mask_to_encode = mask_to_encode.astype(np.uint8)\n    mask_to_encode = np.asfortranarray(mask_to_encode)\n\n    # RLE encode mask --\n    encoded_mask = coco_mask.encode(mask_to_encode)[0][\"counts\"]\n\n    # compress and base64 encoding --\n    binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)\n    base64_str = base64.b64encode(binary_str)\n    return base64_str.decode()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_id_2_order_idx = {test_id: idx for idx, test_id in enumerate(test_ids)}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_id_2_order_idx","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Cell-level predictions**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_predictions_string_classification(img_ids, mask_heights, mask_widths,\n                                          classifier_img_height=IMG_HEIGHT, classifier_img_width=IMG_WIDTH,\n                                          classifier=model_rgby, conf_threshold=0.1, \n                                          batch_size=BATCH_SIZE, vis=False, class_names=class_names): \n    results_list = []\n    img_idx = 0\n    data_gen = DataGenenerator(img_ids, folder_imgs=TEST_IMGS_FOLDER, shuffle=False, batch_size=batch_size,\n                               resized_height=2048, resized_width=2048, resize=True)\n    \n    def get_cell_only(cell_bool_mask, img, background_val=0, vis_cell=False):\n        cell_img = img.copy()\n        cell_img[np.logical_not(cell_bool_mask)] = background_val\n        if vis_cell:\n            plt.figure(figsize=(9, 9))\n            plt.imshow(cell_img[:, :, :3])\n            plt.xticks([])\n            plt.yticks([])\n            plt.title(f'Cell only', fontsize=22)\n            plt.show()\n        return cell_img\n    \n\n    for batch_i in range(len(img_ids)//batch_size + (1 if len(img_ids)%batch_size != 0 else 0)):\n        img_batch_i = 0\n\n        images_batch = data_gen.__getitem__(batch_i)[:len(img_ids) - batch_i*batch_size, :, :]\n        img_batch_ids = img_ids[batch_i*batch_size:(batch_i + 1)*batch_size]\n        try:\n            masks_batch = get_masks(images_batch)\n            images_batch = np.stack([cv2.resize(img, (IMG_HEIGHT, IMG_WIDTH)) for img in images_batch])\n            predictions_batch = classifier.predict(images_batch)\n        except ValueError:\n            current_batch_size = images_batch.shape[0]\n            results_list.extend(['' for _ in range(current_batch_size)])\n            continue\n        \n        for mask_i, mask_init in enumerate(masks_batch):\n            try:\n                cell_2_max_conf = dict()\n                results_list_img = []\n                mask_height, mask_width = mask_heights[img_idx], mask_widths[img_idx]\n                mask = cv2.resize(mask_init, (mask_height, mask_width))\n                mask_classification = cv2.resize(mask_init, (classifier_img_height, classifier_img_width))\n                n_cells = mask_classification.max()\n                if n_cells == 0:\n                    results_list.append('')\n                img_current = images_batch[mask_i]\n                img_background_mean = img_current[mask_classification == 0].mean()\n\n                cell_2_predictions_list = []\n                classifier_batch_next = []\n                for cell_i in range(1, n_cells + 1):\n                    cell_mask_bool = mask_classification == cell_i\n                    cell_masked_img = get_cell_only(cell_mask_bool, img_current, \n                                                    background_val=img_background_mean, vis_cell=vis and mask_i==0)\n                    classifier_batch_next.append(cell_masked_img)\n                    if len(classifier_batch_next) == batch_size:\n                        try:\n                            cell_predictions_batch = classifier.predict(np.stack(classifier_batch_next))\n                        except:\n                            cell_predictions_batch = np.zeros((batch_size, len(class_names) - 1))\n                        classifier_batch_next = []\n                        cell_2_predictions_list.append(cell_predictions_batch)\n                # last incomplete batch\n                if len(classifier_batch_next) > 0:\n                    if len(classifier_batch_next) > 1:\n                        cell_imgs_last = np.stack(classifier_batch_next)\n                    else:\n                        cell_imgs_last = np.expand_dims(classifier_batch_next[0], 0)\n\n                    try:\n                        cell_predictions_batch = classifier.predict(cell_imgs_last)\n                    except:\n                        cell_predictions_batch = np.zeros((cell_imgs_last.shape[0], len(class_names) - 1))\n                    cell_2_predictions_list.append(cell_predictions_batch)\n                cell_2_predictions_np = np.concatenate(cell_2_predictions_list) if len(cell_2_predictions_list) > 1 else cell_2_predictions_list[0]\n                cell_2_rle = dict()\n                for class_i, class_name in enumerate(class_names[:-1]):\n                    class_conf_score = predictions_batch[img_batch_i][class_i]\n                    if class_conf_score > conf_threshold:\n                        for cell_i in range(n_cells):\n                            cell_conf = cell_2_predictions_np[cell_i, class_i]\n                            cell_conf = np.clip(cell_conf, 0, class_conf_score)\n                            if cell_conf > conf_threshold:\n                                if cell_i in cell_2_rle:\n                                    mask_rle = cell_2_rle[cell_i]\n                                else:\n                                    cell_mask_bool = mask == cell_i + 1\n                                    mask_rle = encode_binary_mask(cell_mask_bool)\n                                    cell_2_rle[cell_i] = mask_rle\n                                results_list_img.extend([str(class_i), f'{cell_conf:.4f}', mask_rle])\n                                if not cell_i in cell_2_max_conf:\n                                    cell_2_max_conf[cell_i] = cell_conf\n                                else:\n                                    cell_2_max_conf[cell_i] = max(cell_conf, cell_2_max_conf[cell_i])\n\n                # nothing interesting there\n                for cell_i in range(n_cells):\n                    if not cell_i in cell_2_max_conf:\n                        cell_conf = 0.99\n                    else:\n                        cell_conf = 1 - cell_2_max_conf[cell_i]\n                    if cell_conf > conf_threshold:\n                        if cell_i in cell_2_rle:\n                            mask_rle = cell_2_rle[cell_i]\n                        else:\n                            cell_mask_bool = mask == cell_i + 1\n                            mask_rle = encode_binary_mask(cell_mask_bool)\n                        results_list_img.extend([str(len(class_names) - 1), f'{cell_conf:.4f}', mask_rle])\n\n\n                results_list.append(' '.join(results_list_img))\n                img_idx += 1\n                img_batch_i += 1\n            except:\n                results_list.append('')\n                img_idx += 1\n                img_batch_i += 1\n\n    return results_list\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# sanity check\nsub_df_head = sub_df.head(2)\n# classifier_preds\ninference_step = 1\nfor next_start_block_i in range(0, sub_df_head.shape[0], inference_step):\n    sub_df_head.iloc[next_start_block_i: next_start_block_i+inference_step,\n                     sub_df_head.columns.get_loc('PredictionString')] = get_predictions_string_classification(sub_df_head['ID'].values[next_start_block_i: next_start_block_i+inference_step], \n                                                                                                              sub_df_head['ImageHeight'].values[next_start_block_i: next_start_block_i+inference_step],\n                                                                                                              sub_df_head['ImageWidth'].values[next_start_block_i: next_start_block_i+inference_step], vis=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_predictions_string_integrated_grads(img_ids, mask_heights, mask_widths,\n                                            classifier_img_height=IMG_HEIGHT, classifier_img_width=IMG_WIDTH,\n                                            max_cell_level_conf_2_image_level_conf=0.01, \n                                            model=model_rgby, quantile_level=0.9, conf_threshold=0.005, \n                                            batch_size=BATCH_SIZE, class_names=class_names): \n    results_list = []\n    img_idx = 0\n    data_gen = DataGenenerator(img_ids, folder_imgs=TEST_IMGS_FOLDER, shuffle=False, batch_size=batch_size,\n                               resized_height=2048, resized_width=2048, resize=True)\n    \n    def get_cell_only(cell_bool_mask, img, background_val=0, vis_cell=False):\n        cell_img = img.copy()\n        cell_img[np.logical_not(cell_bool_mask)] = background_val\n        if vis_cell:\n            plt.figure(figsize=(9, 9))\n            plt.imshow(cell_img[:, :, :3])\n            plt.xticks([])\n            plt.yticks([])\n            plt.title(f'Cell only', fontsize=22)\n            plt.show()\n        return cell_img\n    \n\n    for batch_i in range(len(img_ids)//batch_size + (1 if len(img_ids)%batch_size != 0 else 0)):\n        img_batch_i = 0\n\n        images_batch = data_gen.__getitem__(batch_i)[:len(img_ids) - batch_i*batch_size, :, :]\n        img_batch_ids = img_ids[batch_i*batch_size:(batch_i + 1)*batch_size]\n        try:\n            masks_batch = get_masks(images_batch)\n            images_batch = np.stack([cv2.resize(img, (IMG_HEIGHT, IMG_WIDTH)) for img in images_batch])\n            predictions_batch = model.predict(images_batch)\n        except ValueError:\n            current_batch_size = images_batch.shape[0]\n            results_list.extend(['' for _ in range(current_batch_size)])\n            continue\n        \n        for mask_i, mask_init in enumerate(masks_batch):\n            cell_2_max_conf = dict()\n            results_list_img = []\n            mask_height, mask_width = mask_heights[img_idx], mask_widths[img_idx]\n            mask = cv2.resize(mask_init, (mask_height, mask_width))\n            n_cells = mask.max()\n            if n_cells == 0:\n                results_list.append('')\n            img_current = images_batch[mask_i]\n            \n            cell_2_rle = dict()\n            cell_2_mask = dict()\n            for class_i, class_name in enumerate(class_names[:-1]):\n                class_conf_score = predictions_batch[img_batch_i][class_i]\n                if class_conf_score > conf_threshold:                    \n                    try:\n                        explanation = explainer.explain(([img_current], None), model, class_i, n_steps=15)\n                        explanation_img = cv2.resize(explanation, (mask_height, mask_width))\n                        explanation_total_level = np.quantile(explanation_img.flatten(), quantile_level)\n                    except:\n                        continue\n                    for cell_i in range(n_cells):\n                        if cell_i in cell_2_mask:\n                            cell_mask_bool = cell_2_mask[cell_i]\n                        else:\n                            cell_mask_bool = mask == cell_i\n                            cell_2_mask[cell_i] = cell_mask_bool\n                        cell_explanation_perc = np.quantile(explanation_img[cell_mask_bool], quantile_level)\n                        cell_conf = np.clip(cell_explanation_perc*class_conf_score/explanation_total_level, 0, class_conf_score)\n                        if cell_conf/class_conf_score >= max_cell_level_conf_2_image_level_conf and cell_conf > 1e-3:  \n                            if cell_i in cell_2_rle:\n                                mask_rle = cell_2_rle[cell_i]\n                            else:\n                                mask_rle = encode_binary_mask(cell_mask_bool)\n                                cell_2_rle[cell_i] = mask_rle\n                            results_list_img.extend([str(class_i), f'{cell_conf:.4f}', mask_rle])\n                            if not cell_i in cell_2_max_conf:\n                                cell_2_max_conf[cell_i] = cell_conf\n                            else:\n                                cell_2_max_conf[cell_i] = max(cell_conf, cell_2_max_conf[cell_i])\n\n            # nothing interesting there\n            for cell_i in range(n_cells):\n                if not cell_i in cell_2_max_conf:\n                    cell_conf = 0.99\n                else:\n                    cell_conf = 1 - cell_2_max_conf[cell_i]\n                if cell_conf > conf_threshold:\n                    if cell_i in cell_2_rle:\n                        mask_rle = cell_2_rle[cell_i]\n                    else:\n                        cell_mask_bool = mask == cell_i + 1\n                        mask_rle = encode_binary_mask(cell_mask_bool)\n                    results_list_img.extend([str(len(class_names) - 1), f'{cell_conf:.4f}', mask_rle])\n\n\n            results_list.append(' '.join(results_list_img))\n            img_idx += 1\n            img_batch_i += 1\n#             except:\n#                 results_list.append('')\n#                 img_idx += 1\n#                 img_batch_i += 1\n\n    return results_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# sanity check\nsub_df_head = sub_df.head(2)\ninference_step = 1\nfor next_start_block_i in range(0, sub_df_head.shape[0], inference_step):\n    sub_df_head.iloc[next_start_block_i: next_start_block_i+inference_step,\n                     sub_df_head.columns.get_loc('PredictionString')] = get_predictions_string_integrated_grads(sub_df_head['ID'].values[next_start_block_i: next_start_block_i+inference_step], \n                                                                                         sub_df_head['ImageHeight'].values[next_start_block_i: next_start_block_i+inference_step],\n                                                                                         sub_df_head['ImageWidth'].values[next_start_block_i: next_start_block_i+inference_step])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del sub_df_head\ndel model_rgb\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# to save the time for the public test set run\nif is_public_test_run:\n    sub_df.to_csv('submission.csv', index=None)\nelse:   \n    sub_df['PredictionString'] = ''\n    gc.collect()\n    \n    inference_step = 16\n    for next_start_block_i in range(0, sub_df.shape[0], inference_step):\n        sub_df.iloc[next_start_block_i: next_start_block_i+inference_step,\n                    sub_df.columns.get_loc('PredictionString')] = get_predictions_string_integrated_grads(sub_df['ID'].values[next_start_block_i: next_start_block_i+inference_step], \n                                                                                   sub_df['ImageHeight'].values[next_start_block_i: next_start_block_i+inference_step],\n                                                                                   sub_df['ImageWidth'].values[next_start_block_i: next_start_block_i+inference_step])\n    sub_df.to_csv('submission.csv', index=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}