{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install \"../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl\"\n!pip install \"../input/hpapytorchzoozip/pytorch_zoo-master\"\n!pip install \"../input/hpacellsegmentatorraman/HPA-Cell-Segmentation/\"\n!pip install \"../input/efficientnet-keras-source-code/\" --no-deps\n!pip install \"../input/kerasapplications\"\n#!pip install \"../input/pythonpoetry-poetrycore\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install \"../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0\"\n#!pip install \"../input/faustomorales-vitkeras\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#EB4:E3+F4_E4_R101:E3+F1_E3_W622_R101_E9_RGB_TTA\n#EB4:E3+F4_E4_R101:E3+F1_E3_W64_R101E9_EB7_TTA\n#EB4:E3+F4_E4_R101:E3+F1_E3_W64_R101E9_EB7_TTA_SCALE","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader, random_split\nimport torchvision\nimport torchvision.transforms as tfms\nimport numpy as np\nimport pandas as pd \nimport matplotlib.pyplot as plt \nimport cv2 \nimport torch\nfrom PIL import Image\nfrom collections import defaultdict\nfrom tqdm import tqdm_notebook as tqdm\nimport sys\nimport copy\nimport scipy\nimport time\nimport tensorflow as tf\nimport efficientnet.tfkeras as efn\nfrom pympler import asizeof\n\nfrom hpacellseg.cellsegmentator import *\nfrom hpacellseg import cellsegmentator, utils\n\nnp.set_printoptions(precision=4, suppress=True)\n\nsys.path.append(\"../input/puzzle-cam\")\nsys.path.append(\"../input/timm045\")\nimport timm\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gpus = tf.config.list_physical_devices('GPU')\nif gpus:\n  try:\n    # Currently, memory growth needs to be the same across GPUs\n    for gpu in gpus:\n      tf.config.experimental.set_virtual_device_configuration(\n        gpu,\n        [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=5600)])\n    \n      #tf.config.experimental.set_memory_growth(gpu, True)\n    logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n    print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n  except RuntimeError as e:\n    # Memory growth must be set before GPUs have been initialized\n    print(e)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nimport re\n\ndata_df_sample_submission = pd.read_csv('../input/hpa-single-cell-image-classification/sample_submission.csv')\n\ntest_files = os.listdir(\"../input/hpa-single-cell-image-classification/test\")\ncolor_list = [\"_red.png\", \"_green.png\", \"_yellow.png\", \"_blue.png\"]\ntest_files_names =  [re.sub(r'|'.join(map(re.escape, color_list)), '', elem) for elem in test_files]\ntest_files_names = list(set(test_files_names))\n\nd = []\nfor i, file in enumerate(test_files_names):\n    img = cv2.imread(f\"../input/hpa-single-cell-image-classification/test/{file}_red.png\")\n    height, width, channels = img.shape\n    d.append({\n        \"ID\" : file,\n        \"ImageWidth\" : width,\n        \"ImageHeight\": height,\n        \"PredictionString\" : \"0 1 eNoLCAgIMAEABJkBdQ==\"\n    })\n    if i%50 == 0:\n        print(i)\n\ndf_from_files = pd.DataFrame(d).sort_values(\"ID\").reset_index(drop=True)\nassert all(df_from_files == data_df_sample_submission), \"Dataframes do not match\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH_TEST = \"../input/hpa-single-cell-image-classification/test\"\nMODEL_PATH = \"../input/models-hpa/\"\n\nMODELS_LIST_EFFNET = [\n                      \"efficientnet-b4_rgby_lr_0.001_ADAM_steplr_g085_focal1_g1.0_resize640_mediumaug_3.pth\",\n                      \"efficientnet-b4_rgby_lr_0.0015_ADAM_focal1_g1.0_resize640_10pcTest_HEAVY_AUG_E3_F0.pth\",\n                      #\"efficientnet-b4_rgby_lr_0.001_ADAM_steplr_g085_focal1_g1.0_resize640_mediumaug_E5_F0.pth\",\n                      #\"efficientnet-b4_rgby_lr_0.001_ADAM_steplr_g085_focal1_g1.0_resize640_mediumaug_E4_F1.pth\",\n                      #\"efficientnet-b4_rgby_lr_0.001_ADAM_steplr_g085_focal1_g1.0_resize640_mediumaug_E4_F2.pth\",\n                      #\"efficientnet-b4_rgby_lr_0.001_ADAM_steplr_g085_focal1_g1.0_resize640_mediumaug_E4_F3.pth\",\n                      \"efficientnet-b4_rgby_lr_0.001_ADAM_steplr_g085_focal1_g1.0_resize640_mediumaug_E4_F4.pth\",\n                      #\"efficientnet-b4_rgby_lr_0.003_SGD_polyoptim_focal1_g1.0_resize640_mediumaug_folds_E4_F1.pth\"\n                      ]\n                      \nMODELS_LIST_RESNEST = [\n                       \"resnest101_rgby_lr_0.002_SGD_polyoptim_focal1_g1.0_scheduler_largedataset_resize640_mediumaug_3.pth\",\n                       #\"resnest101_rgby_lr_0.002_SGD_polyoptim_focal1_g1.0_resize640_mediumaug_folds_E3_F0.pth\",\n                       \"resnest101_rgby_lr_0.002_SGD_focal1_g1.0_resize640_5pcTest_BS8_E3_F0.pth\",\n                       #\"resnest101_rgby_lr_0.002_SGD_polyoptim_focal1_g1.0_resize640_mediumaug_folds_E3_F2.pth\",\n                       #\"resnest101_rgby_lr_0.002_SGD_polyoptim_focal1_g1.0_resize640_mediumaug_folds_E3_F3.pth\",\n                       #\"resnest101_rgby_lr_0.002_SGD_polyoptim_focal1_g1.0_resize640_mediumaug_folds_E3_F4.pth\"\n                      ]       \n\nMODEL_LABELS_EFF = \"../input/hpa-tensorflow-models/model_green.06-0.07.h5\"\n#MODEL_LABELS_EFF = \"../input/hpa-tensorflow-models/model_green_large_dataset.h5\"\nMODEL_LABELS_RN = \"../input/hpa-tensorflow-models/model_rgb_resnext101.09-0.10.h5\"\nMODEL_LABELS_VIT = \"../input/hpa-tensorflow-models/ggg_ViTB16_RedPlat_ADAMW_BCE_EPOCH12-VAL0.0957.h5\"\n\n\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_df = data_df_sample_submission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    debug=False\n    verbose = False\n    num_workers=8\n    model_name_effnet = 'efficientnet-b4' #'resnest101' #resnest50\n    model_name_resnest = 'resnest101'\n    size=640 #640\n    seed=2002\n    classes = 19\n    color_mode = \"rgby\"\n    resnest = True\n    effnet = True\n    extra_model_for_labels = True\n    extra_model_is_tf = True\n    only_green_extra_model = True\n    color_mode_image_level = \"rgb\"\n    split = [0.6, 0.4, 0] #mask_probas, img_level_model, mask_model\n    size_seg = None\n    split_image_level = [0.33, 0.33, 0.34, 0] #effnet ,resnest ,vit, densenet ::: image level\n    split_cam_level = [0.5, 0.5] #effnet cam level, resnest cam level\n    split_sigmoid_graboost = [0.5, 0.5]\n    sigmoid_factor = 2.0\n    sigmoid_move = 0.2\n    is_demo = len(data_df)==559\n    \nif CFG.is_demo:\n    data_df = data_df[:10]\n\nbatch_size_ = 4","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations.pytorch\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\nfrom albumentations import (\n    Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip, \n    RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout, \n    IAAAdditiveGaussianNoise, Transpose, HueSaturationValue, CoarseDropout\n    )\n\ndataset_mean = [0.0994, 0.0466, 0.0606, 0.0879]\ndataset_std = [0.1406, 0.0724, 0.1541, 0.1264]\n\ndef get_transforms(*, data_type):\n    if data_type == 'valid':\n        return Compose([\n            Resize(CFG.size, CFG.size),\n            ToTensorV2(),\n        ])\n\n    elif data_type == 'test_green_model':\n      return Compose([\n            Resize(600, 600),\n            #ToTensorV2(),\n        ])\n    elif data_type == 'test_green_model_torch':\n      return Compose([\n            Resize(CFG.size, CFG.size),\n            ToTensorV2(),\n        ])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_RGBY_image(image_id, path, mode=\"cam\", image_size=None): \n    if mode == \"green_model\":\n      green = read_img_scale255(image_id,  \"green\",path, image_size)\n      stacked_images = np.transpose(np.array([green, green, green]), (1,2,0))\n      return stacked_images\n\n    if mode==\"cam\":\n      red = read_img(image_id, \"red\", path, image_size)\n      green = read_img(image_id,  \"green\",path, image_size)\n      blue = read_img(image_id,  \"blue\",path, image_size)\n      yellow = read_img(image_id,  \"yellow\",path, image_size)\n    \n      if CFG.color_mode == \"rgby\":\n        stacked_images = np.transpose(np.array([red, green, blue,yellow]), (1,2,0))\n      else:\n        stacked_images = np.transpose(np.array([red, green, blue]), (1,2,0))\n      return stacked_images\n\n  \ndef read_img(image_id, color, path, image_size=None):\n    filename = f'{path}/{image_id}_{color}.png'\n    assert os.path.exists(filename), f'not found {filename}'\n    img = cv2.imread(filename, cv2.IMREAD_UNCHANGED)\n    if image_size is not None:\n        img = cv2.resize(img, (image_size, image_size))\n    \n    if img.max() > 255:\n        img_max = img.max()\n        img = (img/255).astype('uint8')\n    \n    return img\n\ndef read_img_scale255(image_id, color, path, image_size=None):\n    filename = f'{path}/{image_id}_{color}.png'\n    assert os.path.exists(filename), f'not found {filename}'\n    img = cv2.imread(filename, cv2.IMREAD_UNCHANGED)\n    if image_size is not None:\n        img = cv2.resize(img, (image_size, image_size))\n    if img.max() > 255:\n        img_max = img.max()\n    img = (img/255).astype('uint8')/255\n    \n    return img\n\ndef one_hot_embedding(label, classes):\n    vector = np.zeros((classes), dtype = np.float32)\n    if len(label) > 0:\n        vector[label] = 1.\n    return vector\n\nclass HPADataset_Test(Dataset):\n    def __init__(self, ids, path=None, transforms=None, mode=\"cam\"):\n      self.ids = ids\n      self.transforms = transforms\n      self.mode = mode\n      self.path = path\n    \n    def __len__(self):\n      return len(self.ids)\n    \n    def __getitem__(self, idx):\n      _ids = self.ids.iloc[idx]\n      image = load_RGBY_image(_ids, self.path, self.mode)\n\n      if self.transforms:\n        augmented = self.transforms(image=image)\n        image = augmented['image']\n        #image = tfms.ToPILImage()(image)\n\n      return image, _ids","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_decoder(with_labels=True, target_size=(300, 300), ext='jpg'):\n    def decode(path):\n        if CFG.color_mode_image_level == \"ggg\":\n            file_bytes = tf.io.read_file(path + \"_green.png\")\n            if ext == 'png':\n                img = tf.image.decode_png(file_bytes, channels=3)\n            elif ext in ['jpg', 'jpeg']:\n                img = tf.image.decode_jpeg(file_bytes, channels=3)\n            else:\n                raise ValueError(\"Image extension not supported\")\n\n            img = tf.cast(img, tf.float32) / 255.0\n            img = tf.image.resize(img, target_size)\n\n            return img\n        if CFG.color_mode_image_level == \"rgb\":\n            r = tf.io.read_file(path + \"_red.png\")\n            g = tf.io.read_file(path + \"_green.png\")\n            b = tf.io.read_file(path + \"_blue.png\")\n\n            red = tf.io.decode_png(r, channels=1)\n            blue = tf.io.decode_png(g, channels=1)\n            green = tf.io.decode_png(b, channels=1)\n\n            red = tf.image.resize(red, target_size)\n            blue = tf.image.resize(blue, target_size)\n            green = tf.image.resize(green, target_size)\n\n            img = tf.stack([red, green, blue], axis=-1)\n            img = tf.squeeze(img)\n            img = tf.image.convert_image_dtype(img, tf.float32) / 255\n            return img\n\n    def decode_with_labels(path, label):\n        return decode(path), label\n\n    return decode_with_labels if with_labels else decode\n\ndef build_augmenter(with_labels=True):\n    def augment(img):\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_flip_up_down(img)\n        return img\n\n    def augment_with_labels(img, label):\n        return augment(img), label\n\n    return augment_with_labels if with_labels else augment\n\ndef build_dataset_tf(paths, labels=None, bsize=32, cache=True,\n                      decode_fn=None, augment_fn=None,\n                      augment=True, repeat=True, shuffle=1024, img_size=300,\n                      cache_dir=\"\"):\n        if cache_dir != \"\" and cache is True:\n            os.makedirs(cache_dir, exist_ok=True)\n\n        if decode_fn is None:\n            decode_fn = build_decoder(labels is not None, target_size=(img_size, img_size))\n\n        if augment_fn is None:\n            augment_fn = build_augmenter(labels is not None)\n\n        AUTO = tf.data.experimental.AUTOTUNE\n        slices = paths if labels is None else (paths, labels)\n\n        dset = tf.data.Dataset.from_tensor_slices(slices)\n        dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n        dset = dset.cache(cache_dir) if cache else dset\n        dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset\n        dset = dset.repeat() if repeat else dset\n        dset = dset.shuffle(shuffle) if shuffle else dset\n        dset = dset.batch(bsize).prefetch(AUTO)\n\n        return dset\n\n\ntest_paths = PATH_TEST + \"/\" + data_df['ID']\n\ntest_decoder_600 = build_decoder(with_labels=False, target_size=(600, 600))\ntest_decoder_384 = build_decoder(with_labels=False, target_size=(384, 384))\n\nCFG.color_mode_image_level = \"ggg\"\ndtest_tf_green_600 = build_dataset_tf(\n        test_paths, bsize=batch_size_, repeat=False, \n        shuffle=False, augment=False, cache=False,\n        decode_fn=test_decoder_600\n    )\nCFG.color_mode_image_level = \"rgb\"\ndtest_tf_rgb_600 = build_dataset_tf(\n        test_paths, bsize=batch_size_, repeat=False, \n        shuffle=False, augment=False, cache=False,\n        decode_fn=test_decoder_600\n    )\nCFG.color_mode_image_level = \"ggg\"\ndtest_tf_ggg_384 = build_dataset_tf(\n        test_paths, bsize=batch_size_, repeat=False, \n        shuffle=False, augment=False, cache=False,\n        decode_fn=test_decoder_384\n    )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass Yield_Images_Dataset(Dataset):\n    def __init__(self, csv_file, root=PATH_TEST, transform=None):\n        self.images_df = csv_file\n        self.transform = transform\n        self.root = root\n\n    def __len__(self):\n        return len(self.images_df)\n\n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n        \n        _id = self.images_df[\"ID\"].iloc[idx]\n        r = os.path.join(self.root, f'{_id}_red.png')\n        y = os.path.join(self.root, f'{_id}_yellow.png')\n        b = os.path.join(self.root, f'{_id}_blue.png')\n\n        r = cv2.imread(r, 0)\n        y = cv2.imread(y, 0)\n        b = cv2.imread(b, 0)\n\n        #don't resize\n        size = r.shape[0]\n        if CFG.size_seg == None:\n            ryb_image = np.stack((r, y, b), axis=2)/255.\n            blue_image = b/255.\n\n            return blue_image, ryb_image, size, _id\n\n        if size != CFG.size_seg:\n            blue_image = cv2.resize(b, (CFG.size_seg, CFG.size_seg))/255.\n            ryb_image = np.stack((r, y, b), axis=2)\n            ryb_image = cv2.resize(ryb_image, (CFG.size_seg, CFG.size_seg))/255.\n        else:\n            ryb_image = np.stack((r, y, b), axis=2)/255.\n            blue_image = b/255.\n\n\n        return blue_image, ryb_image, size, _id\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"Package for loading and running the nuclei and cell segmentation models programmaticly.\"\"\"\nimport os\nimport sys\n\nimport cv2\nimport imageio\nimport numpy as np\nimport torch\nimport torch.nn\nimport torch.nn.functional as F\nfrom skimage import transform, util\n\nfrom hpacellseg.constants import (MULTI_CHANNEL_CELL_MODEL_URL,\n                                  NUCLEI_MODEL_URL, TWO_CHANNEL_CELL_MODEL_URL)\nfrom hpacellseg.utils import download_with_url\n\nNORMALIZE = {\"mean\": [124 / 255, 117 / 255, 104 / 255], \"std\": [1 / (0.0167 * 255)] * 3}\n\n\nclass CellSegmentator(object):\n    \"\"\"Uses pretrained DPN-Unet models to segment cells from images.\"\"\"\n\n    def __init__(\n            self,\n            nuclei_model=\"./nuclei_model.pth\",\n            cell_model=\"./cell_model.pth\",\n            model_width_height=None,\n            device=\"cuda\",\n            multi_channel_model=True,\n            return_without_scale_restore=False,\n            scale_factor=0.25,\n            padding=False\n    ):\n\n        if device != \"cuda\" and device != \"cpu\" and \"cuda\" not in device:\n            raise ValueError(f\"{device} is not a valid device (cuda/cpu)\")\n        if device != \"cpu\":\n            try:\n                assert torch.cuda.is_available()\n            except AssertionError:\n                print(\"No GPU found, using CPU.\", file=sys.stderr)\n                device = \"cpu\"\n        self.device = device\n\n        if isinstance(nuclei_model, str):\n            if not os.path.exists(nuclei_model):\n                print(\n                    f\"Could not find {nuclei_model}. Downloading it now\",\n                    file=sys.stderr,\n                )\n                download_with_url(NUCLEI_MODEL_URL, nuclei_model)\n            nuclei_model = torch.load(\n                nuclei_model, map_location=torch.device(self.device)\n            )\n        if isinstance(nuclei_model, torch.nn.DataParallel) and device == \"cpu\":\n            nuclei_model = nuclei_model.module\n\n        self.nuclei_model = nuclei_model.to(self.device)\n\n        self.multi_channel_model = multi_channel_model\n        if isinstance(cell_model, str):\n            if not os.path.exists(cell_model):\n                print(\n                    f\"Could not find {cell_model}. Downloading it now\", file=sys.stderr\n                )\n                if self.multi_channel_model:\n                    download_with_url(MULTI_CHANNEL_CELL_MODEL_URL, cell_model)\n                else:\n                    download_with_url(TWO_CHANNEL_CELL_MODEL_URL, cell_model)\n            cell_model = torch.load(cell_model, map_location=torch.device(self.device))\n        self.cell_model = cell_model.to(self.device)\n        self.model_width_height = model_width_height\n        self.return_without_scale_restore = return_without_scale_restore\n        self.scale_factor = scale_factor\n        self.padding = padding\n\n    def _image_conversion(self, images):\n\n        microtubule_imgs, er_imgs, nuclei_imgs = images\n        if self.multi_channel_model:\n            if not isinstance(er_imgs, list):\n                raise ValueError(\"Please speicify the image path(s) for er channels!\")\n        else:\n            if not er_imgs is None:\n                raise ValueError(\n                    \"second channel should be None for two channel model predition!\"\n                )\n\n        if not isinstance(microtubule_imgs, list):\n            raise ValueError(\"The microtubule images should be a list\")\n        if not isinstance(nuclei_imgs, list):\n            raise ValueError(\"The microtubule images should be a list\")\n\n        if er_imgs:\n            if not len(microtubule_imgs) == len(er_imgs) == len(nuclei_imgs):\n                raise ValueError(\"The lists of images needs to be the same length\")\n        else:\n            if not len(microtubule_imgs) == len(nuclei_imgs):\n                raise ValueError(\"The lists of images needs to be the same length\")\n\n        if not all(isinstance(item, np.ndarray) for item in microtubule_imgs):\n            microtubule_imgs = [\n                os.path.expanduser(item) for _, item in enumerate(microtubule_imgs)\n            ]\n            nuclei_imgs = [\n                os.path.expanduser(item) for _, item in enumerate(nuclei_imgs)\n            ]\n\n            microtubule_imgs = list(\n                map(lambda item: imageio.imread(item), microtubule_imgs)\n            )\n            nuclei_imgs = list(map(lambda item: imageio.imread(item), nuclei_imgs))\n            if er_imgs:\n                er_imgs = [os.path.expanduser(item) for _, item in enumerate(er_imgs)]\n                er_imgs = list(map(lambda item: imageio.imread(item), er_imgs))\n\n        if not er_imgs:\n            er_imgs = [\n                np.zeros(item.shape, dtype=item.dtype)\n                for _, item in enumerate(microtubule_imgs)\n            ]\n        cell_imgs = list(\n            map(\n                lambda item: np.dstack((item[0], item[1], item[2])),\n                list(zip(microtubule_imgs, er_imgs, nuclei_imgs)),\n            )\n        )\n\n        return cell_imgs\n\n    def _pad(self, image):\n          rows, cols = image.shape[:2]\n          self.scaled_shape = rows, cols\n          img_pad= cv2.copyMakeBorder(\n                    image,\n                    32,\n                    (32 - rows % 32),\n                    32,\n                    (32 - cols % 32),\n                    cv2.BORDER_REFLECT,\n                )\n          return img_pad\n\n    def pred_nuclei(self, images):\n        \n\n        def _preprocess(images):\n            if isinstance(images[0], str):\n                raise NotImplementedError('Currently the model requires images as numpy arrays, not paths.')\n                # images = [imageio.imread(image_path) for image_path in images]\n            self.target_shapes = [image.shape for image in images]\n            #print(images.shape)\n            #resize like in original implementation with https://scikit-image.org/docs/dev/api/skimage.transform.html#skimage.transform.resize\n            if self.model_width_height:\n                images = np.array([transform.resize(image, (self.model_width_height,self.model_width_height)) \n                                  for image in images])\n            else:\n                images = [transform.rescale(image, self.scale_factor) for image in images]\n\n            if self.padding:\n              images = [self._pad(image) for image in images]\n\n            nuc_images = np.array([np.dstack((image[..., 2], image[..., 2], image[..., 2])) if len(image.shape) >= 3\n                                   else np.dstack((image, image, image)) for image in images])\n            \n            nuc_images = nuc_images.transpose([0, 3, 1, 2])\n            #print(\"nuc\", nuc_images.shape)\n\n            return nuc_images\n\n        def _segment_helper(imgs):\n            with torch.no_grad():\n                mean = torch.as_tensor(NORMALIZE[\"mean\"], device=self.device)\n                std = torch.as_tensor(NORMALIZE[\"std\"], device=self.device)\n                imgs = torch.tensor(imgs).float()\n                imgs = imgs.to(self.device)\n                imgs = imgs.sub_(mean[:, None, None]).div_(std[:, None, None])\n\n                imgs = self.nuclei_model(imgs)\n                imgs = F.softmax(imgs, dim=1)\n                return imgs\n\n        preprocessed_imgs = _preprocess(images)\n        predictions = _segment_helper(preprocessed_imgs)\n        predictions = predictions.to(\"cpu\").numpy()\n        #dont restore scaling, just save and scale later ...\n        predictions = [self._restore_scaling(util.img_as_ubyte(pred), target_shape)\n                       for pred, target_shape in zip(predictions, self.target_shapes)]\n        return predictions\n\n    def _restore_scaling(self, n_prediction, target_shape):\n        \"\"\"Restore an image from scaling and padding.\n        This method is intended for internal use.\n        It takes the output from the nuclei model as input.\n        \"\"\"\n        n_prediction = n_prediction.transpose([1, 2, 0])\n        if self.padding:\n          n_prediction = n_prediction[\n                32 : 32 + self.scaled_shape[0], 32 : 32 + self.scaled_shape[1], ...\n            ]\n        n_prediction[..., 0] = 0\n        if not self.return_without_scale_restore:\n            n_prediction = cv2.resize(\n                n_prediction,\n                (target_shape[0], target_shape[1]),\n                #try INTER_NEAREST_EXACT\n                interpolation=cv2.INTER_AREA,\n            )\n        return n_prediction\n\n    def pred_cells(self, images, precombined=False):\n\n        def _preprocess(images):\n            self.target_shapes = [image.shape for image in images]\n            for image in images:\n                if not len(image.shape) == 3:\n                    raise ValueError(\"image should has 3 channels\")\n            #resize like in original implementation with https://scikit-image.org/docs/dev/api/skimage.transform.html#skimage.transform.resize\n            if self.model_width_height:\n                images = np.array([transform.resize(image, (self.model_width_height,self.model_width_height)) \n                                  for image in images])\n            else:\n                images = np.array([transform.rescale(image, self.scale_factor, multichannel=True) for image in images])\n\n            if self.padding:\n              images = np.array([self._pad(image) for image in images])\n\n            cell_images = images.transpose([0, 3, 1, 2])\n\n            return cell_images\n\n        def _segment_helper(imgs):\n            with torch.no_grad():\n                mean = torch.as_tensor(NORMALIZE[\"mean\"], device=self.device)\n                std = torch.as_tensor(NORMALIZE[\"std\"], device=self.device)\n                imgs = torch.tensor(imgs).float()\n                imgs = imgs.to(self.device)\n                imgs = imgs.sub_(mean[:, None, None]).div_(std[:, None, None])\n                imgs = self.cell_model(imgs)\n                imgs = F.softmax(imgs, dim=1)\n                return imgs\n\n        if not precombined:\n            images = self._image_conversion(images)\n        preprocessed_imgs = _preprocess(images)\n        predictions = _segment_helper(preprocessed_imgs)\n        predictions = predictions.to(\"cpu\").numpy()\n        predictions = [self._restore_scaling(util.img_as_ubyte(pred), target_shape)\n                       for pred, target_shape in zip(predictions, self.target_shapes)]\n        return predictions","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nIMAGE_SIZES = [1728, 2048, 3072, 4096]\npredict_df_1728 = data_df[data_df.ImageWidth==IMAGE_SIZES[0]]\npredict_df_2048 = data_df[data_df.ImageWidth==IMAGE_SIZES[1]]\npredict_df_3072 = data_df[data_df.ImageWidth==IMAGE_SIZES[2]]\npredict_df_4096 = data_df[data_df.ImageWidth==IMAGE_SIZES[3]]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert len(predict_df_1728) + len(predict_df_2048) + len(predict_df_3072) + len(predict_df_4096) == len(data_df), \"IMAGE SIZE DFS DONT MATCH SAMPLE SUBMISSION\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUC_MODEL = \"../input/hpacellsegmentatormodelweights/dpn_unet_nuclei_v1.pth\"\nCELL_MODEL = \"../input/hpacellsegmentatormodelweights/dpn_unet_cell_3ch_v1.pth\"\nsegmentator_even_faster = CellSegmentator(\n    NUC_MODEL,\n    CELL_MODEL,\n    device=\"cuda\",\n    multi_channel_model=True,\n    padding=True,\n    return_without_scale_restore=True\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"###############WITH DATALODER################\nyield_ims_1728 = Yield_Images_Dataset(predict_df_1728)\nyield_ims_2048 = Yield_Images_Dataset(predict_df_2048)\nyield_ims_3072 = Yield_Images_Dataset(predict_df_3072)\nyield_ims_4096 = Yield_Images_Dataset(predict_df_4096)\n\ndataloader_ims_seg_1728 = DataLoader(yield_ims_1728, batch_size=24, #was 24\n                        shuffle=False, num_workers=0)\n\ndataloader_ims_seg_2048 = DataLoader(yield_ims_2048, batch_size=12, #was 24\n                        shuffle=False, num_workers=0)\n\ndataloader_ims_seg_3072 = DataLoader(yield_ims_3072, batch_size=3, #was 24\n                        shuffle=False, num_workers=0)\n\ndataloader_ims_seg_4096 = DataLoader(yield_ims_4096, batch_size=3, #was 24\n                        shuffle=False, num_workers=0)\n\ndataloaders_all_sizes = [dataloader_ims_seg_1728, dataloader_ims_seg_2048, dataloader_ims_seg_3072, dataloader_ims_seg_4096]\n\nstart_time = time.time()\neven_faster_outputs = []\noutput_ids = []\nbatch_size = 24\nsizes_list = []\nim_proc = 0\nfor i, dataloader_ims_seg in enumerate(dataloaders_all_sizes):\n    print(f\"GETTING IMAGE SIZES: {IMAGE_SIZES[i]}, BATCHES: {len(dataloader_ims_seg)}\")\n    print\n    for blue_images, ryb_images, sizes, _ids in dataloader_ims_seg:\n\n        print(f\"SEGMENT COUNT: {im_proc}\")\n\n        blue_batch = blue_images.numpy()\n        ryb_batch = ryb_images.numpy()\n\n        #print(blue_batch.shape)\n        nuc_segmentations = segmentator_even_faster.pred_nuclei(blue_batch)\n        cell_segmentations = segmentator_even_faster.pred_cells(ryb_batch, precombined=True)\n\n        for data_id, nuc_seg, cell_seg, size in zip(_ids, nuc_segmentations, cell_segmentations, sizes):\n            _, cell = utils.label_cell(nuc_seg, cell_seg)\n            even_faster_outputs.append(np.ubyte(cell))\n            output_ids.append(data_id)\n            sizes_list.append(size.numpy())\n        im_proc += len(_ids)\n        #if im_proc > 20:\n          #break\n    del dataloader_ims_seg\n    print(time.time() - start_time)\n\ncell_masks_df = pd.DataFrame(list(zip(output_ids, even_faster_outputs,sizes_list)),\n                             columns=[\"ID\", \"mask\", \"ori_size\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_masks_df = cell_masks_df.set_index('ID')\ncell_masks_df = cell_masks_df.reindex(index=data_df['ID'])\ncell_masks_df = cell_masks_df.reset_index()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.rcParams['figure.figsize'] = [10, 10]\nplt.imshow(cell_masks_df[\"mask\"][4])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del sizes_list\ndel even_faster_outputs\ndel output_ids\ndel segmentator_even_faster\ndel yield_ims_1728 \ndel yield_ims_2048 \ndel yield_ims_3072 \ndel yield_ims_4096\n\ndel dataloader_ims_seg_1728 \n\ndel dataloader_ims_seg_2048\n\ndel dataloader_ims_seg_3072 \n\ndel dataloader_ims_seg_4096\ndel dataloaders_all_sizes\n\nimport gc\nimport ctypes\n\nlibc = ctypes.CDLL(\"libc.so.6\")\nlibc.malloc_trim(0)\ngc.collect()\ntorch.cuda.empty_cache()\ntorch.cuda.empty_cache()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport ctypes\n\nlibc = ctypes.CDLL(\"libc.so.6\")\nlibc.malloc_trim(0)\ngc.collect()\ntorch.cuda.empty_cache()\ntorch.cuda.empty_cache()\n\n\ntorch.cuda.empty_cache()\ntorch.cuda.empty_cache()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = data_df[\"ID\"]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert list(X_test) == list(cell_masks_df[\"ID\"]), \"X_Test and cellmask dont match\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = HPADataset_Test(X_test, path=PATH_TEST, transforms=get_transforms(data_type='valid'), mode=\"cam\")\ntest_data_green_model = HPADataset_Test(X_test, path=PATH_TEST,transforms=get_transforms(data_type='valid'), mode=\"green_model\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from core.networks import *\nfrom core.datasets import *\n\nfrom tools.general.io_utils import *\nfrom tools.general.time_utils import *\nfrom tools.general.json_utils import *\n\nfrom tools.ai.log_utils import *\nfrom tools.ai.demo_utils import *\nfrom tools.ai.optim_utils import *\nfrom tools.ai.torch_utils import *\nfrom tools.ai.evaluate_utils import *\n\nfrom tools.ai.augment_utils import *\nfrom tools.ai.randaugment import *\n\nfrom matplotlib.pyplot import imshow\n\nfrom torchvision.transforms import ToTensor, ToPILImage\ndef swish(x, beta=1.0):\n    #https://paperswithcode.com/method/swish\n    return x * torch.sigmoid(beta*x)\n\ndef get_all_cams(batch_cam_scaled, model, scales, ims_per_batch):\n  bs = ims_per_batch\n  with torch.no_grad():\n        ori_w, ori_h = CFG.size, CFG.size\n        strided_up_size = (CFG.size, CFG.size)\n        all_scale_cams = torch.from_numpy(np.zeros((bs, len(scales), 19, CFG.size, CFG.size))).cuda()\n        all_scale_preds = torch.from_numpy(np.zeros((bs, len(scales), 19))).cuda()\n        \n###########################make this for resnest and efficient net at the same time (or at least for multiple states of the models...)#########################\n#######do scaling beforehand, do augmenting beforehand?#################\n        num_channels = 4\n        for i, images in enumerate(batch_cam_scaled):\n            #image_batch_pil = torch.from_numpy(np.zeros((bs, num_channels, round(ori_h*scale), round(ori_h*scale))))\n            #image_batch = copy.deepcopy(ori_image_batch)\n\n            #to PIL, resize, back to tensor, because F.interpolate works different from PIL BICUBIC (maybe F.interpolate is better, so try!)\n\n            #TODO: get them as PIL from Dataset to avoid transforming, then use this pipeline (also double check if the perfomance is the same)\n            \"\"\"\n            torch_pil_tfms = tfms.Compose([tfms.Resize([round(ori_h*scale), round(ori_h*scale)]), tfms.ToTensor()])\n            im = torch_pil_tfms(im)*255\n            Benchmark, first image, first four sigmoid logits: 2.55380799e-01 3.94755906e-02 2.51886857e-01 2.89894098e-01\n            \"\"\"\n            \n            #st = time.time()\n            #for j, im in enumerate(image_batch):\n              #im = ToPILImage()(im)\n              #im = im.resize((round(ori_w*scale), round(ori_h*scale)), resample=PIL.Image.BICUBIC)\n              #im = torchvision.transforms.functional.to_tensor(im)*255 #find a way to convert PIL to tensor without scaling (dividing everything by 255)\n              #image_batch_pil[j] = im\n            #print(f\"TIME FOR RESIZING WITH PIL {time.time() - st}\")\n            \n            #st = time.time()\n            #image_batch_resized = F.interpolate(image_batch, (round(ori_h*scale), round(ori_w*scale)), mode=\"bicubic\")\n            #print(f\"TIME FOR RESIZING WITH TORCH {time.time() - st}\") \n            #image_batch_resized = image_batch_pil.float()\n            #image_batch_augs_fl2 = image_batch_resized.flip(2)\n            #image_batch_augs_fl3 = image_batch_resized.flip(3)\n            #image_batch_augs_fl32 = torch.flip(image_batch_resized,(3,2))\n  \n            #concat to one vector: im1, im2, .., im1fl, im2fl, .., im1fl2, im2fl2, ...\n            #images = torch.cat([image_batch_resized, image_batch_augs_fl2, image_batch_augs_fl3, image_batch_augs_fl32], dim=0)\n            #images = images.cuda()\n\n            # inferenece\n            with torch.cuda.amp.autocast():\n                logits, features = model(images, with_cam=True)\n            #print(features.max(), features.min())\n            features = swish(features)\n\n            #reshape augmented images to im1, im2, ...\n            logits = logits.reshape(bs*4//bs//4, 4, bs, 19).mean(1).view(bs*4//4, 19)\n            all_scale_preds[:, i, :] = logits#.cpu()\n            \n            #deaugment features\n            features = torch.cat([features[0:bs], features[bs:bs*2].flip(2), features[bs*2:bs*3].flip(3), torch.flip(features[bs*3:bs*4],(3,2))])\n\n            #reshape augmented features to im1, im2, ...\n            size_feats = features.shape[-1]\n            features = features.reshape(bs*4//bs//4, 4, bs, 19, size_feats, size_feats).sum(1).view(bs*4//4, 19, size_feats, size_feats)\n            \n            #post-processing\n            ###IDEAS###\n            #normalize cams (cam = cam/max(cam))\n            #threshold cams (threshold CAMs --- cam[cam < 0.1] = 0)\n            #bicubic upsampling cam (mode=\"bicubic\")\n            #no ReLu on CAM (features = features)\n            #dont multiply cam by model probas?? (because they are actually multiplied already?)\n            cams = F.interpolate(features,( CFG.size, CFG.size), mode='bicubic', align_corners=False) #try bicubic here :)\n\n            #cams /= F.adaptive_avg_pool2d(cams, (1, 1)) + 1e-5\n\n\n            all_scale_cams[:, i, :, :, :] = cams#.cpu()\n\n        \n        all_logits = np.sum(all_scale_preds.detach().cpu().numpy(), axis=1)\n        all_cams = np.sum(all_scale_cams.detach().cpu().numpy(), axis=1)  #*1.5 #add factor to keep the numerical values high when lowering the scales amount\n        \n        print(\"CAMS DONE\")\n  return {\"hr_cams\": all_cams, \"logits\" : all_logits}\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_names = [\n'0-Nucleoplasm',\n'1-Nuclear membrane',\n'2-Nucleoli',\n'3-Nucleoli fibrillar center',\n'4-Nuclear speckles',\n'5-Nuclear bodies',\n'6-Endoplasmic reticulum',\n'7-Golgi apparatus',\n'8-Intermediate filaments',\n'9-Actin filaments',\n'10-Microtubules',\n'11-Mitotic spindle',\n'12-Centrosome',\n'13-Plasma membrane',\n'14-Mitochondria',\n'15-Aggresome',\n'16-Cytosol',\n'17-Vesicles + cytosolic patterns',\n'18-Negative'\n]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib.colors import LinearSegmentedColormap\n\nncolors = 256\ncolor_array = plt.get_cmap('jet')(range(ncolors))\ncolor_array[:,-1] = np.linspace(0.15,1.0,ncolors)\nmap_object = LinearSegmentedColormap.from_list(name='rainbow_alpha',colors=color_array)\nplt.register_cmap(cmap=map_object)\n#plt.rcParams['figure.figsize'] = [200, 25]\n\ndef sigmoid_factor(x, factor=1, move=0):\n  return 1 / (1 + np.exp(-factor*(x-move)))\n\n\ndef get_hrcams_vis(data, show_image, model, model_state, scales, ims_per_batch):\n    model.load_state_dict(model_state['model_state_dict'])           \n    all_cams_test = get_all_cams(data, model, scales, ims_per_batch)\n\n    pred_label = all_cams_test[\"logits\"]/len(scales) #there was an erroring dividing this by len(scales)+1, maybe it's important later!\n    print(\"---------------\")\n    sig_labels = sigmoid_factor(pred_label) \n    print(sig_labels)\n    #keep value of cams always in range as if there are only 3 scales (because GradBoost was trained on 3!)\n    all_hr_cams_test = all_cams_test[\"hr_cams\"]*(3/len(scales))\n    all_label_names_map = [label_names[i] for i in range(19)]\n    \n    if show_image:\n      plt.rcParams['figure.figsize'] = [200, 25]\n      plt.figure()\n      f, axarr = plt.subplots(ims_per_batch, 19)\n      for b in range(ims_per_batch):\n        for i, cam in enumerate(all_hr_cams_test[b]): \n          axarr[b, i].imshow(data[0][b, 0:3].cpu().permute(1, 2, 0)/255, interpolation=\"bicubic\") #image 0 is non augmented!\n          axarr[b, i].set_title(f\"{all_label_names_map[i]}::{sig_labels[b][i]}\")\n          axarr[b, i].imshow(cam.squeeze(), cmap=\"rainbow_alpha\",alpha=0.9)\n\n      plt.show()\n\n    return all_hr_cams_test, sig_labels\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import base64\nimport numpy as np\nfrom pycocotools import _mask as coco_mask\nimport typing as t\nimport zlib\n\ndef encode_binary_mask(mask: np.ndarray) -> t.Text:\n  \"\"\"Converts a binary mask into OID challenge encoding ascii text.\"\"\"\n\n  # check input mask --\n  if mask.dtype != np.bool:\n    raise ValueError(\n        \"encode_binary_mask expects a binary mask, received dtype == %s\" %\n        mask.dtype)\n\n  mask = np.squeeze(mask)\n  if len(mask.shape) != 2:\n    raise ValueError(\n        \"encode_binary_mask expects a 2d mask, received shape == %s\" %\n        mask.shape)\n\n  # convert input mask to expected COCO API input --\n  mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)\n  mask_to_encode = mask_to_encode.astype(np.uint8)\n  mask_to_encode = np.asfortranarray(mask_to_encode)\n\n  # RLE encode mask --\n  encoded_mask = coco_mask.encode(mask_to_encode)[0][\"counts\"]\n\n  # compress and base64 encoding --\n  binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)\n  base64_str = base64.b64encode(binary_str)\n  return base64_str\n\ndef get_all_encoded_cells(mask):\n  print(mask.shape)\n  cell_masks = []\n  for i in range(1, np.max(mask)+1):\n    enc_mask = encode_binary_mask((mask == i))\n    cell_masks.append(enc_mask)\n  return cell_masks","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import scipy\n\ndef resize_mask(mask):\n    resized_mask = resize_full_mask(mask, CFG.size)\n    cell_masks = []\n    for i in range(1, np.max(mask)+1):\n      cell_masks.append((resized_mask == i))\n\n    return cell_masks\n\ndef resize_full_mask(mask, size):\n    #zoom_factor = size / mask.shape[0]\n    #resized_mask = scipy.ndimage.zoom(mask, zoom_factor, order=0) #change zoom method....\n    resized_mask = cv2.resize(mask,(size,size),interpolation=cv2.INTER_NEAREST_EXACT)\n    return resized_mask\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_pred_string(mask_probas, cell_masks_fullsize_enc):\n  assert len(mask_probas) == len(cell_masks_fullsize_enc), \"Probas have different length than masks\"\n  string = \"\"\n  for enc_mask, mask_proba in zip(cell_masks_fullsize_enc, mask_probas):\n      for cls, proba in enumerate(mask_proba):\n        #print(cls, proba)\n        string += str(cls) + \" \" + str(proba) + \" \"  + enc_mask.decode(\"utf-8\") + \" \"\n\n  return string","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import skimage\nimport scipy\n\ndef show_seg_cells(data, cell_mask_list, batch_ids, batch_ims):\n    plt.rcParams['figure.figsize'] = [20, 10]\n    plt.figure()\n    f, axarr = plt.subplots(1, len(cell_mask_list))\n\n    cell_mask_list = copy.deepcopy(cell_mask_list)\n    for b, cell_mask in enumerate(cell_mask_list):\n          cell_mask = copy.deepcopy(cell_mask)\n\n          id = batch_ids[b]\n          resized_masks = cell_mask\n          resized_image = skimage.transform.resize(batch_ims[b, 0:3].permute(1, 2, 0), (CFG.size,CFG.size), order=1)  #image called for showing\n\n          for label, mask  in enumerate(resized_masks):\n              uint_img = np.array(mask*255).astype('uint8')\n              M = cv2.moments(uint_img)\n              cX = int(M[\"m10\"] / M[\"m00\"])\n              cY = int(M[\"m01\"] / M[\"m00\"])\n              axarr[b].text(cX, cY, label, fontsize=10,weight='bold', color=\"white\",bbox=dict(facecolor='black', edgecolor='none'))\n              axarr[b].contour(mask, 1, colors='cyan', linewidths=1)\n          axarr[b].set_title(str(id))\n          axarr[b].imshow(resized_image, interpolation=\"bicubic\")\n          \n\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Probas from Masks","metadata":{}},{"cell_type":"code","source":"import sklearn\nfrom sklearn import preprocessing\nimport pickle\n\n\nPATH_SCALER_GRADBOOST = \"../input/scaler-and-gradboost\"\n\nscaler_resnest0 = pickle.load(open(f\"{PATH_SCALER_GRADBOOST}/scaler_resnest0.pkl\", 'rb'))\nscaler_resnest1 = pickle.load(open(f\"{PATH_SCALER_GRADBOOST}/scaler_resnest1.pkl\", 'rb'))\nscaler_effnet0 = pickle.load(open(f\"{PATH_SCALER_GRADBOOST}/scaler_effnet0.pkl\", 'rb'))\nscaler_effnet1 = pickle.load(open(f\"{PATH_SCALER_GRADBOOST}/scaler_effnet1.pkl\", 'rb'))\nscaler_effnet2 = pickle.load(open(f\"{PATH_SCALER_GRADBOOST}/scaler_effnet2.pkl\", 'rb'))\n\nmodel_gradboost_resnest0 =  pickle.load(open(f\"{PATH_SCALER_GRADBOOST}/GradientBoostingRegressor_resnest0.pkl\", 'rb'))\nmodel_gradboost_resnest1 =  pickle.load(open(f\"{PATH_SCALER_GRADBOOST}/GradientBoostingRegressor_resnest1.pkl\", 'rb'))\nmodel_gradboost_effnet0 =  pickle.load(open(f\"{PATH_SCALER_GRADBOOST}/GradientBoostingRegressor_effnet0.pkl\", 'rb'))\nmodel_gradboost_effnet1 =  pickle.load(open(f\"{PATH_SCALER_GRADBOOST}/GradientBoostingRegressor_effnet1.pkl\", 'rb'))\nmodel_gradboost_effnet2 =  pickle.load(open(f\"{PATH_SCALER_GRADBOOST}/GradientBoostingRegressor_effnet2.pkl\", 'rb'))\n\ndef get_prob_from_cams_masks(cams, masks, labels, labels_from_labelmodel, verbose=True, typ=None):\n    \n    print(f\"GETTING MODEL {typ}\")\n    if typ==\"resnest0\":\n      scaler = scaler_resnest0\n      model_gradboost = model_gradboost_resnest0\n    if typ==\"resnest1\":\n      scaler = scaler_resnest1\n      model_gradboost = model_gradboost_resnest1\n    if typ==\"resnest2\":\n      scaler = scaler_resnest1\n      model_gradboost = model_gradboost_resnest1\n    if typ==\"effnet0\":\n      scaler = scaler_effnet0\n      model_gradboost = model_gradboost_effnet0\n    if typ==\"effnet1\":\n      scaler = scaler_effnet1\n      model_gradboost = model_gradboost_effnet1\n    if typ==\"effnet2\":\n      scaler = scaler_effnet2\n      model_gradboost = model_gradboost_effnet2\n    if typ==\"effnet3\":\n      scaler = scaler_effnet2\n      model_gradboost = model_gradboost_effnet2\n\n    masks_probas = np.zeros((len(masks), 19)) #shape: (n_masks, classes) -> probablities (products of CAM and Cell Mask) for every mask and class\n    for i, mask in enumerate(masks):\n      for label, cam in enumerate(cams):\n\n        cam_by_mask = np.multiply(mask, cam)\n        cam_mask_product = np.multiply(cam_by_mask, labels[label])#**0.7) \n                                                          #normalize different sizes of organellese\n        masks_probas[i, label] = np.sum(cam_mask_product)# * size_mask_organelles[label]\n\n      if verbose:\n        print(f\"MASK: {i} PROB-RAW: {masks_probas[i, :]}\") #add 50 to class \"negative\" ?\n        print(\"--------------------------------------\")\n\n    #scaling standardization\n    #try sklearn.preprocessing.RobustScaler\n    std_scaler = preprocessing.RobustScaler().fit(masks_probas.reshape(-1, 1))\n    \n    for i, mask in enumerate(masks):\n      std_scaled = std_scaler.transform(masks_probas[i, :].reshape(-1, 1))[:,0]\n      model_scaled = scaler.transform(masks_probas[i, :].reshape(-1, 1))[:,0]\n      if verbose:\n        print(f\"MASK: {i} STD-Scaled: {std_scaled}\")\n        print(f\"MASK: {i} Model-Scaled: {model_scaled}\")\n        print(\"--------------------------------------\")\n\n      sigmoid_probas = sigmoid_factor(std_scaled, factor=CFG.sigmoid_factor, move=CFG.sigmoid_move) #put that scaler in (0,1)!\n      gradboost_probas = model_gradboost.predict(model_scaled.reshape(-1, 1))\n      if verbose:\n        print(f\"MASK: {i} PROB-SIGMOID: {sigmoid_probas}\")\n        print(f\"MASK: {i} PROB-GradBoost: {gradboost_probas}\")\n        print(\"--------------------------------------\")\n\n      if CFG.extra_model_for_labels:\n        masks_probas[i, :] = sigmoid_probas*CFG.split_sigmoid_graboost[0] + gradboost_probas*CFG.split_sigmoid_graboost[1]\n        #weight each final label output with the according label output from the model\n        masks_probas[i, :] = CFG.split[0] * masks_probas[i, :] + CFG.split[1] * labels_from_labelmodel + CFG.split[2] * labels\n\n      if verbose:\n        print(f\"MASK: {i} PROB-WITH-LABELMODEL: {masks_probas[i, :]}\")\n        #print(\"LABELS FROM LABEL MODEL WAS\")\n        #print(labels_from_labelmodel)\n\n    return masks_probas","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from core.abc_modules import ABC_Model\nfrom efficientnet_pytorch import EfficientNet\n\nclass Classifier_EffNet(nn.Module, ABC_Model):\n    def __init__(self, backbone, num_classes=19):\n        super(Classifier_EffNet, self).__init__()\n        self.enet = EfficientNet.from_name(backbone, num_classes=num_classes, in_channels=3, include_top=False)\n\n        dict_sizes = {\n            'efficientnet-b0' : 1280,\n            'efficientnet-b1' : 1280,\n            'efficientnet-b2' : 1408,\n            'efficientnet-b3' : 1536,\n            'efficientnet-b4' : 1792,\n            'efficientnet-b5' : 2048\n            }\n        size_conv2d = dict_sizes[backbone]\n\n        self.classifier = nn.Conv2d(size_conv2d, num_classes, 1, bias=False)\n        self.num_classes = num_classes\n\n        self.initialize([self.classifier])\n    \n    def forward(self, x, with_cam=False):\n\n        x = self.enet.extract_features(x)\n       \n        if with_cam:\n            features = self.classifier(x)\n            logits = self.global_average_pooling_2d(features)\n            return logits, features\n        else:\n            x = self.global_average_pooling_2d(x, keepdims=True) \n            logits = self.classifier(x).view(-1, self.num_classes)\n            return logits\n            \nclass Classifier_EffNet_GREEN(nn.Module, ABC_Model):\n    def __init__(self, backbone, num_classes=19):\n        super(Classifier_EffNet_GREEN, self).__init__()\n        self.enet = EfficientNet.from_name(backbone, num_classes=num_classes, in_channels=3, include_top=False)\n\n        dict_sizes = {\n            'efficientnet-b0' : 1280,\n            'efficientnet-b1' : 1280,\n            'efficientnet-b2' : 1408,\n            'efficientnet-b3' : 1536,\n            'efficientnet-b4' : 1792,\n            'efficientnet-b5' : 2048,\n            'efficientnet-b7' : 2560,\n            }\n\n        self.dense = nn.Linear(dict_sizes[backbone],19)\n        #self.sigmoid = nn.Sigmoid()\n        self.initialize([self.dense])\n\n    def forward(self, x, with_cam=False):\n        x = self.enet.extract_features(x)\n        x = self.global_average_pooling_2d(x)\n        logits = self.dense(x)\n        #logits = self.sigmoid(logits)\n        return logits\n\nimport math\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom torchvision import models\nimport torch.utils.model_zoo as model_zoo\n\nfrom core.arch_resnet import resnet\nfrom core.arch_resnest import resnest\nfrom core.abc_modules import ABC_Model\n\nfrom core.deeplab_utils import ASPP, Decoder\nfrom core.aff_utils import PathIndex\nfrom core.puzzle_utils import tile_features, merge_features\n\nfrom tools.ai.torch_utils import resize_for_tensors\n\n#######################################################################\n# Normalization\n#######################################################################\n#from core.sync_batchnorm.sync_batchnorm.batchnorm import SynchronizedBatchNorm2d\n\nclass FixedBatchNorm(nn.BatchNorm2d):\n    def forward(self, x):\n        return F.batch_norm(x, self.running_mean, self.running_var, self.weight, self.bias, training=False, eps=self.eps)\n\ndef group_norm(features):\n    return nn.GroupNorm(4, features)\n#######################################################################\n\nclass Backbone(nn.Module, ABC_Model):\n    def __init__(self, model_name, num_classes=20, mode='fix', segmentation=False):\n        super().__init__()\n\n        self.mode = mode\n\n        if self.mode == 'fix': \n            self.norm_fn = FixedBatchNorm\n        else:\n            self.norm_fn = nn.BatchNorm2d\n        \n        if 'resnet' in model_name:\n            self.model = resnet.ResNet(resnet.Bottleneck, resnet.layers_dic[model_name], strides=(2, 2, 2, 1), batch_norm_fn=self.norm_fn)\n\n            state_dict = model_zoo.load_url(resnet.urls_dic[model_name])\n            state_dict.pop('fc.weight')\n            state_dict.pop('fc.bias')\n\n            self.model.load_state_dict(state_dict)\n        else:\n            if segmentation:\n                dilation, dilated = 4, True\n            else:\n                dilation, dilated = 2, False\n\n            self.model = eval(\"resnest.\" + model_name)(pretrained=False, dilated=dilated, dilation=dilation, norm_layer=self.norm_fn)\n\n            del self.model.avgpool\n            del self.model.fc\n\n        self.stage1 = nn.Sequential(self.model.conv1, \n                                    self.model.bn1, \n                                    self.model.relu, \n                                    self.model.maxpool)\n        self.stage2 = nn.Sequential(self.model.layer1)\n        self.stage3 = nn.Sequential(self.model.layer2)\n        self.stage4 = nn.Sequential(self.model.layer3)\n        self.stage5 = nn.Sequential(self.model.layer4)\n\nclass Classifier(Backbone):\n    def __init__(self, model_name, num_classes=20, mode='fix'):\n        super().__init__(model_name, num_classes, mode)\n        \n        self.classifier = nn.Conv2d(2048, num_classes, 1, bias=False)\n        self.num_classes = num_classes\n\n        self.initialize([self.classifier])\n    \n    def forward(self, x, with_cam=False):\n        x = self.stage1(x)\n        x = self.stage2(x)\n        x = self.stage3(x)\n        x = self.stage4(x)\n        x = self.stage5(x)\n        \n        if with_cam:\n            features = self.classifier(x)\n            logits = self.global_average_pooling_2d(features)\n            return logits, features\n        else:\n            x = self.global_average_pooling_2d(x, keepdims=True) \n            logits = self.classifier(x).view(-1, self.num_classes)\n            return logits","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get all Image Level Labels","metadata":{}},{"cell_type":"code","source":"def get_separate_labels(ims, model, model_for_labels_state, ims_per_batch, verbose=False):\n    bs = ims_per_batch\n    model.load_state_dict(model_for_labels_state[\"model_state_dict\"])\n    with torch.no_grad():\n      image_batch = copy.deepcopy(ims)\n      image_batch = image_batch.float()\n      image_batch_augs_fl2 = image_batch.flip(2)\n      image_batch_augs_fl3 = image_batch.flip(3)\n      image_batch_augs_fl32 = torch.flip(image_batch,(3,2))\n      images = torch.cat([image_batch, image_batch_augs_fl2, image_batch_augs_fl3, image_batch_augs_fl32], dim=0)\n      images = images.cuda()\n      print(images.shape)\n      with torch.cuda.amp.autocast():\n          logits = model(images, with_cam=True)\n      \n      logits = logits.reshape(bs*4//bs//4, 4, bs, 19).mean(1).view(bs*4//4, 19)\n    labels = torch.sigmoid(logits)\n    if verbose:\n      print(\"LABELS FROM LABELS MODEL TORCH\")\n      print(labels)\n\n    return labels.cpu().numpy()\n\ndef get_separate_labels_tf(ims, model, verbose=False, name=None):\n    bs = ims_per_batch\n    image_batch = copy.deepcopy(ims)\n    image_batch_fl_lr = tf.image.flip_left_right(image_batch)\n    image_batch_fl_up = tf.image.flip_up_down(image_batch)\n    image_batch_fl_up_lr = tf.image.flip_up_down(image_batch_fl_lr)\n\n    ims = tf.concat([image_batch, image_batch_fl_lr, image_batch_fl_up, image_batch_fl_up_lr], axis=0)\n    labels = model.predict(ims, verbose=1)\n    labels = labels.reshape(bs*4//bs//4, 4, bs, 19).mean(1)#.view(bs*4//4, 19)\n    if verbose:\n      print(f\"LABELS FROM LABELS MODEL {name}\")\n      print(labels)\n\n    return labels[0]\n\n#Everything above 0.3 is probably right (F1-Macro-0.3: 0.7599178063871298 ---- F1-Micro-0.3: 0.7793390324588123 ---- Accuracy-0.3: 0.5323157894736842) so maybe cut below or inrease above?","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load ViT","metadata":{}},{"cell_type":"code","source":"#from vit_keras import vit, utils, layers\nsys.path.append(\"../input/faustomorales-vitkeras/vit_keras/\")\nimport layers\n\nCONFIG_B = {\n    \"dropout\": 0.1,\n    \"mlp_dim\": 3072,\n    \"num_heads\": 12,\n    \"num_layers\": 12,\n    \"hidden_size\": 768,\n}\n\nclass TransformerBlock(tf.keras.layers.Layer):\n    \"\"\"Implements a Transformer block.\"\"\"\n\n    def __init__(self, *args, num_heads=12, mlp_dim=3072, dropout=0.1, **kwargs):\n        super().__init__(*args, **kwargs)\n        self.num_heads = num_heads\n        self.mlp_dim = mlp_dim\n        self.dropout = dropout\n\n    def build(self, input_shape):\n        self.att = MultiHeadSelfAttention(\n            num_heads=self.num_heads,\n            name=\"MultiHeadDotProductAttention_1\",\n        )\n        self.mlpblock = tf.keras.Sequential(\n            [\n                tf.keras.layers.Dense(\n                    self.mlp_dim,\n                    activation=\"linear\",\n                    name=f\"{self.name}/Dense_0\",\n                ),\n                tf.keras.layers.Lambda(\n                    lambda x: tf.keras.activations.gelu(x, approximate=False)\n                )\n                if hasattr(tf.keras.activations, \"gelu\")\n                else tf.keras.layers.Lambda(\n                    lambda x: tfa.activations.gelu(x, approximate=False)\n                ),\n                tf.keras.layers.Dropout(self.dropout),\n                tf.keras.layers.Dense(input_shape[-1], name=f\"{self.name}/Dense_1\"),\n                tf.keras.layers.Dropout(self.dropout),\n            ],\n            name=\"MlpBlock_3\",\n        )\n        self.layernorm1 = tf.keras.layers.LayerNormalization(\n            epsilon=1e-6, name=\"LayerNorm_0\"\n        )\n        self.layernorm2 = tf.keras.layers.LayerNormalization(\n            epsilon=1e-6, name=\"LayerNorm_2\"\n        )\n        self.dropout = tf.keras.layers.Dropout(self.dropout)\n\n    def call(self, inputs, training):\n        x = self.layernorm1(inputs)\n        x, weights = self.att(x)\n        x = self.dropout(x, training=training)\n        x = x + inputs\n        y = self.layernorm2(x)\n        y = self.mlpblock(y)\n        return x + y, weights\n\nclass MultiHeadSelfAttention(tf.keras.layers.Layer):\n    def __init__(self, *args, num_heads=12, **kwargs):\n        super().__init__(*args, **kwargs)\n        self.num_heads = num_heads\n\n    def build(self, input_shape):\n        hidden_size = input_shape[-1]\n        num_heads = self.num_heads\n        if hidden_size % num_heads != 0:\n            raise ValueError(\n                f\"embedding dimension = {hidden_size} should be divisible by number of heads = {num_heads}\"\n            )\n        self.hidden_size = hidden_size\n        self.projection_dim = hidden_size // num_heads\n        self.query_dense = tf.keras.layers.Dense(hidden_size, name=\"query\")\n        self.key_dense = tf.keras.layers.Dense(hidden_size, name=\"key\")\n        self.value_dense = tf.keras.layers.Dense(hidden_size, name=\"value\")\n        self.combine_heads = tf.keras.layers.Dense(hidden_size, name=\"out\")\n\n    # pylint: disable=no-self-use\n    def attention(self, query, key, value):\n        score = tf.matmul(query, key, transpose_b=True)\n        dim_key = tf.cast(tf.shape(key)[-1], score.dtype)\n        scaled_score = score / tf.math.sqrt(dim_key)\n        weights = tf.nn.softmax(scaled_score, axis=-1)\n        output = tf.matmul(weights, value)\n        return output, weights\n\n    def separate_heads(self, x, batch_size):\n        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.projection_dim))\n        return tf.transpose(x, perm=[0, 2, 1, 3])\n\n    def call(self, inputs):\n        batch_size = tf.shape(inputs)[0]\n        query = self.query_dense(inputs)\n        key = self.key_dense(inputs)\n        value = self.value_dense(inputs)\n        query = self.separate_heads(query, batch_size)\n        key = self.separate_heads(key, batch_size)\n        value = self.separate_heads(value, batch_size)\n\n        attention, weights = self.attention(query, key, value)\n        attention = tf.transpose(attention, perm=[0, 2, 1, 3])\n        concat_attention = tf.reshape(attention, (batch_size, -1, self.hidden_size))\n        output = self.combine_heads(concat_attention)\n        return output, weights\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference one Batch","metadata":{}},{"cell_type":"code","source":"if CFG.resnest:\n    model_resnest =  Classifier(CFG.model_name_resnest, CFG.classes, mode=\"normal\")\n    if CFG.color_mode == \"rgby\":\n      weight = model_resnest.model.conv1[0].weight.clone()\n      model_resnest.model.conv1[0] = nn.Conv2d(4, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) #64 for resnest101, 32 for resnest50\n      with torch.no_grad():\n        model_resnest.model.conv1[0].weight[:, :3] = weight\n        model_resnest.model.conv1[0].weight[:, 3] = model_resnest.model.conv1[0].weight[:, 0]\n\n    model_resnest.to(device)\n    model_resnest.eval()\n    model_states_resnet = [torch.load(MODEL_PATH + f\"{model}\") for model in MODELS_LIST_RESNEST] #do this only once!!!\n\nif CFG.effnet:\n    model_effnet = Classifier_EffNet(CFG.model_name_effnet)\n    if CFG.color_mode == \"rgby\":\n        model_effnet.enet._conv_stem.in_channels = 4\n        model_effnet.enet._conv_stem.weight = torch.nn.Parameter(torch.cat([model_effnet.enet._conv_stem.weight, model_effnet.enet._conv_stem.weight[:, 0:1, :, :]], axis=1))\n    model_effnet.to(device)\n    model_effnet.eval()\n    model_states_effnet = [torch.load(MODEL_PATH + f\"{model}\") for model in MODELS_LIST_EFFNET] #do this only once!!!\n\nif CFG.extra_model_for_labels:\n    if CFG.extra_model_is_tf:\n      model_green_eff = tf.keras.models.load_model(MODEL_LABELS_EFF)\n      model_green_rn = tf.keras.models.load_model(MODEL_LABELS_RN)\n      model_green_vit = tf.keras.models.load_model(MODEL_LABELS_VIT, custom_objects={\n                                                  'ClassToken': layers.ClassToken, \n                                                  'AddPositionEmbs' : layers.AddPositionEmbs,\n                                                  'TransformerBlock' : TransformerBlock,\n                                                  'MultiHeadSelfAttention' : MultiHeadSelfAttention})\n      #model_green_dn = tf.keras.models.load_model(MODEL_LABELS_DN)\n    else:\n      model_for_labels_state = torch.load(MODEL_LABELS)\n      model_green = Classifier_EffNet_GREEN(\"efficientnet-b7\")\n      model_green.to(device)\n      model_green.eval()\n\n\ndl_test = DataLoader(test_data, batch_size=batch_size_, shuffle=False, num_workers=0)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef inference_one_batch(batch_cam, batch_ids_cam, batch_seg, ims_per_batch, batch_eff_tf=None, batch_rn_tf_600=None, batch_vit_tf_384=None, show_image=True, show_seg=True, verbose=True):\n    batch_ids_seg = tuple(batch_seg[\"ID\"])\n    print(batch_ids_cam)\n    print(batch_ids_seg)\n    assert batch_ids_cam == batch_ids_seg, \"IDS OF SEGMENTATION AND CAMS DONT MATCH\"\n\n    #get mask (from fullsize img)\n    print(f\"GETTING {ims_per_batch} CELL MASKS\")\n    cell_mask_list = batch_seg[\"mask\"] \n    cell_mask_sizes = batch_seg[\"ori_size\"]\n    cell_masks_full_size = []\n\n    #resize masks to original size\n    for e, (mask, size) in enumerate(zip(cell_mask_list, cell_mask_sizes)):\n      cell_masks_full_size.append(resize_full_mask(mask,size)) #change zoom method...    \n    \n    #resize masks to image size\n    res_cell_masks = [resize_mask(cell_mask) for cell_mask in cell_mask_list]\n\n    if show_seg:\n      show_seg_cells(batch_cam, res_cell_masks, batch_ids_seg, batch_cam)\n\n    del cell_mask_list\n    #get encoded cell masks -> save\n    cell_masks_fullsize_enc_list = []\n    for cell_mask in cell_masks_full_size:\n        cell_masks_fullsize_enc = get_all_encoded_cells(cell_mask)\n        cell_masks_fullsize_enc_list.append(cell_masks_fullsize_enc)\n    del cell_masks_full_size\n\n    print(f\"ENCODED {len(cell_masks_fullsize_enc_list)} CELL MASKS\")\n\n\n    #batch_eff_tf -> ggg 600\n    #batch_rn_tf_600 -> rgb 600\n    #batch_vit_tf_384 -> ggg 384\n    labels_model_eff = get_separate_labels_tf(batch_eff_tf, model_green_eff, verbose=verbose, name=MODEL_LABELS_EFF)\n    labels_model_rn = get_separate_labels_tf(batch_rn_tf_600, model_green_rn, verbose=verbose, name=MODEL_LABELS_RN)\n    labels_model_vit = get_separate_labels_tf(batch_vit_tf_384, model_green_vit, verbose=verbose, name=MODEL_LABELS_VIT)\n\n    labels_model = labels_model_eff*CFG.split_image_level[0] + labels_model_rn*CFG.split_image_level[1] + labels_model_vit*CFG.split_image_level[2]\n\n    print(\"LABELS FROM LABEL MODEL\")\n    print(labels_model)\n\n    print(\"RESIZE IMAGES\")\n    batch_cam_scaled = []\n    scales = [1.0, 1.3, 1.6]\n    st = time.time()\n    for i, scale in enumerate(scales):\n      image_batch_pil = torch.from_numpy(np.zeros((ims_per_batch, 4, round(CFG.size*scale), round(CFG.size*scale))))\n      image_batch = copy.deepcopy(batch_cam)\n      for j, im in enumerate(image_batch):\n        im = ToPILImage()(im)\n        im = im.resize((round(CFG.size*scale), round(CFG.size*scale)), resample=PIL.Image.BICUBIC)\n        im = torchvision.transforms.functional.to_tensor(im)*255 #find a way to convert PIL to tensor without scaling (dividing everything by 255)\n        image_batch_pil[j] = im\n\n      image_batch_resized = image_batch_pil.float()\n      image_batch_augs_fl2 = image_batch_resized.flip(2)\n      image_batch_augs_fl3 = image_batch_resized.flip(3)\n      image_batch_augs_fl32 = torch.flip(image_batch_resized,(3,2))\n\n      #concat to one vector: im1, im2, .., im1fl, im2fl, .., im1fl2, im2fl2, ...\n      images = torch.cat([image_batch_resized, image_batch_augs_fl2, image_batch_augs_fl3, image_batch_augs_fl32], dim=0)\n      images = images.cuda()\n      batch_cam_scaled.append(images)\n    print(f\"TIME FOR RESIZING WITH PIL {time.time() - st}\")\n\n    if CFG.resnest:\n        #get cams (from resized img)\n        print(\"GETTING CAMS AND PREDS RESNEST\")\n        #scales = [ 0.7, 0.9, 1.0, 1.3, 1.6]\n        #scales = [0.9, 1.0, 1.3]\n\n        \n        mask_probas_resnest_folds = []\n        folds_resnest = len(MODELS_LIST_RESNEST)\n        all_ims_resnest = []\n        time_spent_mask_probas = []\n\n        for f, model_state_rn in enumerate(model_states_resnet): #iterate through model states\n            all_hr_cams, sig_labels = get_hrcams_vis(batch_cam_scaled, show_image, model_resnest, model_state_rn, scales, ims_per_batch)  #dont get image every time again get at beginning!\n            \n            mask_probas_resnest_batches = []\n            #get probabilities\n            print(f\"GETTING MASKS PROBAS FOLD {f}\")\n            for b, (cams, mask, sig_label, label_model) in enumerate(zip(all_hr_cams, res_cell_masks, sig_labels, labels_model)): #iterate through batch\n                print(f\"GETTING MASKS PROBAS BATCH {b}\")\n                mask_probas_resnest = get_prob_from_cams_masks(cams, mask, sig_label, label_model, verbose, typ=f\"resnest{f}\")\n\n                if verbose:\n                  print(mask_probas_resnest)\n                  \n                mask_probas_resnest_batches.append(mask_probas_resnest)\n            mask_probas_resnest_folds.append(mask_probas_resnest_batches)\n\n        #mask_probas_resnest_folds[0][0] #image0 fold0\n        #mask_probas_resnest_folds[1][0] #image0 fold1\n        for b in range(ims_per_batch):\n            all_ims_resnest.append(np.mean(np.array([mask_probas_resnest_folds[i][b] for i in range(folds_resnest)]), axis=0))\n\n\n    if CFG.effnet:\n        #get cams (from resized img)\n        print(\"GETTING CAMS AND PREDS EFFICIENTNET\")\n        #scales = [ 0.7, 0.9, 1.0, 1.3, 1.6]\n        #scales =[0.9, 1.0, 1.3]\n\n        \n        mask_probas_effnet_folds = []\n        folds_effnet = len(MODELS_LIST_EFFNET)\n        all_ims_effnet = []\n        time_spent_mask_probas = []\n\n        for f, model_state_eff in enumerate(model_states_effnet):\n            all_hr_cams, sig_labels = get_hrcams_vis(batch_cam_scaled, show_image, model_effnet, model_state_eff, scales, ims_per_batch) #dont get image every time again get at beginning!\n            mask_probas_effnet_batches = []\n            #get probabilities\n            print(f\"GETTING MASKS PROBAS FOLD {f}\")\n            for b, (cams, mask, sig_label, label_model) in enumerate(zip(all_hr_cams, res_cell_masks, sig_labels, labels_model)):\n                print(f\"GETTING MASKS PROBAS BATCH {b}\")\n                s_t = time.time()\n                mask_probas_effnet = get_prob_from_cams_masks(cams, mask, sig_label, label_model, verbose, typ=f\"effnet{f}\")\n                \n                if verbose:\n                  print(mask_probas_effnet)\n\n                mask_probas_effnet_batches.append(mask_probas_effnet)\n\n            mask_probas_effnet_folds.append(mask_probas_effnet_batches)\n\n        for b in range(ims_per_batch):\n            all_ims_effnet.append(np.mean(np.array([mask_probas_effnet_folds[i][b] for i in range(folds_effnet)]), axis=0))\n\n    if CFG.resnest and CFG.effnet:\n      mask_probas = CFG.split_cam_level[0]*np.array(all_ims_effnet) +  CFG.split_cam_level[1]*np.array(all_ims_resnest)\n\n    elif CFG.resnest and not CFG.effnet:\n      mask_probas = all_ims_resnest\n      \n    elif CFG.effnet and not CFG.resnest:\n      mask_probas = all_ims_effnet\n\n    if verbose:\n      print(mask_probas)  \n\n    return batch_ids_cam, sizes, cell_masks_fullsize_enc_list, mask_probas","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"libc = ctypes.CDLL(\"libc.so.6\")\nlibc.malloc_trim(0)\ngc.collect()\ngc.collect()\ngc.collect()\ntorch.cuda.empty_cache()\ntorch.cuda.empty_cache()\ntorch.cuda.empty_cache()\ntorch.cuda.empty_cache()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.verbose:\n    batch0, ids0 = next(iter(dl_test))\n    print(batch0.shape)\n\n    if CFG.extra_model_is_tf:\n        batch_tf_green = next(iter(dtest_tf_green_600))\n        img = tf.keras.preprocessing.image.array_to_img(batch_tf_green[1])\n\n        plt.rcParams['figure.figsize'] = [10, 10]\n        plt.imshow(img)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.verbose:\n    if CFG.extra_model_is_tf:\n        batch_tf_rgb = next(iter(dtest_tf_rgb_600))\n        img = tf.keras.preprocessing.image.array_to_img(batch_tf_rgb[1])\n\n        plt.rcParams['figure.figsize'] = [10, 10]\n        plt.imshow(img)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.is_demo:\n    from itertools import islice, count\n    import itertools\n    index = 0\n    batch0, ids0 = next(itertools.islice(dl_test, index, None))\n    if CFG.extra_model_is_tf:\n      batch_tf_green = next(itertools.islice(dtest_tf_green_600, index, None))\n      batch_tf_rgb_600 = next(itertools.islice(dtest_tf_rgb_600, index, None))\n      batch_tf_rgb_384 = next(itertools.islice(dtest_tf_ggg_384, index, None))\n      print(batch_tf_green.shape)\n      print(batch_tf_rgb_600.shape)\n      print(batch_tf_rgb_384.shape)\n\n\n    #batch_seg = cell_masks_df[batch_size_*index:batch_size_*(index+1)]\n    #ims_per_batch = batch_size_\n    #ids, sizes, cell_masks_fullsize_enc_list, probas =  inference_one_batch(batch0, ids0, batch_seg, ims_per_batch, batch_eff_tf=batch_tf_green, batch_vit_tf_384=batch_tf_rgb_384, batch_rn_tf_600=batch_tf_rgb_600, show_image=True, show_seg=True, verbose=True)\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#checked the scaling factor if scales are not original ones###\n\ndf = pd.DataFrame(columns=[\"image_id\", \"pred\"])\ni = 0\n\nstart_time = time.time()\nims_done = 0\nfor i, ((batch_cam, batch_ids_cam), (batch_tf_green_600), (batch_tf_rgb_600), (batch_tf_ggg_384)) in enumerate(zip(dl_test, dtest_tf_green_600, dtest_tf_rgb_600, dtest_tf_ggg_384)):  #dtest_tf\n  ims_per_batch = len(batch_ids_cam)\n  batch_seg = cell_masks_df[i*batch_size_ : i*batch_size_  + batch_size_]\n  \n  ids, sizes, cell_masks_fullsize_enc_list, probas =  inference_one_batch(batch_cam, batch_ids_cam, batch_seg, ims_per_batch, batch_eff_tf=batch_tf_green_600, batch_rn_tf_600=batch_tf_rgb_600, batch_vit_tf_384=batch_tf_ggg_384, show_image=False, show_seg=False, verbose=False)\n  for id, proba, cell_mask_enc in zip(ids, probas, cell_masks_fullsize_enc_list):\n    pred_string = get_pred_string(proba, cell_mask_enc)\n    d = {\"image_id\":id, \"pred\": pred_string}\n    df = df.append(d, ignore_index=True)\n  ims_done += ims_per_batch\n  print(f\"---{ims_done} IMAGES DONE---\")\n  \n  torch.cuda.empty_cache()\n  gc.collect()\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sample_submission = pd.read_csv(\"../input/hpa-single-cell-image-classification/sample_submission.csv\")\nsub = pd.merge(\n    data_df,\n    df,\n    how=\"left\",\n    left_on='ID',\n    right_on='image_id',\n)\n\ndef isNaN(num):\n    return num != num\n\nfor i, row in sub.iterrows():\n    if isNaN(row['pred']): continue\n    sub.PredictionString.loc[i] = row['pred']\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = sub[data_df.columns]\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.head(20)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if all(df_from_files == data_df_sample_submission):\n    sub.to_csv(\"submission.csv\",index=False)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}