{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/input/hpacellsegmentatorraman/HPA-Cell-Segmentation","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install  \"/kaggle/input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl\"\n# !pip install -q \"/kaggle/input/hpapytorchzoozip/pytorch_zoo-master\"\n#!pip install -q \"/kaggle/input/hpacellsegmentatormaster/HPA-Cell-Segmentation-master\"\n!pip install \"/kaggle/input/efficientnetpytorch\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append(\"/kaggle/input/hpacellsegmentatorraman/HPA-Cell-Segmentation\")\nsys.path.append(\"/kaggle/input/hpapytorchzoozip/pytorch_zoo-master\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labeling_model_path = \"/kaggle/input/hpalabeliingmodel/3ch-no-mask-multiflag-mixup.pth\"\nkeepres_labeling_model_path = \"/kaggle/input/hpalabeliingmodel/3ch-keep-resolution-fine-tune-472.pth\"\nlabeling_model_path_472 =  \"/kaggle/input/hpalabeliingmodel/3ch-multiflag-mixup-fine-tune-472.pth\"\n\ntest_path = \"/kaggle/input/hpa-single-cell-image-classification/test/\"\ntest_df_path=  \"/kaggle/input/hpa-single-cell-image-classification/sample_submission.csv\"\nnuclei_model_path = \"/kaggle/input/hpa-cell-segmentation-weights/nuclei-model.pth\"\ncell_model_path = \"/kaggle/input/hpa-cell-segmentation-weights/cell-model.pth\"\nbatch_size=8\nnum_workers=2\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nSUFFIX2TYPE = {\n    \"red\": \"microtubule\",\n    \"yellow\": \"er\",\n    \"blue\": \"nuclei\",\n    \"green\": \"protein\",\n}\n\nIMAGE_TYPES = [\"microtubule\", \"er\", \"nuclei\", \"protein\"]\n\n\nfrom collections import namedtuple\nfrom dataclasses import dataclass\nfrom typing import Tuple\n\nimport numpy as np\n\nCellSample = namedtuple(\"CellSample\", (\"id\", \"microtubule\", \"er\", \"nuclei\", \"protein\", \"image_size\"))\nCellInput = namedtuple(\"CellInput\", (\"nuc_input\", \"cell_input\"))\nCellOutput = namedtuple(\"CellOutput\", (\"nuc_output\", \"cell_output\"))\n\n\n@dataclass\nclass CellLabelInput:\n    image_size: int\n    nuc_prediction: np.array\n    cell_prediction: np.array\n\n\n@dataclass\nclass SegmentedImage:\n    image: np.array\n    sample_id: str\n    segment_id: int\n    image_size: Tuple[int, int] = None\n    encoded_mask: str = None\n    pred_score: np.array = None\n\n    def __post_init__(self):\n        if self.image_size is None:\n            self.image_size = self.image.shape[-2:]\n\n    def file_name(self, prefix):\n        return f\"{self.sample_id}_{self.segment_id}.{prefix}\"\n\n    @property\n    def text_attributes(self):\n        return {\n            \"sample_id\": self.sample_id,\n            \"segment_id\": self.segment_id,\n            \"image_size\": self.image_size,\n        }\n\n    def format(self):\n        return \" \".join([f\"{i} {score} {self.encoded_mask}\" for i, score in enumerate(self.pred_score)])\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom skimage import transform\nfrom skimage.util import img_as_float64\n\nNORMALIZE = {\n    \"mean\": np.array([124 / 255, 117 / 255, 104 / 255]),\n    \"std\": np.array([1 / (0.0167 * 255)] * 3),\n}\n\nIMAGE_SIZE = 512\n\n\ndef normalize(image):\n    return (image - NORMALIZE[\"mean\"][:, None, None]) / NORMALIZE[\"std\"][:, None, None]\n\n\nclass SamplePreprocessor:\n    @staticmethod\n    def run(sample: CellSample):\n        nuclei_input = NucleiPreprocessor.run(sample)\n        cell_input = CellPreprocessor.run(sample)\n        return nuclei_input, cell_input\n\n\nclass NucleiPreprocessor:\n    @classmethod\n    def run(cls, sample: CellSample):\n        image = cls._scale_image(img_as_float64(sample.nuclei), sample.image_size)\n        image = cls._stack_image(image)\n        image = normalize(image)\n        return image.astype(np.float64)\n\n    @staticmethod\n    def _scale_image(image, image_size):\n        return transform.rescale(\n            image,\n            scale=IMAGE_SIZE / image_size,\n            anti_aliasing=True,\n        )\n\n    @staticmethod\n    def _stack_image(image):\n        return np.stack([image, image, image])\n\n\nclass CellPreprocessor:\n    @classmethod\n    def run(cls, sample: CellSample):\n        images = cls._scale_images([img_as_float64(sample.microtubule), img_as_float64(sample.er), img_as_float64(sample.nuclei)], sample.image_size)\n        images = np.stack(images)\n        images = normalize(images)\n        return images.astype(np.float32)\n\n    @staticmethod\n    def _scale_images(images, image_size):\n        scale = IMAGE_SIZE / image_size\n        return [\n            transform.rescale(\n                image,\n                scale=scale,\n                anti_aliasing=True,\n            )\n            for image in images\n        ]\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom logging import getLogger\n\nimport imageio\nimport numpy as np\nfrom PIL import ImageFile\nfrom torch.utils.data import Dataset\n\nfrom skimage.util import img_as_ubyte\n\nImageFile.LOAD_TRUNCATED_IMAGES = True\n\nlogger = getLogger(__name__)\n\n\nclass CellDataset(Dataset):\n    def __init__(self, sample_df, image_root_path):\n        self.sample_df = sample_df\n        self.image_root_path = image_root_path\n\n    def __len__(self):\n        return len(self.sample_df)\n\n    def __getitem__(self, index):\n        sample = self.sample_df.iloc[index]\n        # logger.info(f\"start loading image: {sample.ID}\")\n        cell_sample = self._load_sample(sample.ID)\n        # logger.info(f\"start preprocessing image: {sample.ID}\")\n        input = SamplePreprocessor.run(cell_sample)\n        return cell_sample, CellInput(*input)\n\n    def _load_sample(self, sample_id):\n        image_dict = {\"id\": sample_id}\n        for suffix, type in SUFFIX2TYPE.items():\n            img_path = os.path.join(self.image_root_path, f\"{sample_id}_{suffix}.png\")\n            image_dict[type] = img_as_ubyte(np.array(imageio.imread(img_path)))\n        return CellSample(**image_dict, image_size=image_dict[\"protein\"].shape[0])\n\n    \n    \nimport torch\n\n\ndef my_collate(batch):\n    samples = [s[0] for s in batch]\n    inputs = [s[1] for s in batch]\n    return collate_by_tuple(samples), collate_by_tensor(inputs)\n\n\ndef collate_by_tuple(tuples):\n    clazz = type(tuples[0])\n    return clazz(*[tuple([getattr(t, field) for t in tuples]) for field in clazz._fields])\n\n\ndef collate_by_tensor(tuples):\n    clazz = type(tuples[0])\n    return clazz(*[torch.Tensor([getattr(t, field) for t in tuples]) for field in clazz._fields])\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\n\nimport torch\nimport torch.nn.functional as F\n\n\nclass CellPredictionModel:\n    def __init__(self, nuclei_model_path, cell_model_path, device):\n        self.nuclei_model = SegmentationModel(nuclei_model_path, device)\n        self.cell_model = SegmentationModel(cell_model_path, device)\n\n    def __call__(self, input: CellInput):\n        nuc_preds = self.nuclei_model(torch.FloatTensor(input.nuc_input))\n        cell_preds = self.cell_model(torch.FloatTensor(input.cell_input))\n        return CellOutput(nuc_preds, cell_preds)\n\n\nclass SegmentationModel:\n    def __init__(self, model_path, device):\n        self.nuclei_model = torch.load(model_path, map_location=torch.device(device))\n        self.device = device\n\n    def __call__(self, batch):\n        with torch.no_grad():\n            preds = self.nuclei_model(batch.to(self.device))\n            preds = F.softmax(preds, dim=1)\n            return preds.cpu().numpy()\n\nclass CellPredicitonDecomposer:\n    @staticmethod\n    def run(cell_sample, cell_output):\n        inputs = [\n            CellLabelInput(*attrs)\n            for attrs in zip(\n                cell_sample.image_size,\n                cell_output.nuc_output,\n                cell_output.cell_output,\n            )\n        ]\n\n        samples = [CellSample(*attrs) for attrs in zip(*[getattr(cell_sample, field) for field in cell_sample._fields])]\n        return list(zip(samples, inputs))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from typing import List\n\nimport cv2\nimport numpy as np\nfrom skimage import util\n\nclass InputPreprocessor:\n    @staticmethod\n    def run(input: CellLabelInput):\n        input.cell_prediction = OutputPrprocessor.run(input.cell_prediction, input.image_size)\n        input.nuc_prediction = OutputPrprocessor.run(input.nuc_prediction, input.image_size)\n        return input\n\n\nclass OutputPrprocessor:\n    @classmethod\n    def run(cls, pred, image_size) -> List[np.array]:\n        \"\"\"\n        return 2 channels for each prediction:\n        * pred[0]: border\n        * pred[1]: detected segments\n        \"\"\"\n        pred = cls._remove_first_channel(pred)\n        pred = cls._format(pred)\n        pred = cls._resize(pred, image_size)\n        return pred\n\n    @staticmethod\n    def _resize(pred, image_size):\n        pred = pred.transpose([1, 2, 0])\n        pred = cv2.resize(\n            pred,\n            (image_size, image_size),\n            interpolation=cv2.INTER_AREA,\n        )\n        pred = pred.transpose([2, 0, 1])\n        return pred\n\n    @staticmethod\n    def _format(pred):\n        \"\"\"\n        quantize values\n        \"\"\"\n        return util.img_as_ubyte(pred)\n\n    @staticmethod\n    def _remove_first_channel(pred):\n        return pred[1:]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LabelPredictor:\n    @staticmethod\n    def run(input: CellLabelInput):\n        nuc_label = NucleiLabelPredictor.run(input.nuc_prediction, input.cell_prediction)\n        cell_label = CellLabelPredictor.run(input.cell_prediction, nuc_label)\n        cell_label = SmallLabelRemover.run(cell_label)\n        return cell_label\n\nimport numpy as np\nimport scipy.ndimage as ndi\nfrom skimage import segmentation\nfrom skimage.morphology import remove_small_holes, remove_small_objects\n\nHIGH_THRESHOLD = 0.4 * 255\nLOW_THRESHOLD = HIGH_THRESHOLD - 0.25 * 255\n\n\nclass NucleiLabelPredictor:\n    @classmethod\n    def run(cls, nuclei_pred, cell_pred):\n        nuc_borders, nuc_segments = nuclei_pred\n        cell_borders, _ = cell_pred\n\n        markers = cls._build_markers(nuc_segments, nuc_borders, cell_borders)\n        mask = cls._build_mask(nuc_segments)\n        labels = segmentation.watershed(mask, markers, mask=mask, watershed_line=True)\n\n        labels = ndi.label(labels)[0]\n        labels = remove_small_objects(labels, 2500)\n\n        return labels\n\n    @classmethod\n    def _build_markers(cls, nuc_segments, nuc_borders, cell_borders):\n        borders = cls._build_border_for_marker(nuc_borders, cell_borders)\n        marker_base = nuc_segments * borders\n\n        marker_mask = np.zeros_like(nuc_segments).astype(bool)\n        marker_mask[marker_base > LOW_THRESHOLD] = 1\n        marker_mask = remove_small_objects(marker_mask, 500).astype(np.uint8)\n        markers = ndi.label(marker_mask, output=np.uint32)[0]\n\n        return markers\n\n    @staticmethod\n    def _build_border_for_marker(nuc_borders, cell_borders):\n        return 1 - (nuc_borders + cell_borders) / 255.0 > 0.05\n\n    @staticmethod\n    def _build_mask(nuc_segments):\n        mask = nuc_segments > HIGH_THRESHOLD\n        mask = mask.astype(np.bool)\n        mask = remove_small_holes(mask, 1000)\n        mask = remove_small_objects(mask, 8).astype(np.uint8)\n        return mask\n\n\nimport numpy as np\nimport scipy.ndimage as ndi\nfrom skimage import filters, segmentation\nfrom skimage.morphology import closing, disk, remove_small_objects\n\n\nclass CellLabelPredictor:\n    @classmethod\n    def run(cls, cell_pred, nuclei_label):\n        _, cell_segments = cell_pred\n\n        threshold = cls._calc_distance_thereshold(cell_segments)\n        elevation_map = cls._calc_cell_elevation(cell_segments)\n\n        cell_label = segmentation.watershed(\n            elevation_map,\n            markers=nuclei_label,\n            mask=cls._calc_cell_mask(cell_segments, threshold),\n            watershed_line=True,\n        )\n\n        cell_label = remove_small_objects(cell_label, 5500).astype(np.uint8)\n        cell_label = ndi.label(cell_label)[0]\n        return cell_label\n\n    @staticmethod\n    def _calc_border_mask(cell_borders):\n        return np.asarray(cell_borders / 255 > 0.05, dtype=np.int8)\n\n    @staticmethod\n    def _calc_distance_thereshold(cell_segments):\n        return max(0.22 * 255, filters.threshold_otsu(cell_segments) * 0.5)\n\n    @staticmethod\n    def _calc_cell_elevation(cell_segments):\n        return -cell_segments\n\n    @staticmethod\n    def _calc_cell_mask(cell_segments,  threshold):\n        cell_mask = cell_segments > threshold\n        return cell_mask\n\n    @staticmethod\n    def _fix_broken_shapes(cell_label):\n        return closing(cell_label, disk(6))\n\nimport numpy as np\n\n\nclass SmallLabelRemover:\n    @classmethod\n    def run(cls, label):\n        remove_label_indices = cls._get_remove_label_indices(label)\n        remove_mask = np.invert(np.isin(label, remove_label_indices))\n        remained_label = label * remove_mask\n        return remained_label\n\n    @staticmethod\n    def _get_remove_label_indices(label):\n        stats = np.bincount(label.flatten())[1:]\n        remove_label_indices = np.where(stats < (stats.std() * 0.3))[0] + 1\n        return remove_label_indices\n\n    \nclass CellLabelCalculator:\n    @staticmethod\n    def run(input: CellLabelInput):\n        input = InputPreprocessor.run(input)\n        cell_label = LabelPredictor.run(input)\n        return cell_label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import base64\nimport zlib\n\nimport numpy as np\nfrom pycocotools import _mask as coco_mask\nfrom skimage import img_as_ubyte\n\n\nclass ImageSegmentator:\n    @classmethod\n    def run(cls, sample: CellSample, labels: np.array):\n        image = cls._stack_image(sample)\n        segmented_samples = []\n        for segment_id in cls._find_segment_ids(labels):\n            segmented_sample = SegmentedImage(\n                image=ImageCropper.run(image, labels, segment_id),\n                segment_id=segment_id,\n                sample_id=sample.id,\n                encoded_mask=BinaryMaskEncoder.run(labels == segment_id),\n            )\n            segmented_samples.append(segmented_sample)\n        return segmented_samples\n\n    @staticmethod\n    def _find_segment_ids(nuclei_label):\n        return set(np.unique(nuclei_label)) - {0}\n\n    @staticmethod\n    def _stack_image(sample):\n        types = [\"microtubule\", \"nuclei\", \"protein\"] # ignore yellow\n        return np.stack([getattr(sample, img_type) for img_type in types])\n\n\nclass ImageCropper:\n    @classmethod\n    def run(cls, image: np.array, labels: np.array, segment_id: int):\n        mask = cls._get_specified_label_mask(labels, segment_id)\n        coordnates = cls._get_crop_coordinates(mask)\n\n        cropped_mask = cls._crop_mask(mask, *coordnates)\n        cropped_images = cls._crop_images(image, *coordnates)\n        cropped_images = cropped_images * cropped_mask\n        cropped_images = img_as_ubyte(cropped_images)\n        return cropped_images\n\n    @staticmethod\n    def _get_specified_label_mask(labels, segment_id):\n        return np.where(labels == segment_id, 1, 0).astype(np.bool)\n\n    @staticmethod\n    def _get_crop_coordinates(mask):\n        true_points = np.argwhere(mask)\n        top_left = true_points.min(axis=0)\n        bottom_right = true_points.max(axis=0)\n        top = top_left[0]\n        bottom = bottom_right[0]\n        left = top_left[1]\n        right = bottom_right[1]\n        return top, bottom, left, right\n\n    @staticmethod\n    def _crop_images(images, top, bottom, left, right):\n        return images[\n            :,\n            top : bottom + 1,\n            left : right + 1,\n        ]\n\n    @staticmethod\n    def _crop_mask(mask, top, bottom, left, right):\n        return mask[\n            top : bottom + 1,\n            left : right + 1,\n        ]\n\n\nclass BinaryMaskEncoder:\n    def run(mask: np.ndarray) -> str:\n        mask = np.squeeze(mask)\n        if len(mask.shape) != 2:\n            raise ValueError(\"encode_binary_mask expects a 2d mask, received shape == %s\" % mask.shape)\n\n        # convert input mask to expected COCO API input --\n        mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)\n        mask_to_encode = mask_to_encode.astype(np.uint8)\n        mask_to_encode = np.asfortranarray(mask_to_encode)\n\n        # RLE encode mask --\n        encoded_mask = coco_mask.encode(mask_to_encode)[0][\"counts\"]\n\n        # compress and base64 encoding --\n        binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)\n        base64_str = base64.b64encode(binary_str)\n        return base64_str.decode(\"utf-8\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch.nn import ZeroPad2d\n\nMEAN = [20.503461399549288, 13.71165825908858, 13.297466514210495]\nSTD = [33.8227572424431, 36.53488215123118, 22.72992170966404]\n\n\nclass SymmetricPad(torch.nn.Module):\n    def __init__(self, pad_size):\n        self.pad_size = pad_size\n\n    def __call__(self, img):\n        pad_shape = self._get_padding(img)\n\n        if pad_shape is None:\n            return img\n        else:\n            return ZeroPad2d(pad_shape)(img)\n\n    def _get_padding(self, image):\n        width, height = image.shape[-2:]\n        horizontal_pad_length = max(0, self.pad_size[0] - width)\n        vertical_pad_length = max(0, self.pad_size[1] - height)\n\n        if horizontal_pad_length == 0 and vertical_pad_length == 0:\n            return None\n\n        l_pad = horizontal_pad_length // 2\n        r_pad = horizontal_pad_length // 2 + int(horizontal_pad_length % 2 != 0)\n        t_pad = vertical_pad_length // 2\n        b_pad = vertical_pad_length // 2 + int(vertical_pad_length % 2 != 0)\n\n        return (t_pad, b_pad, l_pad, r_pad)\n\nclass SquarePad(torch.nn.Module):\n    def __call__(self, img):\n        return ZeroPad2d(self._get_padding(img))(img)\n\n    @staticmethod\n    def _get_padding(image):\n        width, height = image.shape[-2:]\n        max_wh = max([width, height])\n        horizontal_padding = (max_wh - width) // 2\n        vertical_padding = (max_wh - height) // 2\n        l_pad = horizontal_padding\n        r_pad = horizontal_padding if horizontal_padding % 1 == 0 else horizontal_padding + 1\n        t_pad = vertical_padding\n        b_pad = vertical_padding if vertical_padding % 1 == 0 else vertical_padding + 1\n\n        return (t_pad, b_pad, l_pad, r_pad)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch.nn import ZeroPad2d\nfrom torchvision.transforms import (\n    CenterCrop,\n    Normalize,\n    RandomCrop,\n    RandomHorizontalFlip,\n    RandomResizedCrop,\n    RandomVerticalFlip,\n    Resize,\n)\n\nSEG_IMG_SIZE_KEEP_RES = (472, 472)\n\n\nclass SegmentPreprocessorKeepRes:\n    resize = torch.nn.Sequential(\n        SymmetricPad(SEG_IMG_SIZE_KEEP_RES),\n        CenterCrop(SEG_IMG_SIZE_KEEP_RES),\n    )\n    trans = Normalize(\n        mean=MEAN,\n        std=STD,\n    )\n\n    @classmethod\n    def run(cls, img):\n        img = torch.Tensor(img).type(torch.uint8).to(0)\n        img = cls.resize(img).type(torch.float32)\n        img = cls.trans(img)\n        return img\n\n    @classmethod\n    def batch_run(cls, imgs):\n        imgs = [cls.run(img) for img in imgs]\n        return torch.stack(imgs)\n\n\nclass AugmentedSegmentPreprocessorKeepRes:\n    resize = torch.nn.Sequential(\n        SymmetricPad(SEG_IMG_SIZE_KEEP_RES),\n        RandomCrop(SEG_IMG_SIZE_KEEP_RES),\n    )\n    trans = torch.nn.Sequential(\n        RandomHorizontalFlip(),\n        RandomVerticalFlip(),\n        Normalize(\n            mean=MEAN,\n            std=STD,\n        ),\n    )\n\n    @classmethod\n    def run(cls, img):\n        img = torch.Tensor(img).type(torch.uint8).to(0)\n        img = cls.resize(img).type(torch.float32)\n        img = cls.trans(img)\n        return img\n\n    @classmethod\n    def batch_run(cls, imgs):\n        imgs = [cls.run(img) for img in imgs]\n        return torch.stack(imgs)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch.nn import ZeroPad2d\nfrom torchvision.transforms import (\n    CenterCrop,\n    Normalize,\n    RandomCrop,\n    RandomHorizontalFlip,\n    RandomResizedCrop,\n    RandomVerticalFlip,\n    Resize,\n)\n\n\nclass SegmentPreprocessor472:\n    resize = torch.nn.Sequential(\n        SquarePad(),\n        Resize(SEG_IMG_SIZE_KEEP_RES),\n    )\n    trans = Normalize(\n        mean=MEAN,\n        std=STD,\n    )\n\n    @classmethod\n    def run(cls, img):\n        img = torch.Tensor(img).type(torch.uint8).to(0)\n        img = cls.resize(img).type(torch.float32)\n        img = cls.trans(img)\n        return img\n\n    @classmethod\n    def batch_run(cls, imgs):\n        imgs = [cls.run(img) for img in imgs]\n        return torch.stack(imgs)\n\n\nclass AugmentedSegmentPreprocessor472:\n    resize = torch.nn.Sequential(\n        Resize(SEG_IMG_SIZE_KEEP_RES),\n        RandomCrop(SEG_IMG_SIZE_KEEP_RES),\n    )\n    trans = torch.nn.Sequential(\n        RandomHorizontalFlip(),\n        RandomVerticalFlip(),\n        Normalize(\n            mean=MEAN,\n            std=STD,\n        ),\n    )\n\n    @classmethod\n    def run(cls, img):\n        img = torch.Tensor(img).type(torch.uint8).to(0)\n        img = cls.resize(img).type(torch.float32)\n        img = cls.trans(img)\n        return img\n\n    @classmethod\n    def batch_run(cls, imgs):\n        imgs = [cls.run(img) for img in imgs]\n        return torch.stack(imgs)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torchvision.transforms import Normalize, RandomHorizontalFlip, RandomResizedCrop, RandomVerticalFlip, Resize\n\nSEG_IMG_SIZE = (300, 300)\n\n\nclass SegmentPreprocessor:\n    resize = torch.nn.Sequential(\n        SquarePad(),\n        Resize(SEG_IMG_SIZE),\n    )\n    trans = Normalize(\n        mean=MEAN,\n        std=STD,\n    )\n\n    @classmethod\n    def run(cls, img):\n        img = torch.Tensor(img).type(torch.uint8).to(0)\n        img = cls.resize(img).type(torch.float32)\n        img = cls.trans(img)\n        return img\n\n    @classmethod\n    def batch_run(cls, imgs):\n        imgs = [cls.run(img) for img in imgs]\n        return torch.stack(imgs)\n\n\nclass AugmentedSegmentPreprocessor:\n    resize = RandomResizedCrop(SEG_IMG_SIZE, scale=(0.5, 1.0))\n    trans = torch.nn.Sequential(\n        RandomHorizontalFlip(),\n        RandomVerticalFlip(),\n        Normalize(\n            mean=MEAN,\n            std=STD,\n        ),\n    )\n\n    @classmethod\n    def run(cls, img):\n        img = torch.Tensor(img).type(torch.uint8).to(0)\n        img = cls.resize(img).type(torch.float32)\n        img = cls.trans(img)\n        return img\n\n    @classmethod\n    def batch_run(cls, imgs):\n        imgs = [cls.run(img) for img in imgs]\n        return torch.stack(imgs)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from itertools import chain\n\nimport numpy as np\nimport torch\n\n\nclass SegmentLabelPredictor:\n    def __init__(self, model, keepres_model, model_472):\n        self.model = model\n        self.keepres_model = keepres_model\n        self.model_472 = model_472\n\n    def run(self, segments, batch_size, n_augments=4):\n        scores = self._predict(segments, batch_size, n_augments)\n\n        for segment, score in zip(segments, scores):\n            segment.pred_score = score\n\n        return segments\n\n    def _predict(self, segments, batch_size, n_augments):\n        scores = []\n\n        for chunk in np.array_split(segments, len(segments) // batch_size + 1):\n            model_score = self._predict_for_each_chunk(chunk, n_augments)\n            keepres_score = self._predict_for_each_chunk_keepres(chunk, n_augments)\n            score_472 = self._predict_for_each_chunk_472(chunk, n_augments)\n            score = model_score * 0.575 + keepres_score * 0.2  + score_472 * 0.225\n            scores.append(score)\n\n        scores = list(chain.from_iterable(scores))\n\n        return scores\n\n    def _predict_for_each_chunk_keepres(self, chunk, n_augments):\n        imgs = [segment.image for segment in chunk]\n        chunk_imgs = [AugmentedSegmentPreprocessorKeepRes.batch_run(imgs) for _ in range(n_augments)]\n        chunk_imgs.append(SegmentPreprocessorKeepRes.batch_run(imgs))\n        aug_scores = [self._predict_core_keepres(chunk_img) for chunk_img in chunk_imgs]\n        scores = np.mean(aug_scores, axis=0)\n        return scores\n\n    def _predict_core_keepres(self, imgs):\n        with torch.no_grad():\n            scores = self.keepres_model(imgs)\n            scores = torch.nn.Sigmoid()(scores)\n\n        return scores.cpu().numpy()\n\n    def _predict_for_each_chunk(self, chunk, n_augments):\n        imgs = [segment.image for segment in chunk]\n        chunk_imgs = [AugmentedSegmentPreprocessor.batch_run(imgs) for _ in range(n_augments)]\n        chunk_imgs.append(SegmentPreprocessor.batch_run(imgs))\n        aug_scores = [self._predict_core(chunk_img) for chunk_img in chunk_imgs]\n        scores = np.mean(aug_scores, axis=0)\n        return scores\n\n    def _predict_core(self, imgs):\n        with torch.no_grad():\n            scores = self.model(imgs)\n            scores = torch.nn.Sigmoid()(scores)\n\n        return scores.cpu().numpy()\n\n    \n    def _predict_for_each_chunk_472(self, chunk, n_augments):\n        imgs = [segment.image for segment in chunk]\n        chunk_imgs = [AugmentedSegmentPreprocessor472.batch_run(imgs) for _ in range(n_augments)]\n        chunk_imgs.append(SegmentPreprocessor472.batch_run(imgs))\n        aug_scores = [self._predict_472(chunk_img) for chunk_img in chunk_imgs]\n        scores = np.mean(aug_scores, axis=0)\n        return scores\n\n    def _predict_472(self, imgs):\n        with torch.no_grad():\n            scores = self.model_472(imgs)\n            scores = torch.nn.Sigmoid()(scores)\n\n        return scores.cpu().numpy()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom efficientnet_pytorch.model import MemoryEfficientSwish\nfrom torch.nn import AdaptiveMaxPool2d, BatchNorm1d, Dropout, Linear\nfrom torch.nn.modules import Module\n\n\nclass CustomEfficientNet(Module):\n    def __init__(self, model_path, device):\n        super().__init__()\n        self.device = device\n        self.model = torch.load(model_path).to(device).train()\n\n        self._max_pool = AdaptiveMaxPool2d(output_size=1)\n\n        self._output = torch.nn.Sequential(\n            BatchNorm1d(num_features=1536 * 2),\n            MemoryEfficientSwish(),\n            Linear(in_features=1536 * 2, out_features=500, bias=True),\n            BatchNorm1d(num_features=500),\n            MemoryEfficientSwish(),\n            Dropout(p=0.3),\n            Linear(in_features=500, out_features=19),\n        )\n\n    def forward(self, inp):\n        with torch.no_grad():\n            x = self._extract_fixed_features(inp)\n    \n        x = self.model._swish(self.model._bn1(x))\n        x1 = self.model._avg_pooling(x).flatten(start_dim=1)\n        x2 = self._max_pool(x).flatten(start_dim=1)\n        x = torch.cat([x1, x2], axis=1)\n        x = self._output(x)\n\n        return x\n\n    def _extract_fixed_features(self, inputs):\n        model = self.model\n        x = model._swish(model._bn0(model._conv_stem(inputs)))\n        # Blocks\n        for idx, block in enumerate(model._blocks):\n            drop_connect_rate = model._global_params.drop_connect_rate\n            if drop_connect_rate:\n                drop_connect_rate *= float(idx) / len(model._blocks)  # scale drop connect_rate\n            x = block(x, drop_connect_rate=drop_connect_rate)\n\n        x = model._conv_head(x)\n\n        return x\n\n    def finetune_params(self):\n        return list(self._output.parameters()) + list(self.model._bn1.parameters())\n\n    def base_params(self):\n        return (\n            list(self.model._blocks.parameters())\n            + list(self.model._conv_stem.parameters())\n            + list(self.model._bn0.parameters())\n        )\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet\n\n\ndef load_labeling_model(labeling_model_path, device):\n    # effnet = EfficientNet.from_name(\"efficientnet-b3\",in_channels=4,num_classes=19)\n    ckpt = torch.load(labeling_model_path).eval()\n    # effnet.load_state_dict(ckpt)\n    return ckpt.to(device)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def format_sample(cell_sample, segments):\n    return \",\".join(\n        [\n            str(cell_sample.id), \n            str(cell_sample.protein.shape[0]), \n            str(cell_sample.protein.shape[1]),\n            \" \".join([segment.format() for segment in segments])\n        ]\n    ) + \"\\n\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"segment_model = CellPredictionModel(nuclei_model_path, cell_model_path, device=0)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labeling_model_keepres = load_labeling_model(keepres_labeling_model_path, device=0)\nlabeling_model = load_labeling_model(labeling_model_path, device=0)\nlabeling_model_472 = load_labeling_model(labeling_model_path_472, device=0)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"individual_label_predictor = SegmentLabelPredictor(labeling_model, labeling_model_keepres, labeling_model_472)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(test_df_path)\ndataset = CellDataset(test_df, test_path)\n\nfrom torch.utils.data import DataLoader\ndata_loader = DataLoader(\n            dataset,\n            batch_size=batch_size,\n            num_workers=num_workers,\n            collate_fn=my_collate,\n        )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\n\nlabel_path = \"/kaggle/input/calc-label-for-publiclb/labels\"\n\ndef inference_with_precomputed_labels():\n    with open(\"/kaggle/working/submission_base.csv\", \"w\") as file:\n        file.write(\",\".join([\"ID\",\"ImageWidth\", \"ImageHeight\", \"PredictionString\"]) + \"\\n\")\n        for batch in tqdm(data_loader):\n            cell_samples, _ = batch\n            samples = [CellSample(*attrs) for attrs in zip(*[getattr(cell_samples, field) for field in cell_samples._fields])]\n            for cell_sample in samples:\n                labels = np.load(os.path.join(label_path, f\"{cell_sample.id}.npy\"))\n                segments = ImageSegmentator.run(cell_sample, labels)\n                segments = individual_label_predictor.run(segments, batch_size=16, n_augments=4)\n                file.write(format_sample(cell_sample, segments))\n\ndef inference():\n    with open(\"/kaggle/working/submission_base.csv\", \"w\") as file:\n        file.write(\",\".join([\"ID\",\"ImageWidth\", \"ImageHeight\", \"PredictionString\"]) + \"\\n\")\n        for batch in tqdm(data_loader):\n            cell_samples, cell_inputs = batch\n            cell_outputs = segment_model(cell_inputs)\n            for cell_sample, label_input in CellPredicitonDecomposer.run(cell_samples, cell_outputs):\n                labels = CellLabelCalculator.run(label_input)\n                segments = ImageSegmentator.run(cell_sample, labels)\n                segments = individual_label_predictor.run(segments, batch_size=16, n_augments=4)\n                file.write(format_sample(cell_sample, segments))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nimport numpy\nimport torch\n\n\ndef set_random_seed(seed):\n    torch.manual_seed(seed)\n    random.seed(seed)\n    numpy.random.seed(seed)\n\nset_random_seed(1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if len(test_df) == 559:\n    inference_with_precomputed_labels()\nelse:\n    inference()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del segment_model\ndel dataset\ndel data_loader\ndel individual_label_predictor\ndel labeling_model\ndel labeling_model_keepres","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()\ntorch.cuda.empty_cache()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## calc additional score","metadata":{}},{"cell_type":"code","source":"def build_decoder(with_labels=True, target_size=(300, 300), ext='jpg'):\n    \n    def decode(path):\n        file_bytes = tf.io.read_file(path)\n        if ext == 'png':\n            img = tf.image.decode_png(file_bytes, channels=3)\n        elif ext in ['jpg', 'jpeg']:\n            img = tf.image.decode_jpeg(file_bytes, channels=3)\n        else:\n            raise ValueError(\"Image extension not supported\")\n        img = tf.cast(img, tf.float32) / 255.0\n        img = tf.image.resize(img, target_size)\n        return img\n    \n    def decode_with_labels(path, label):\n        return decode(path), label\n\n    return decode_with_labels if with_labels else decode\n\ndef build_augmenter(with_labels=True):\n    \n    def augment(img):\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_flip_up_down(img)\n        return img\n    \n    def augment_with_labels(img, label):\n        return augment(img), label\n    \n    return augment_with_labels if with_labels else augment\n\n\ndef build_dataset(paths, labels=None, bsize=32, cache=True,\n                  decode_fn=None, augment_fn=None,\n                  augment=True, repeat=True, shuffle=1024, \n                  cache_dir=\"\"):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n    if augment_fn is None:\n        augment_fn = build_augmenter(labels is not None)\n\n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths if labels is None else (paths, labels)\n\n    dset = tf.data.Dataset.from_tensor_slices(slices)\n    dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n    dset = dset.cache(cache_dir) if cache else dset\n    dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.repeat() if repeat else dset\n    dset = dset.shuffle(shuffle) if shuffle else dset\n    dset = dset.batch(bsize).prefetch(AUTO)\n\n    return dset","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sys.path.append(\"/kaggle/input/efficientnet-keras-source-code/\")\nsys.path.append(\"/kaggle/input/kerasapplications\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom efficientnet.tfkeras import EfficientNetB4\n\nwhole_image_model = tf.keras.models.load_model('../input/hpa-models/HPA classification efnb7 train 13cc0d 20/model_green.h5')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"additional_score_df = pd.read_csv(test_df_path)\nadditional_score_df  = additional_score_df.drop(additional_score_df.columns[1:],axis=1)\nfor i in range(19):\n    additional_score_df[f'{i}'] = pd.Series(np.zeros(additional_score_df.shape[0]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"load_dir = f\"/kaggle/input/hpa-single-cell-image-classification/\"\n\ntest_paths = load_dir + \"/test/\" + additional_score_df['ID'] + '_' + 'green' + '.png' # Start making individul label\nlabel_cols = additional_score_df.columns[1:] # Get the multi-labels\ntest_decoder = build_decoder(with_labels=False, target_size=(720,720))\ndtest = build_dataset(\n    test_paths, bsize=8, repeat=False, \n    shuffle=False, augment=False, cache=False,\n    decode_fn=test_decoder\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"additional_score_df[label_cols] = whole_image_model.predict(dtest, verbose=1)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_df = pd.read_csv(\"submission_base.csv\")\nbase_df = pd.merge(base_df, additional_score_df, on = 'ID', how = 'left')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for row in range(base_df.shape[0]):\n    pred = base_df.loc[row,'PredictionString']\n    pred_split = pred.split()\n    for j in range(int(len(pred_split)/3)):        \n        for k in range(19):\n            if int(pred_split[ 3*j ]) == k:\n                p = pred_split[ 3*j + 1 ]               \n                pred_split[ 3*j + 1 ] = str( base_df.loc[row, f'{k}']*0.6 + float(p)*0.4 )\n    base_df.loc[row,'PredictionString'] = ' '.join(pred_split)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_df = base_df[['ID','ImageWidth','ImageHeight','PredictionString']]\nbase_df.to_csv('submission.csv',index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}