{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":70173,"databundleVersionId":7683204,"sourceType":"competition"},{"sourceId":7342249,"sourceType":"datasetVersion","datasetId":4263116},{"sourceId":6244581,"sourceType":"datasetVersion","datasetId":3587755}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"2stage のMILで１段目の特徴抽出を行います。\nUBCコンペなどを参考にしています。","metadata":{}},{"cell_type":"code","source":"import hydra\nfrom omegaconf import DictConfig, OmegaConf\nimport sys,gc,os,random,time,math,glob\nos.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n\nimport matplotlib.pyplot as plt\nfrom contextlib import contextmanager\nfrom pathlib import Path\nfrom collections import defaultdict, Counter\nfrom  torch.cuda.amp import autocast, GradScaler \nimport cv2,timm\nfrom sklearn.metrics import roc_auc_score\nfrom PIL import Image\nimport numpy as np\nimport pandas as pd\nimport scipy as sp\nimport sklearn.metrics as metrics\nfrom sklearn.model_selection import StratifiedKFold,GroupKFold\nfrom sklearn.metrics import log_loss\nfrom functools import partial\nfrom tqdm import tqdm\nfrom sklearn.metrics import precision_score,recall_score,f1_score,log_loss\nfrom  sklearn.metrics import accuracy_score as acc\nimport torch\nimport torch.nn as nn\nfrom torch.optim import Adam, SGD,AdamW\nfrom torch.optim.lr_scheduler import CosineAnnealingLR, ReduceLROnPlateau,CosineAnnealingWarmRestarts\nfrom torch.utils.data import DataLoader, Dataset\nfrom albumentations import Compose, Normalize, HorizontalFlip, VerticalFlip,RandomGamma, RandomRotate90,GaussNoise,Cutout,RandomBrightnessContrast,RandomContrast,Resize\nfrom albumentations.pytorch import ToTensorV2\nimport transformers as T\n\nimport albumentations as A\nimport vision_transformer as vits\n\n\n###\n\nimport logging\n#from mylib.\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n# Copyright (c) Facebook, Inc. and its affiliates.\n# \n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n# \n#     http://www.apache.org/licenses/LICENSE-2.0\n# \n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"\nMostly copy-paste from timm library.\nhttps://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py\n\"\"\"\nimport math\nfrom functools import partial\n\nimport torch\nimport torch.nn as nn\n\n\n\ndef _no_grad_trunc_normal_(tensor, mean, std, a, b):\n    # Cut & paste from PyTorch official master until it's in a few official releases - RW\n    # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf\n    def norm_cdf(x):\n        # Computes standard normal cumulative distribution function\n        return (1. + math.erf(x / math.sqrt(2.))) / 2.\n\n    if (mean < a - 2 * std) or (mean > b + 2 * std):\n        warnings.warn(\"mean is more than 2 std from [a, b] in nn.init.trunc_normal_. \"\n                      \"The distribution of values may be incorrect.\",\n                      stacklevel=2)\n\n    with torch.no_grad():\n        # Values are generated by using a truncated uniform distribution and\n        # then using the inverse CDF for the normal distribution.\n        # Get upper and lower cdf values\n        l = norm_cdf((a - mean) / std)\n        u = norm_cdf((b - mean) / std)\n\n        # Uniformly fill tensor with values from [l, u], then translate to\n        # [2l-1, 2u-1].\n        tensor.uniform_(2 * l - 1, 2 * u - 1)\n\n        # Use inverse cdf transform for normal distribution to get truncated\n        # standard normal\n        tensor.erfinv_()\n\n        # Transform to proper mean, std\n        tensor.mul_(std * math.sqrt(2.))\n        tensor.add_(mean)\n\n        # Clamp to ensure it's in the proper range\n        tensor.clamp_(min=a, max=b)\n        return tensor\n\n\ndef trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.):\n    # type: (Tensor, float, float, float, float) -> Tensor\n    return _no_grad_trunc_normal_(tensor, mean, std, a, b)\n\n\ndef drop_path(x, drop_prob: float = 0., training: bool = False):\n    if drop_prob == 0. or not training:\n        return x\n    keep_prob = 1 - drop_prob\n    shape = (x.shape[0],) + (1,) * (x.ndim - 1)  # work with diff dim tensors, not just 2D ConvNets\n    random_tensor = keep_prob + torch.rand(shape, dtype=x.dtype, device=x.device)\n    random_tensor.floor_()  # binarize\n    output = x.div(keep_prob) * random_tensor\n    return output\n\n\nclass DropPath(nn.Module):\n    \"\"\"Drop paths (Stochastic Depth) per sample  (when applied in main path of residual blocks).\n    \"\"\"\n    def __init__(self, drop_prob=None):\n        super(DropPath, self).__init__()\n        self.drop_prob = drop_prob\n\n    def forward(self, x):\n        return drop_path(x, self.drop_prob, self.training)\n\n\nclass Mlp(nn.Module):\n    def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.):\n        super().__init__()\n        out_features = out_features or in_features\n        hidden_features = hidden_features or in_features\n        self.fc1 = nn.Linear(in_features, hidden_features)\n        self.act = act_layer()\n        self.fc2 = nn.Linear(hidden_features, out_features)\n        self.drop = nn.Dropout(drop)\n\n    def forward(self, x):\n        x = self.fc1(x)\n        x = self.act(x)\n        x = self.drop(x)\n        x = self.fc2(x)\n        x = self.drop(x)\n        return x\n\n\nclass Attention(nn.Module):\n    def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0.):\n        super().__init__()\n        self.num_heads = num_heads\n        head_dim = dim // num_heads\n        self.scale = qk_scale or head_dim ** -0.5\n\n        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)\n        self.attn_drop = nn.Dropout(attn_drop)\n        self.proj = nn.Linear(dim, dim)\n        self.proj_drop = nn.Dropout(proj_drop)\n\n    def forward(self, x):\n        B, N, C = x.shape\n        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)\n        q, k, v = qkv[0], qkv[1], qkv[2]\n\n        attn = (q @ k.transpose(-2, -1)) * self.scale\n        attn = attn.softmax(dim=-1)\n        attn = self.attn_drop(attn)\n\n        x = (attn @ v).transpose(1, 2).reshape(B, N, C)\n        x = self.proj(x)\n        x = self.proj_drop(x)\n        return x, attn\n\n\nclass Block(nn.Module):\n    def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0.,\n                 drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm):\n        super().__init__()\n        self.norm1 = norm_layer(dim)\n        self.attn = Attention(\n            dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop)\n        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()\n        self.norm2 = norm_layer(dim)\n        mlp_hidden_dim = int(dim * mlp_ratio)\n        self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)\n\n    def forward(self, x, return_attention=False):\n        y, attn = self.attn(self.norm1(x))\n        if return_attention:\n            return attn\n        x = x + self.drop_path(y)\n        x = x + self.drop_path(self.mlp(self.norm2(x)))\n        return x\n\n\nclass PatchEmbed(nn.Module):\n    \"\"\" Image to Patch Embedding\n    \"\"\"\n    def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768):\n        super().__init__()\n        num_patches = (img_size // patch_size) * (img_size // patch_size)\n        self.img_size = img_size\n        self.patch_size = patch_size\n        self.num_patches = num_patches\n\n        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)\n\n    def forward(self, x):\n        B, C, H, W = x.shape\n        x = self.proj(x).flatten(2).transpose(1, 2)\n        return x\n\n\nclass VisionTransformer(nn.Module):\n    \"\"\" Vision Transformer \"\"\"\n    def __init__(self, img_size=[224], patch_size=16, in_chans=3, num_classes=0, embed_dim=768, depth=12,\n                 num_heads=12, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop_rate=0., attn_drop_rate=0.,\n                 drop_path_rate=0., norm_layer=nn.LayerNorm, **kwargs):\n        super().__init__()\n        self.num_features = self.embed_dim = embed_dim\n\n        self.patch_embed = PatchEmbed(\n            img_size=img_size[0], patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)\n        num_patches = self.patch_embed.num_patches\n\n        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))\n        self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, embed_dim))\n        self.pos_drop = nn.Dropout(p=drop_rate)\n\n        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule\n        self.blocks = nn.ModuleList([\n            Block(\n                dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale,\n                drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer)\n            for i in range(depth)])\n        self.norm = norm_layer(embed_dim)\n\n        # Classifier head\n        self.head = nn.Linear(embed_dim, num_classes) if num_classes > 0 else nn.Identity()\n\n        trunc_normal_(self.pos_embed, std=.02)\n        trunc_normal_(self.cls_token, std=.02)\n        self.apply(self._init_weights)\n\n    def _init_weights(self, m):\n        if isinstance(m, nn.Linear):\n            trunc_normal_(m.weight, std=.02)\n            if isinstance(m, nn.Linear) and m.bias is not None:\n                nn.init.constant_(m.bias, 0)\n        elif isinstance(m, nn.LayerNorm):\n            nn.init.constant_(m.bias, 0)\n            nn.init.constant_(m.weight, 1.0)\n\n    def interpolate_pos_encoding(self, x, w, h):\n        npatch = x.shape[1] - 1\n        N = self.pos_embed.shape[1] - 1\n        if npatch == N and w == h:\n            return self.pos_embed\n        class_pos_embed = self.pos_embed[:, 0]\n        patch_pos_embed = self.pos_embed[:, 1:]\n        dim = x.shape[-1]\n        w0 = w // self.patch_embed.patch_size\n        h0 = h // self.patch_embed.patch_size\n        # we add a small number to avoid floating point error in the interpolation\n        # see discussion at https://github.com/facebookresearch/dino/issues/8\n        w0, h0 = w0 + 0.1, h0 + 0.1\n        patch_pos_embed = nn.functional.interpolate(\n            patch_pos_embed.reshape(1, int(math.sqrt(N)), int(math.sqrt(N)), dim).permute(0, 3, 1, 2),\n            scale_factor=(w0 / math.sqrt(N), h0 / math.sqrt(N)),\n            mode='bicubic',\n        )\n        assert int(w0) == patch_pos_embed.shape[-2] and int(h0) == patch_pos_embed.shape[-1]\n        patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)\n        return torch.cat((class_pos_embed.unsqueeze(0), patch_pos_embed), dim=1)\n\n    def prepare_tokens(self, x):\n        B, nc, w, h = x.shape\n        x = self.patch_embed(x)  # patch linear embedding\n\n        # add the [CLS] token to the embed patch tokens\n        cls_tokens = self.cls_token.expand(B, -1, -1)\n        x = torch.cat((cls_tokens, x), dim=1)\n\n        # add positional encoding to each token\n        x = x + self.interpolate_pos_encoding(x, w, h)\n\n        return self.pos_drop(x)\n\n    def forward(self, x):\n        x = self.prepare_tokens(x)\n        for blk in self.blocks:\n            x = blk(x)\n        x = self.norm(x)\n        return self.head(x[:, 0])\n    \n\n\n    def get_last_selfattention(self, x):\n        x = self.prepare_tokens(x)\n        for i, blk in enumerate(self.blocks):\n            if i < len(self.blocks) - 1:\n                x = blk(x)\n            else:\n                # return attention of the last block\n                return blk(x, return_attention=True)\n\n    def get_intermediate_layers(self, x, n=1):\n        x = self.prepare_tokens(x)\n        # we return the output tokens from the `n` last blocks\n        output = []\n        for i, blk in enumerate(self.blocks):\n            x = blk(x)\n            if len(self.blocks) - i <= n:\n                output.append(self.norm(x))\n        return output\n    def get_intermediate_forward(self, x, n=4):\n        x = self.prepare_tokens(x)\n        # we return the output tokens from the `n` last blocks\n        output = []\n        for i, blk in enumerate(self.blocks):\n            x = blk(x)\n            if len(self.blocks) - i <= n:\n                output.append(self.norm(x))\n        output = torch.cat([x[:, 0] for x in output], dim=-1)\n        return self.head(output)\n\n\ndef vit_tiny(patch_size=16, **kwargs):\n    model = VisionTransformer(\n        patch_size=patch_size, embed_dim=192, depth=12, num_heads=3, mlp_ratio=4,\n        qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs)\n    return model\n\n\ndef vit_small(patch_size=16, **kwargs):\n    model = VisionTransformer(\n        patch_size=patch_size, embed_dim=384, depth=12, num_heads=6, mlp_ratio=4,\n        qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs)\n    return model\n\n\ndef vit_base(patch_size=16, **kwargs):\n    model = VisionTransformer(\n        patch_size=patch_size, embed_dim=768, depth=12, num_heads=12, mlp_ratio=4,\n        qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs)\n    return model\n\ndef vit_large(patch_size=16, **kwargs):\n    model = VisionTransformer(\n        patch_size=patch_size, embed_dim=1024, depth=24, num_heads=16, mlp_ratio=4,\n        qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs)\n    return model\n\n\n    \n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/aipathology202403/train.csv\")\n\nmodel_name = \"owkin\"\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform = Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n\nif model_name==\"lunit\":\n    from timm.models.vision_transformer import VisionTransformer\n\n\n    def get_pretrained_url(key):\n        URL_PREFIX = \"https://github.com/lunit-io/benchmark-ssl-pathology/releases/download/pretrained-weights\"\n        model_zoo_registry = {\n            \"DINO_p16\": \"dino_vit_small_patch16_ep200.torch\",\n            \"DINO_p8\": \"dino_vit_small_patch8_ep200.torch\",\n        }\n        pretrained_url = f\"{URL_PREFIX}/{model_zoo_registry.get(key)}\"\n        return pretrained_url\n\n\n    def vit_small(pretrained, progress, key, **kwargs):\n        patch_size = kwargs.get(\"patch_size\", 16)\n        model = VisionTransformer(\n            img_size=224, patch_size=patch_size, embed_dim=384, num_heads=6, num_classes=0\n        )\n        if pretrained:\n            pretrained_url = get_pretrained_url(key)\n            verbose = model.load_state_dict(\n                torch.hub.load_state_dict_from_url(pretrained_url, progress=progress)\n            )\n            print(verbose)\n        return model\n    model = vit_small(pretrained=True, progress=False, key=\"DINO_p16\", patch_size=16)\n    \n    transform = Compose([\n            Resize(224,224),\n            Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n            ])\n    \nelif model_name==\"hipt\":\n    import vision_transformer as vits\n    model = vits.__dict__[\"vit_small\"](patch_size=16)\n    pretrained_weights = \"/kaggle/input/hiptweight/vit256_small_dino.pth\"\n    state_dict = torch.load(pretrained_weights, map_location=\"cpu\")[\"teacher\"]\n    state_dict = {k.replace(\"module.\", \"\"): v for k, v in state_dict.items()}\n    state_dict = {k.replace(\"backbone.\", \"\"): v for k, v in state_dict.items()}\n    model.load_state_dict(state_dict, strict=False)\n\n    model.head = nn.Identity()\n    \nelif model_name==\"ctransptah\":\n    from  Ctranspath import CTransPath\n    model = CTransPath(num_classes=0)\n    model.load_state_dict(torch.load(f'/kaggle/input/ubc-ocean-7th/weights/ctranspath/1.pt')['model'], strict=False)\nelif model_name==\"owkin\":\n    from transformers import AutoImageProcessor, ViTModel,ViTConfig\n    \n\n    class ViTModel_owkin(nn.Module):\n        def __init__(self):\n            super(ViTModel_owkin, self).__init__() \n            config = ViTConfig.from_pretrained(\"owkin/phikon\")\n            config.image_size = 256\n            self.model = ViTModel.from_pretrained(\"owkin/phikon\", add_pooling_layer=True,config=config,ignore_mismatched_sizes=True)\n        def forward(self, x):\n            x = self.model(pixel_values=x)[\"pooler_output\"]\n            return x \n        \n    model = ViTModel_owkin()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"   \nmodel.cuda()\nmodel.eval()\n\nos.makedirs(f\"/kaggle/working/train/{model_name}\",exist_ok=True)\nos.makedirs(f\"/kaggle/working/test/{model_name}\",exist_ok=True)\n\n\n#作成済み\nfor i in range(df.shape[0]):\n    row =df.iloc[i]\n    image_id = int(row[\"ID\"])\n    if os.path.exists(f\"/kaggle/working/train/{model_name}/{image_id}.npy\"):continue\n    imgs = [cv2.imread(f\"/home/abe/pandasub/subset/train/{image_id}_{i}.png\")[:,:,::-1] for i in range(36)]\n    \n    imgs =  np.stack([transform(image=img)['image']  for img in imgs])\n    image = torch.from_numpy(imgs.transpose(0,3,1,2)).float().cuda()\n    \n    \n    with torch.no_grad():\n        out = model(image).to('cpu').numpy()\n    \n    np.save(f\"/kaggle/working/train/{model_name}/{image_id}.npy\",out)\n\n    \n    \ndf = pd.read_csv(\"/kaggle/input/aipathology202403/sample_submission.csv\")  \nfor i in range(df.shape[0]):\n    row =df.iloc[i]\n    image_id = int(row[\"ID\"])\n    if os.path.exists(f\"/kaggle/working/test/{model_name}/{image_id}.npy\"):continue\n\n    imgs = [cv2.imread(f\"/kaggle/input/aipathology202403/test (1)/{image_id}_{i}.png\")[:,:,::-1] for i in range(36)]\n    \n    imgs =  np.stack([transform(image=img)['image']  for img in imgs])\n    image = torch.from_numpy(imgs.transpose(0,3,1,2)).float().cuda()\n    \n    \n    with torch.no_grad():\n        out = model(image).to('cpu').numpy()\n    \n    np.save(f\"/kaggle/working/test/{model_name}/{image_id}.npy\",out)\n    \n    \n    \n    \n    \n    \n    \n    ","metadata":{},"execution_count":null,"outputs":[]}]}