{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"_____\n**Credits:**\n- Parts of this notebook are based on [this](https://www.kaggle.com/code/carloalbertobarbano/pytorch-ensemble-pretrained-baselines-training) great notebook by [Carlo Alberto](https://www.kaggle.com/carloalbertobarbano).\n- Parts of this notebook are based on [this](https://www.kaggle.com/code/hwigeon/dino-pretrained-weight-inference) great notebook by [HWIGEON OH](https://www.kaggle.com/hwigeon).\n_____","metadata":{}},{"cell_type":"markdown","source":"## Embeddings Brute Force\n\nOn this notebook we try to find the best combination of embeddings simply by trial and error. \nFor each submission we try to combine multiple efficientnet networks and observe the resulting score. \n\nUse it as a reference to check what works well (and what doesn't work well). \n\n**Make sure to check previous versions so you can compare different ensembles and base models**","metadata":{}},{"cell_type":"code","source":"import gc\nimport os\nimport sys\nimport torch\nimport torch.nn as nn\nfrom zipfile import ZipFile\nfrom functools import partial\nfrom torchvision import models\nfrom torchvision import transforms","metadata":{"execution":{"iopub.status.busy":"2022-07-21T13:34:39.6725Z","iopub.execute_input":"2022-07-21T13:34:39.673034Z","iopub.status.idle":"2022-07-21T13:34:41.74747Z","shell.execute_reply.started":"2022-07-21T13:34:39.67293Z","shell.execute_reply":"2022-07-21T13:34:41.746414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Since we don't have internet..\n..we have to \"bring along\" the trained models weights as a datasets. \nI made multiple datasets containing pretrained models of multiple backbones. (see: supported models down below)\n\n**Making the dataset**\n\nMaking this dataset was extremely easy:\n- I simply executed this notebook with the internet turned on. \n- Saved everything that was downloaded on to the machine. \n- Uploaded the cached files as a seperate dataset.\n- Turned the internet off. \n- (profit)\n","metadata":{}},{"cell_type":"code","source":"!mkdir -p ~/.cache/torch/hub/checkpoints/\n\n# Timm::ConvNext\n!cp -r ../input/convnext-weights/convnext/* /root/.cache/torch/hub/checkpoints\n\n# Torchvision::EfficientNet\n!cp -r ../input/torchvision-efficientnets/* ~/.cache/torch/hub/checkpoints/\n\n# Hub::DINO-VIT\n!cp -r ../input/torchhub-dino-vit/hub/* ~/.cache/torch/hub/\nsys.path.append('/root/.cache/torch/hub/facebookresearch_dino_main')\n\n# Timm::ConvNext\nsys.path.append(\"../input/timmmaster/\")","metadata":{"execution":{"iopub.status.busy":"2022-07-21T13:34:41.749192Z","iopub.execute_input":"2022-07-21T13:34:41.74969Z","iopub.status.idle":"2022-07-21T13:36:44.464197Z","shell.execute_reply.started":"2022-07-21T13:34:41.749659Z","shell.execute_reply":"2022-07-21T13:36:44.46289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# code from https://github.com/facebookresearch/dino/blob/main/vision_transformer.py\n# Little fix for jit\n\n\"\"\"\nMostly copy-paste from timm library.\nhttps://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py\n\"\"\"\nimport math\nimport timm\nfrom functools import partial\n\nimport torch\nimport torch.nn as nn\n\nfrom utils import trunc_normal_\n\n\ndef drop_path(x, drop_prob: float = 0., training: bool = False):\n    if drop_prob == 0. or not training:\n        return x\n    keep_prob = 1 - drop_prob\n    shape = (x.shape[0],) + (1,) * (x.ndim - 1)  # work with diff dim tensors, not just 2D ConvNets\n    random_tensor = keep_prob + torch.rand(shape, dtype=x.dtype, device=x.device)\n    random_tensor.floor_()  # binarize\n    output = x.div(keep_prob) * random_tensor\n    return output\n\n\nclass DropPath(nn.Module):\n    \"\"\"Drop paths (Stochastic Depth) per sample  (when applied in main path of residual blocks).\n    \"\"\"\n    def __init__(self, drop_prob=None):\n        super(DropPath, self).__init__()\n        self.drop_prob = drop_prob\n\n    def forward(self, x):\n        return drop_path(x, self.drop_prob, self.training)\n\n\nclass Mlp(nn.Module):\n    def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.):\n        super().__init__()\n        out_features = out_features or in_features\n        hidden_features = hidden_features or in_features\n        self.fc1 = nn.Linear(in_features, hidden_features)\n        self.act = act_layer()\n        self.fc2 = nn.Linear(hidden_features, out_features)\n        self.drop = nn.Dropout(drop)\n\n    def forward(self, x):\n        x = self.fc1(x)\n        x = self.act(x)\n        x = self.drop(x)\n        x = self.fc2(x)\n        x = self.drop(x)\n        return x\n\n\nclass Attention(nn.Module):\n    def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0.):\n        super().__init__()\n        self.num_heads = num_heads\n        head_dim = dim // num_heads\n        self.scale = qk_scale or head_dim ** -0.5\n\n        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)\n        self.attn_drop = nn.Dropout(attn_drop)\n        self.proj = nn.Linear(dim, dim)\n        self.proj_drop = nn.Dropout(proj_drop)\n\n    def forward(self, x):\n        B, N, C = x.shape\n        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)\n        q, k, v = qkv[0], qkv[1], qkv[2]\n\n        attn = (q @ k.transpose(-2, -1)) * self.scale\n        attn = attn.softmax(dim=-1)\n        attn = self.attn_drop(attn)\n\n        x = (attn @ v).transpose(1, 2).reshape(B, N, C)\n        x = self.proj(x)\n        x = self.proj_drop(x)\n        return x, attn\n\n\nclass Block(nn.Module):\n    def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0.,\n                 drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm):\n        super().__init__()\n        self.norm1 = norm_layer(dim)\n        self.attn = Attention(\n            dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop)\n        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()\n        self.norm2 = norm_layer(dim)\n        mlp_hidden_dim = int(dim * mlp_ratio)\n        self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)\n\n    def forward(self, x):\n        y, attn = self.attn(self.norm1(x))\n        x = x + self.drop_path(y)\n        x = x + self.drop_path(self.mlp(self.norm2(x)))\n        return x\n\n\nclass PatchEmbed(nn.Module):\n    \"\"\" Image to Patch Embedding\n    \"\"\"\n    def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768):\n        super().__init__()\n        num_patches = (img_size // patch_size) * (img_size // patch_size)\n        self.img_size = img_size\n        self.patch_size = patch_size\n        self.num_patches = num_patches\n\n        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)\n\n    def forward(self, x):\n        B, C, H, W = x.shape\n        x = self.proj(x).flatten(2).transpose(1, 2)\n        return x\n\n\nclass VisionTransformer(nn.Module):\n    \"\"\" Vision Transformer \"\"\"\n    def __init__(self, img_size=[224], patch_size=16, in_chans=3, num_classes=0, embed_dim=768, depth=12,\n                 num_heads=12, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop_rate=0., attn_drop_rate=0.,\n                 drop_path_rate=0., norm_layer=nn.LayerNorm, **kwargs):\n        super().__init__()\n        self.num_features = self.embed_dim = embed_dim\n\n        self.patch_embed = PatchEmbed(\n            img_size=img_size[0], patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)\n        num_patches = self.patch_embed.num_patches\n\n        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))\n        self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, embed_dim))\n        self.pos_drop = nn.Dropout(p=drop_rate)\n\n        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule\n        self.blocks = nn.ModuleList([\n            Block(\n                dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale,\n                drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer)\n            for i in range(depth)])\n        self.norm = norm_layer(embed_dim)\n\n        # Classifier head\n        self.head = nn.Linear(embed_dim, num_classes) if num_classes > 0 else nn.Identity()\n\n        trunc_normal_(self.pos_embed, std=.02)\n        trunc_normal_(self.cls_token, std=.02)\n        self.apply(self._init_weights)\n\n    def _init_weights(self, m):\n        if isinstance(m, nn.Linear):\n            trunc_normal_(m.weight, std=.02)\n            if isinstance(m, nn.Linear) and m.bias is not None:\n                nn.init.constant_(m.bias, 0)\n        elif isinstance(m, nn.LayerNorm):\n            nn.init.constant_(m.bias, 0)\n            nn.init.constant_(m.weight, 1.0)\n\n    def interpolate_pos_encoding(self, x, w: int, h: int):\n        npatch = x.shape[1] - 1\n        N = self.pos_embed.shape[1] - 1\n        if npatch == N and w == h:\n            return self.pos_embed\n        class_pos_embed = self.pos_embed[:, 0]\n        patch_pos_embed = self.pos_embed[:, 1:]\n        dim = x.shape[-1]\n        w0 = w // self.patch_embed.patch_size\n        h0 = h // self.patch_embed.patch_size\n        # we add a small number to avoid floating point error in the interpolation\n        # see discussion at https://github.com/facebookresearch/dino/issues/8\n        w0, h0 = w0 + 0.1, h0 + 0.1\n        patch_pos_embed = nn.functional.interpolate(\n            patch_pos_embed.reshape(1, int(math.sqrt(N)), int(math.sqrt(N)), dim).permute(0, 3, 1, 2),\n            scale_factor=[float(w0 / math.sqrt(N)), float(h0 / math.sqrt(N))],\n            mode='bicubic',\n        )\n        assert int(w0) == patch_pos_embed.shape[-2] and int(h0) == patch_pos_embed.shape[-1]\n        patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)\n        return torch.cat((class_pos_embed.unsqueeze(0), patch_pos_embed), dim=1)\n\n    def prepare_tokens(self, x):\n        B, nc, w, h = x.shape\n        x = self.patch_embed(x)  # patch linear embedding\n\n        # add the [CLS] token to the embed patch tokens\n        cls_tokens = self.cls_token.expand(B, -1, -1)\n        x = torch.cat((cls_tokens, x), dim=1)\n\n        # add positional encoding to each token\n        x = x + self.interpolate_pos_encoding(x, w, h)\n\n        return self.pos_drop(x)\n\n    def forward(self, x):\n        x = self.prepare_tokens(x)\n        for blk in self.blocks:\n            x = blk(x)\n        x = self.norm(x)\n        return x[:, 0]\n\n    def get_last_selfattention(self, x):\n        x = self.prepare_tokens(x)\n        for i, blk in enumerate(self.blocks):\n            if i < len(self.blocks) - 1:\n                x = blk(x)\n            else:\n                # return attention of the last block\n                return blk(x, return_attention=True)\n\n    def get_intermediate_layers(self, x, n=1):\n        x = self.prepare_tokens(x)\n        # we return the output tokens from the `n` last blocks\n        output = []\n        for i, blk in enumerate(self.blocks):\n            x = blk(x)\n            if len(self.blocks) - i <= n:\n                output.append(self.norm(x))\n        return output\n\ndef vit_tiny(patch_size=16, **kwargs):\n    model = VisionTransformer(\n        patch_size=patch_size, embed_dim=192, depth=12, num_heads=3, mlp_ratio=4,\n        qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs)\n    return model\n\n\ndef vit_small(patch_size=16, **kwargs):\n    model = VisionTransformer(\n        patch_size=patch_size, embed_dim=384, depth=12, num_heads=6, mlp_ratio=4,\n        qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs)\n    return model\n\n\ndef vit_base(patch_size=16, **kwargs):\n    model = VisionTransformer(\n        patch_size=patch_size, embed_dim=768, depth=12, num_heads=12, mlp_ratio=4,\n        qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs)\n    return model    ","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-21T13:36:44.466281Z","iopub.execute_input":"2022-07-21T13:36:44.466657Z","iopub.status.idle":"2022-07-21T13:36:56.285932Z","shell.execute_reply.started":"2022-07-21T13:36:44.466624Z","shell.execute_reply":"2022-07-21T13:36:56.284876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Bruteforce Configurations\n\n**The fun part:** Now we get to test out different bruteforce configurations. \nWe first start with what backbone architectures we want to use and if we should use mean or median. \n\n\nI will update this later with different approaches and ideas. \n\n_____\n\n#### Supported Models \n###### (Last update: 07.21.2022)\n\n**EfficientNet**\n\n- `b1` - EfficientNet B1\n- `b2` - EfficientNet B2\n- `b3` - EfficientNet B3\n- `b4` - EfficientNet B4\n- `b5` - EfficientNet B5\n- `b6` - EfficientNet B6\n- `b7` - EfficientNet B7\n\n**Vision Transformers (ViT) + DINO**\n \n- `dino_vitb8` - DINO ViT B8\n- `dino_vits8` - DINO ViT S8\n- `dino_vits16` - DINO ViT S16\n- `dino_vitsb16` - DINO ViT S16\n\n**ConvNeXt Models**\n \n- `convnext_base_in22k` - ConvNeXt-Base\n- `convnext_xlarge_in22k` - ConvNeXt-XL\n- `convnext_large_in22k` - ConvNeXt-Large\n- `convnext_xlarge_in22ft1k` - ConvNeXt-XL\n- `convnext_large_in22ft1k` - ConvNeXt-Large\n- `convnext_base_384_in22ft1k` - ConvNeXt-Base\n- `convnext_xlarge_384_in22ft1k` - ConvNeXt-XL\n- `convnext_large_384_in22ft1k` - ConvNeXt-Large\n\n_____","metadata":{}},{"cell_type":"code","source":"BATCH_BLENDING_TYPE = 'mean'\nBACKBONES = ['convnext_xlarge_in22k', 'b4', 'dino_vits8'] \n\n# Top LB models:\n# 1. 'convnext_xlarge_in22k'\n# 2. 'convnext_base_384_in22ft1k'\n# 3. 'convnext_large_in22k'","metadata":{"execution":{"iopub.status.busy":"2022-07-21T13:56:20.801099Z","iopub.execute_input":"2022-07-21T13:56:20.80153Z","iopub.status.idle":"2022-07-21T13:56:20.807213Z","shell.execute_reply.started":"2022-07-21T13:56:20.801494Z","shell.execute_reply":"2022-07-21T13:56:20.805963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### ConvNeXt Model\n\n> Credit: [Notebook](https://www.kaggle.com/code/omomo17/convnext-xlarge-baseline/notebook) by [omomo](https://www.kaggle.com/omomo17).","metadata":{}},{"cell_type":"code","source":"class ConvNeXt(nn.Module):\n  def __init__(self, model_name, resize):\n    super().__init__()\n    self.resize = resize\n    this_model = base_model = timm.create_model(model_name, pretrained=True)\n    this_model.head.fc = nn.AdaptiveAvgPool1d(64)\n    self.feature_extractor = this_model\n\n  def forward(self, x):\n    x = transforms.functional.resize(x,size = [self.resize, self.resize])\n    x = x/255.0\n    x = transforms.functional.normalize(x, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n    return self.feature_extractor(x)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T13:36:56.294866Z","iopub.execute_input":"2022-07-21T13:36:56.295215Z","iopub.status.idle":"2022-07-21T13:36:56.310893Z","shell.execute_reply.started":"2022-07-21T13:36:56.295186Z","shell.execute_reply":"2022-07-21T13:36:56.309719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### EfficientNet Model\n\n> Credit: [Notebook](https://www.kaggle.com/code/carloalbertobarbano/pytorch-ensemble-pretrained-baselines-training) by [Carlo Alberto](https://www.kaggle.com/carloalbertobarbano).","metadata":{}},{"cell_type":"code","source":"class EfficientNet(nn.Module):\n  def __init__(self, encoder_fn, resize, size):\n    super().__init__()\n    encoder = encoder_fn(pretrained=True)\n    encoder.classifier = nn.AdaptiveAvgPool1d(64)\n    self.encoder = encoder\n    self.resize = resize \n    self.size = size\n\n  def forward(self, x):\n    x = transforms.functional.resize(x, self.resize)\n    x = transforms.functional.center_crop(x, self.size)\n    x = x/255.\n    x = transforms.functional.normalize(x, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n    return self.encoder(x)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T13:36:56.31257Z","iopub.execute_input":"2022-07-21T13:36:56.312929Z","iopub.status.idle":"2022-07-21T13:36:56.329645Z","shell.execute_reply.started":"2022-07-21T13:36:56.312897Z","shell.execute_reply":"2022-07-21T13:36:56.328608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### DINO Vit Model\n> Credit: [Notebook](https://www.kaggle.com/code/hwigeon/dino-pretrained-weight-inference) by [HWIGEON OH](https://www.kaggle.com/hwigeon).","metadata":{}},{"cell_type":"code","source":"class DINO(nn.Module):\n    def __init__(self, encoder, resize, size):\n        super().__init__()\n        self.encoder = encoder\n        self.pool = nn.AdaptiveAvgPool1d(64)\n        self.resize = resize \n        self.size = size\n        \n    def forward(self, x):\n        x = transforms.functional.resize(x, [self.resize, self.resize])\n        x = transforms.functional.center_crop(x, [self.size, self.size])\n        x = x / 255.\n        x = transforms.functional.normalize(x, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n        x = self.encoder(x)\n        x = self.pool(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-07-21T13:36:56.331081Z","iopub.execute_input":"2022-07-21T13:36:56.331496Z","iopub.status.idle":"2022-07-21T13:36:56.34082Z","shell.execute_reply.started":"2022-07-21T13:36:56.331454Z","shell.execute_reply":"2022-07-21T13:36:56.339853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Bruteforce Parameter Options: \"BACKBONES\"","metadata":{}},{"cell_type":"code","source":"eff_encoders_fn = {\n    'b0': models.efficientnet_b0,\n    'b1': models.efficientnet_b1,\n    'b2': models.efficientnet_b2,\n    'b3': models.efficientnet_b3,\n    'b4': models.efficientnet_b4,\n    'b5': models.efficientnet_b5,\n    'b6': models.efficientnet_b6,\n    'b7': models.efficientnet_b7,    \n}\n\ndino_encoders_names = {        \n    'dino_vitb8': vit_base,\n    'dino_vitb16': vit_base,\n    'dino_vits8': vit_small,\n    'dino_vits16': vit_small,    \n}\n\nsizes = {\n    'b0': (256, 224), 'b1': (256, 240), 'b2': (288, 288), 'b3': (320, 300),\n    'b4': (384, 380), 'b5': (489, 456), 'b6': (561, 528), 'b7': (633, 600),\n    'dino_vitb8': (256, 224), 'dino_vits8': (256, 224), \n    'dino_vitb16': (256, 224), 'dino_vits16': (256, 224),    \n        \n    'convnext_large_in22ft1k': (224, 224),\n    'convnext_xlarge_in22ft1k': (224, 224),    \n    'convnext_base_384_in22ft1k': (384, 384),\n    'convnext_large_384_in22ft1k': (384, 384),\n    'convnext_xlarge_384_in22ft1k': (384, 384),    \n    'convnext_base_in22k': (224, 224),\n    'convnext_large_in22k': (224, 224),\n    'convnext_xlarge_in22k': (224, 224),\n\n}","metadata":{"execution":{"iopub.status.busy":"2022-07-21T13:53:31.243226Z","iopub.execute_input":"2022-07-21T13:53:31.244935Z","iopub.status.idle":"2022-07-21T13:53:31.260058Z","shell.execute_reply.started":"2022-07-21T13:53:31.24488Z","shell.execute_reply":"2022-07-21T13:53:31.258796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Bruteforce Parameter Options: \"BLENDING_TYPE\"","metadata":{}},{"cell_type":"code","source":"class EnsembleMean(nn.Module):\n    def __init__(self, encoders):\n        super().__init__()\n        for idx, encoder in enumerate(encoders): setattr(self, f'encoder{idx}', encoder)\n        self.num_encoders = len(encoders)\n    \n    def forward(self, x):\n        y = []\n        for name, encoder in self.named_children(): y.append(encoder(x))\n        y = torch.cat(y, dim=0)\n        return y.mean(dim=0).unsqueeze(0)\n\nclass EnsembleMedian(nn.Module):\n    def __init__(self, encoders):\n        super().__init__()\n        for idx, encoder in enumerate(encoders): setattr(self, f'encoder{idx}', encoder)\n        self.num_encoders = len(encoders)\n    \n    def forward(self, x):\n        y = []\n        for name, encoder in self.named_children(): y.append(encoder(x))\n        y = torch.cat(y, dim=0)\n        return y.median(dim=0).unsqueeze(0)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T13:36:56.356038Z","iopub.execute_input":"2022-07-21T13:36:56.356771Z","iopub.status.idle":"2022-07-21T13:36:56.373081Z","shell.execute_reply.started":"2022-07-21T13:36:56.356712Z","shell.execute_reply":"2022-07-21T13:36:56.372009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Brute Force Execution","metadata":{}},{"cell_type":"markdown","source":"**BACKBONES**","metadata":{}},{"cell_type":"code","source":"encoders = []\nfor encoder_name in BACKBONES:\n    if encoder_name in sizes: size = sizes[encoder_name]\n    else: size = None\n    if 'convnext' in encoder_name:\n        encoders.append(ConvNeXt(model_name = encoder_name, resize = size[0]))        \n    if encoder_name in eff_encoders_fn:\n        encoders.append(EfficientNet(encoder_fn=eff_encoders_fn[encoder_name], resize=(size[0], size[0]), size=size))\n    elif encoder_name in dino_encoders_names:\n        encoder = dino_encoders_names[encoder_name](patch_size=8 if not '16' in encoder_name else 16)\n        encoder.load_state_dict(torch.hub.load('facebookresearch/dino:main', encoder_name).state_dict(),)\n        encoders.append(DINO(encoder, resize=size[0], size = size[1]))\nprint('Backbones done.')","metadata":{"execution":{"iopub.status.busy":"2022-07-21T13:53:32.693245Z","iopub.execute_input":"2022-07-21T13:53:32.694091Z","iopub.status.idle":"2022-07-21T13:54:33.91663Z","shell.execute_reply.started":"2022-07-21T13:53:32.694046Z","shell.execute_reply":"2022-07-21T13:54:33.91535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**BLENDING_TYPE**","metadata":{}},{"cell_type":"code","source":"if BATCH_BLENDING_TYPE == 'mean': ensemble = EnsembleMean(encoders)\nelif BATCH_BLENDING_TYPE == 'median': ensemble = EnsembleMedian(encoders)\nensemble.eval()\nprint('Batch blending type done.')","metadata":{"execution":{"iopub.status.busy":"2022-07-21T13:38:10.54904Z","iopub.execute_input":"2022-07-21T13:38:10.549463Z","iopub.status.idle":"2022-07-21T13:38:10.557397Z","shell.execute_reply.started":"2022-07-21T13:38:10.549429Z","shell.execute_reply":"2022-07-21T13:38:10.556081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submission\n\nNow that we have our models ready, we can submit them and observe the LB score.","metadata":{}},{"cell_type":"code","source":"saved_model = torch.jit.script(ensemble)\nsaved_model.save('saved_model.pt')\n\nwith ZipFile('submission.zip','w') as zip:           \n    zip.write('./saved_model.pt', arcname='saved_model.pt') ","metadata":{"execution":{"iopub.status.busy":"2022-07-21T13:38:10.5592Z","iopub.execute_input":"2022-07-21T13:38:10.559606Z","iopub.status.idle":"2022-07-21T13:38:18.628696Z","shell.execute_reply.started":"2022-07-21T13:38:10.559525Z","shell.execute_reply":"2022-07-21T13:38:18.627637Z"},"trusted":true},"execution_count":null,"outputs":[]}]}