{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport torch\nimport random\nimport albumentations\nimport cv2\nimport math\nimport copy\nimport pytorch_lightning as pl\nimport torchmetrics\nimport wandb\nfrom sklearn.model_selection import train_test_split\nfrom torch.utils.data import Dataset\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom torchvision import transforms\nfrom pytorch_lightning import Callback, LightningModule, Trainer\nfrom pytorch_lightning.core.lightning import LightningModule\nfrom pytorch_lightning.loggers import WandbLogger\nfrom torch import nn, Tensor\nfrom torch.nn import functional as F\n!pip install timm\nfrom timm.models.layers import trunc_normal_, DropPath\nfrom timm.loss import LabelSmoothingCrossEntropy\nfrom sklearn.model_selection import StratifiedKFold","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-16T11:18:42.354689Z","iopub.execute_input":"2022-02-16T11:18:42.355581Z","iopub.status.idle":"2022-02-16T11:19:02.590903Z","shell.execute_reply.started":"2022-02-16T11:18:42.355460Z","shell.execute_reply":"2022-02-16T11:19:02.590116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\n\nuser_secrets = UserSecretsClient()\n\n# I have saved my API token with \"wandb_api\" as Label. \n# If you use some other Label make sure to change the same below. \nwandb_api = user_secrets.get_secret(\"wandb_api\") \n\nwandb.login(key=wandb_api)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T11:19:02.593037Z","iopub.execute_input":"2022-02-16T11:19:02.593302Z","iopub.status.idle":"2022-02-16T11:19:03.910192Z","shell.execute_reply.started":"2022-02-16T11:19:02.593265Z","shell.execute_reply":"2022-02-16T11:19:03.909476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    SEED = 0\n    size = 224\n    batch_size = 128#64\n    \n    learning_rate = 2e-4\n    epochs = 20\n    \n    num_workers = 2\n    FOLDS = 4","metadata":{"execution":{"iopub.status.busy":"2022-02-16T11:19:03.911268Z","iopub.execute_input":"2022-02-16T11:19:03.911499Z","iopub.status.idle":"2022-02-16T11:19:03.915401Z","shell.execute_reply.started":"2022-02-16T11:19:03.911464Z","shell.execute_reply":"2022-02-16T11:19:03.914775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fix_seed(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    #gpu randomseed fixed\n    torch.backends.cudnn.deterministic = True\n\nfix_seed(CFG.SEED)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T11:19:03.917718Z","iopub.execute_input":"2022-02-16T11:19:03.918271Z","iopub.status.idle":"2022-02-16T11:19:03.930708Z","shell.execute_reply.started":"2022-02-16T11:19:03.918228Z","shell.execute_reply":"2022-02-16T11:19:03.929865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n\n    Data Loaders\n    \n\"\"\"\n\nclass WandDLoader(Dataset):\n    def __init__(self, data, train):\n        self.images = data[\"image\"].values.tolist()\n        self.labels = data[\"species_id\"].values.tolist()\n        if train:\n            transformations = albumentations.Compose([\n                albumentations.HorizontalFlip(p=0.5),\n                albumentations.Cutout(num_holes=1, max_h_size=2*(CFG.size//18), max_w_size=2*(CFG.size//18), p=0.8),\n                albumentations.Normalize(),\n                ToTensorV2(p=1.0)\n            ])\n        else:\n            transformations = albumentations.Compose([\n                albumentations.Normalize(),\n                ToTensorV2(p=1.0)\n            ])\n            #path = \"../input/happy-whale-and-dolphin/test_images\"\n\n        def albumentations_transform(image, transform=transformations):\n            if transform:\n                image_np = np.array(image)\n                augmented = transform(image=image_np)\n            return augmented\n\n        self.transforms = transforms.Compose([\n            transforms.Lambda(albumentations_transform),\n        ])\n        #path = \"../input/happy-whale-and-dolphin/train_images\"\n        path = \"../input/w-d-224x224-fast-dataset/train_images\"\n        self.base_path = path\n                        \n    def __getitem__(self, idx):\n        image_name = self.images[idx]\n        path = os.path.join(self.base_path, image_name)\n        img = cv2.imread(path)\n        if self.transforms is not None:\n            img = self.transforms(img)[\"image\"]\n        return img, torch.tensor(self.labels[idx], dtype=torch.long)\n    \n    def __len__(self):\n        return len(self.images)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T11:19:03.933010Z","iopub.execute_input":"2022-02-16T11:19:03.933705Z","iopub.status.idle":"2022-02-16T11:19:03.945918Z","shell.execute_reply.started":"2022-02-16T11:19:03.933664Z","shell.execute_reply":"2022-02-16T11:19:03.945225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_csv(\"../input/w-d-224x224-fast-dataset/train.csv\")\ndata[\"species\"][data[\"species\"] == \"bottlenose_dolpin\"] = \"bottlenose_dolphin\"\ndata[\"species\"][data[\"species\"] == \"kiler_whale\"] = \"killer_whale\"\ndata[\"species\"][data[\"species\"] == \"globis\"] = \"short_finned_pilot_whale\"\ndata[\"species\"][data[\"species\"] == \"pilot_whale\"] = \"short_finned_pilot_whale\"\nN_CLASSES = len(data[\"species\"].unique())\nclass_map = {k:i for i, k in enumerate(data[\"species\"].unique())}\ndata[\"species_id\"] = data[\"species\"].map(class_map)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T11:19:03.947389Z","iopub.execute_input":"2022-02-16T11:19:03.947980Z","iopub.status.idle":"2022-02-16T11:19:04.092776Z","shell.execute_reply.started":"2022-02-16T11:19:03.947942Z","shell.execute_reply":"2022-02-16T11:19:04.092093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n    https://pytorch.org/vision/main/_modules/torchvision/models/efficientnet.html\n\"\"\"\n\nfrom typing import Any, Callable, Optional, List, Sequence\nfrom functools import partial\ntry:\n    from torch.hub import load_state_dict_from_url  # noqa: 401\nexcept ImportError:\n    from torch.utils.model_zoo import load_url as load_state_dict_from_url  # noqa: 401\n\ndef stochastic_depth(input: Tensor, p: float, mode: str, training: bool = True) -> Tensor:\n    \"\"\"\n    Implements the Stochastic Depth from `\"Deep Networks with Stochastic Depth\"\n    <https://arxiv.org/abs/1603.09382>`_ used for randomly dropping residual\n    branches of residual architectures.\n    Args:\n        input (Tensor[N, ...]): The input tensor or arbitrary dimensions with the first one\n                    being its batch i.e. a batch with ``N`` rows.\n        p (float): probability of the input to be zeroed.\n        mode (str): ``\"batch\"`` or ``\"row\"``.\n                    ``\"batch\"`` randomly zeroes the entire input, ``\"row\"`` zeroes\n                    randomly selected rows from the batch.\n        training: apply stochastic depth if is ``True``. Default: ``True``\n    Returns:\n        Tensor[N, ...]: The randomly zeroed tensor.\n    \"\"\"\n    if p < 0.0 or p > 1.0:\n        raise ValueError(f\"drop probability has to be between 0 and 1, but got {p}\")\n    if mode not in [\"batch\", \"row\"]:\n        raise ValueError(f\"mode has to be either 'batch' or 'row', but got {mode}\")\n    if not training or p == 0.0:\n        return input\n\n    survival_rate = 1.0 - p\n    if mode == \"row\":\n        size = [input.shape[0]] + [1] * (input.ndim - 1)\n    else:\n        size = [1] * input.ndim\n    noise = torch.empty(size, dtype=input.dtype, device=input.device)\n    noise = noise.bernoulli_(survival_rate)\n    if survival_rate > 0.0:\n        noise.div_(survival_rate)\n    return input * noise\n\nclass StochasticDepth(nn.Module):\n    \"\"\"\n    See :func:`stochastic_depth`.\n    \"\"\"\n\n    def __init__(self, p: float, mode: str) -> None:\n        super().__init__()\n        self.p = p\n        self.mode = mode\n\n    def forward(self, input: Tensor) -> Tensor:\n        return stochastic_depth(input, self.p, self.mode, self.training)\n\n    def __repr__(self) -> str:\n        tmpstr = self.__class__.__name__ + \"(\"\n        tmpstr += \"p=\" + str(self.p)\n        tmpstr += \", mode=\" + str(self.mode)\n        tmpstr += \")\"\n        return tmpstr\n\ndef _make_divisible(v: float, divisor: int, min_value: Optional[int] = None) -> int:\n    \"\"\"\n    This function is taken from the original tf repo.\n    It ensures that all layers have a channel number that is divisible by 8\n    It can be seen here:\n    https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py\n    \"\"\"\n    if min_value is None:\n        min_value = divisor\n    new_v = max(min_value, int(v + divisor / 2) // divisor * divisor)\n    # Make sure that round down does not go down by more than 10%.\n    if new_v < 0.9 * v:\n        new_v += divisor\n    return new_v\n\nclass ConvNormActivation(torch.nn.Sequential):\n    \"\"\"\n    Configurable block used for Convolution-Normalzation-Activation blocks.\n\n    Args:\n        in_channels (int): Number of channels in the input image\n        out_channels (int): Number of channels produced by the Convolution-Normalzation-Activation block\n        kernel_size: (int, optional): Size of the convolving kernel. Default: 3\n        stride (int, optional): Stride of the convolution. Default: 1\n        padding (int, tuple or str, optional): Padding added to all four sides of the input. Default: None, in wich case it will calculated as ``padding = (kernel_size - 1) // 2 * dilation``\n        groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1\n        norm_layer (Callable[..., torch.nn.Module], optional): Norm layer that will be stacked on top of the convolutiuon layer. If ``None`` this layer wont be used. Default: ``torch.nn.BatchNorm2d``\n        activation_layer (Callable[..., torch.nn.Module], optinal): Activation function which will be stacked on top of the normalization layer (if not None), otherwise on top of the conv layer. If ``None`` this layer wont be used. Default: ``torch.nn.ReLU``\n        dilation (int): Spacing between kernel elements. Default: 1\n        inplace (bool): Parameter for the activation layer, which can optionally do the operation in-place. Default ``True``\n        bias (bool, optional): Whether to use bias in the convolution layer. By default, biases are included if ``norm_layer is None``.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        in_channels: int,\n        out_channels: int,\n        kernel_size: int = 3,\n        stride: int = 1,\n        padding: Optional[int] = None,\n        groups: int = 1,\n        norm_layer: Optional[Callable[..., torch.nn.Module]] = torch.nn.BatchNorm2d,\n        activation_layer: Optional[Callable[..., torch.nn.Module]] = torch.nn.ReLU,\n        dilation: int = 1,\n        inplace: Optional[bool] = True,\n        bias: Optional[bool] = None,\n    ) -> None:\n        if padding is None:\n            padding = (kernel_size - 1) // 2 * dilation\n        if bias is None:\n            bias = norm_layer is None\n        layers = [\n            torch.nn.Conv2d(\n                in_channels,\n                out_channels,\n                kernel_size,\n                stride,\n                padding,\n                dilation=dilation,\n                groups=groups,\n                bias=bias,\n            )\n        ]\n        if norm_layer is not None:\n            layers.append(norm_layer(out_channels))\n        if activation_layer is not None:\n            params = {} if inplace is None else {\"inplace\": inplace}\n            layers.append(activation_layer(**params))\n        super().__init__(*layers)\n        self.out_channels = out_channels\n\nclass SqueezeExcitation(torch.nn.Module):\n    \"\"\"\n    This block implements the Squeeze-and-Excitation block from https://arxiv.org/abs/1709.01507 (see Fig. 1).\n    Parameters ``activation``, and ``scale_activation`` correspond to ``delta`` and ``sigma`` in in eq. 3.\n\n    Args:\n        input_channels (int): Number of channels in the input image\n        squeeze_channels (int): Number of squeeze channels\n        activation (Callable[..., torch.nn.Module], optional): ``delta`` activation. Default: ``torch.nn.ReLU``\n        scale_activation (Callable[..., torch.nn.Module]): ``sigma`` activation. Default: ``torch.nn.Sigmoid``\n    \"\"\"\n\n    def __init__(\n        self,\n        input_channels: int,\n        squeeze_channels: int,\n        activation: Callable[..., torch.nn.Module] = torch.nn.ReLU,\n        scale_activation: Callable[..., torch.nn.Module] = torch.nn.Sigmoid,\n    ) -> None:\n        super().__init__()\n        self.avgpool = torch.nn.AdaptiveAvgPool2d(1)\n        self.fc1 = torch.nn.Conv2d(input_channels, squeeze_channels, 1)\n        self.fc2 = torch.nn.Conv2d(squeeze_channels, input_channels, 1)\n        self.activation = activation()\n        self.scale_activation = scale_activation()\n\n    def _scale(self, input: Tensor) -> Tensor:\n        scale = self.avgpool(input)\n        scale = self.fc1(scale)\n        scale = self.activation(scale)\n        scale = self.fc2(scale)\n        return self.scale_activation(scale)\n\n    def forward(self, input: Tensor) -> Tensor:\n        scale = self._scale(input)\n        return scale * input\n\n\nmodel_urls = {\n    # Weights ported from https://github.com/rwightman/pytorch-image-models/\n    \"efficientnet_b0\": \"https://download.pytorch.org/models/efficientnet_b0_rwightman-3dd342df.pth\",\n    \"efficientnet_b1\": \"https://download.pytorch.org/models/efficientnet_b1_rwightman-533bc792.pth\",\n    \"efficientnet_b2\": \"https://download.pytorch.org/models/efficientnet_b2_rwightman-bcdf34b7.pth\",\n    \"efficientnet_b3\": \"https://download.pytorch.org/models/efficientnet_b3_rwightman-cf984f9c.pth\",\n    \"efficientnet_b4\": \"https://download.pytorch.org/models/efficientnet_b4_rwightman-7eb33cd5.pth\",\n    # Weights ported from https://github.com/lukemelas/EfficientNet-PyTorch/\n    \"efficientnet_b5\": \"https://download.pytorch.org/models/efficientnet_b5_lukemelas-b6417697.pth\",\n    \"efficientnet_b6\": \"https://download.pytorch.org/models/efficientnet_b6_lukemelas-c76e70fd.pth\",\n    \"efficientnet_b7\": \"https://download.pytorch.org/models/efficientnet_b7_lukemelas-dcc49843.pth\",\n}\n\n\nclass MBConvConfig:\n    # Stores information listed at Table 1 of the EfficientNet paper\n    def __init__(\n        self,\n        expand_ratio: float,\n        kernel: int,\n        stride: int,\n        input_channels: int,\n        out_channels: int,\n        num_layers: int,\n        width_mult: float,\n        depth_mult: float,\n    ) -> None:\n        self.expand_ratio = expand_ratio\n        self.kernel = kernel\n        self.stride = stride\n        self.input_channels = self.adjust_channels(input_channels, width_mult)\n        self.out_channels = self.adjust_channels(out_channels, width_mult)\n        self.num_layers = self.adjust_depth(num_layers, depth_mult)\n\n    def __repr__(self) -> str:\n        s = self.__class__.__name__ + \"(\"\n        s += \"expand_ratio={expand_ratio}\"\n        s += \", kernel={kernel}\"\n        s += \", stride={stride}\"\n        s += \", input_channels={input_channels}\"\n        s += \", out_channels={out_channels}\"\n        s += \", num_layers={num_layers}\"\n        s += \")\"\n        return s.format(**self.__dict__)\n\n    @staticmethod\n    def adjust_channels(channels: int, width_mult: float, min_value: Optional[int] = None) -> int:\n        return _make_divisible(channels * width_mult, 8, min_value)\n\n    @staticmethod\n    def adjust_depth(num_layers: int, depth_mult: float):\n        return int(math.ceil(num_layers * depth_mult))\n\n\nclass MBConv(nn.Module):\n    def __init__(\n        self,\n        cnf: MBConvConfig,\n        stochastic_depth_prob: float,\n        norm_layer: Callable[..., nn.Module],\n        se_layer: Callable[..., nn.Module] = SqueezeExcitation,\n    ) -> None:\n        super().__init__()\n\n        if not (1 <= cnf.stride <= 2):\n            raise ValueError(\"illegal stride value\")\n\n        self.use_res_connect = cnf.stride == 1 and cnf.input_channels == cnf.out_channels\n\n        layers: List[nn.Module] = []\n        activation_layer = nn.SiLU\n\n        # expand\n        expanded_channels = cnf.adjust_channels(cnf.input_channels, cnf.expand_ratio)\n        if expanded_channels != cnf.input_channels:\n            layers.append(\n                ConvNormActivation(\n                    cnf.input_channels,\n                    expanded_channels,\n                    kernel_size=1,\n                    norm_layer=norm_layer,\n                    activation_layer=activation_layer,\n                )\n            )\n\n        # depthwise\n        layers.append(\n            ConvNormActivation(\n                expanded_channels,\n                expanded_channels,\n                kernel_size=cnf.kernel,\n                stride=cnf.stride,\n                groups=expanded_channels,\n                norm_layer=norm_layer,\n                activation_layer=activation_layer,\n            )\n        )\n\n        # squeeze and excitation\n        squeeze_channels = max(1, cnf.input_channels // 4)\n        layers.append(se_layer(expanded_channels, squeeze_channels, activation=partial(nn.SiLU, inplace=True)))\n\n        # project\n        layers.append(\n            ConvNormActivation(\n                expanded_channels, cnf.out_channels, kernel_size=1, norm_layer=norm_layer, activation_layer=None\n            )\n        )\n\n        self.block = nn.Sequential(*layers)\n        self.stochastic_depth = StochasticDepth(stochastic_depth_prob, \"row\")\n        self.out_channels = cnf.out_channels\n\n    def forward(self, input: Tensor) -> Tensor:\n        result = self.block(input)\n        if self.use_res_connect:\n            result = self.stochastic_depth(result)\n            result += input\n        return result\n\n\nclass EfficientNet(nn.Module):\n    def __init__(\n        self,\n        inverted_residual_setting: List[MBConvConfig],\n        dropout: float,\n        stochastic_depth_prob: float = 0.2,\n        num_classes: int = 1000,\n        block: Optional[Callable[..., nn.Module]] = None,\n        norm_layer: Optional[Callable[..., nn.Module]] = None,\n        **kwargs: Any,\n    ) -> None:\n        \"\"\"\n        EfficientNet main class\n\n        Args:\n            inverted_residual_setting (List[MBConvConfig]): Network structure\n            dropout (float): The droupout probability\n            stochastic_depth_prob (float): The stochastic depth probability\n            num_classes (int): Number of classes\n            block (Optional[Callable[..., nn.Module]]): Module specifying inverted residual building block for mobilenet\n            norm_layer (Optional[Callable[..., nn.Module]]): Module specifying the normalization layer to use\n        \"\"\"\n        super().__init__()\n\n        if not inverted_residual_setting:\n            raise ValueError(\"The inverted_residual_setting should not be empty\")\n        elif not (\n            isinstance(inverted_residual_setting, Sequence)\n            and all([isinstance(s, MBConvConfig) for s in inverted_residual_setting])\n        ):\n            raise TypeError(\"The inverted_residual_setting should be List[MBConvConfig]\")\n\n        if block is None:\n            block = MBConv\n\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n\n        layers: List[nn.Module] = []\n\n        # building first layer\n        firstconv_output_channels = inverted_residual_setting[0].input_channels\n        layers.append(\n            ConvNormActivation(\n                3, firstconv_output_channels, kernel_size=3, stride=2, norm_layer=norm_layer, activation_layer=nn.SiLU\n            )\n        )\n\n        # building inverted residual blocks\n        total_stage_blocks = sum(cnf.num_layers for cnf in inverted_residual_setting)\n        stage_block_id = 0\n        for cnf in inverted_residual_setting:\n            stage: List[nn.Module] = []\n            for _ in range(cnf.num_layers):\n                # copy to avoid modifications. shallow copy is enough\n                block_cnf = copy.copy(cnf)\n\n                # overwrite info if not the first conv in the stage\n                if stage:\n                    block_cnf.input_channels = block_cnf.out_channels\n                    block_cnf.stride = 1\n\n                # adjust stochastic depth probability based on the depth of the stage block\n                sd_prob = stochastic_depth_prob * float(stage_block_id) / total_stage_blocks\n\n                stage.append(block(block_cnf, sd_prob, norm_layer))\n                stage_block_id += 1\n\n            layers.append(nn.Sequential(*stage))\n\n        # building last several layers\n        lastconv_input_channels = inverted_residual_setting[-1].out_channels\n        lastconv_output_channels = 4 * lastconv_input_channels\n        layers.append(\n            ConvNormActivation(\n                lastconv_input_channels,\n                lastconv_output_channels,\n                kernel_size=1,\n                norm_layer=norm_layer,\n                activation_layer=nn.SiLU,\n            )\n        )\n\n        self.features = nn.Sequential(*layers)\n        self.avgpool = nn.AdaptiveAvgPool2d(1)\n        self.classifier = nn.Sequential(\n            nn.Dropout(p=dropout, inplace=True),\n            nn.Linear(lastconv_output_channels, num_classes),\n        )\n\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode=\"fan_out\")\n                if m.bias is not None:\n                    nn.init.zeros_(m.bias)\n            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):\n                nn.init.ones_(m.weight)\n                nn.init.zeros_(m.bias)\n            elif isinstance(m, nn.Linear):\n                init_range = 1.0 / math.sqrt(m.out_features)\n                nn.init.uniform_(m.weight, -init_range, init_range)\n                nn.init.zeros_(m.bias)\n\n    def _forward_impl(self, x: Tensor) -> Tensor:\n        channels = self.features(x)\n\n        x = self.avgpool(channels)\n        features = torch.flatten(x, 1)\n\n        x = self.classifier(features)\n\n        return x, features\n\n    def forward(self, x: Tensor) -> Tensor:\n        return self._forward_impl(x)\n    \n    \ndef _efficientnet(\n    arch: str,\n    width_mult: float,\n    depth_mult: float,\n    dropout: float,\n    pretrained: bool,\n    progress: bool,\n    **kwargs: Any,\n) -> EfficientNet:\n    bneck_conf = partial(MBConvConfig, width_mult=width_mult, depth_mult=depth_mult)\n    inverted_residual_setting = [\n        bneck_conf(1, 3, 1, 32, 16, 1),\n        bneck_conf(6, 3, 2, 16, 24, 2),\n        bneck_conf(6, 5, 2, 24, 40, 2),\n        bneck_conf(6, 3, 2, 40, 80, 3),\n        bneck_conf(6, 5, 1, 80, 112, 3),\n        bneck_conf(6, 5, 2, 112, 192, 4),\n        bneck_conf(6, 3, 1, 192, 320, 1),\n    ]\n    model = EfficientNet(inverted_residual_setting, dropout, **kwargs)\n    if pretrained:\n        if model_urls.get(arch, None) is None:\n            raise ValueError(f\"No checkpoint is available for model type {arch}\")\n        state_dict = load_state_dict_from_url(model_urls[arch], progress=progress)\n        model.load_state_dict(state_dict)\n    return model\n\ndef efficientnet_b0(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> EfficientNet:\n    \"\"\"\n    Constructs a EfficientNet B0 architecture from\n    `\"EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks\" <https://arxiv.org/abs/1905.11946>`_.\n\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    return _efficientnet(\"efficientnet_b0\", 1.0, 1.0, 0.2, pretrained, progress, **kwargs)\n\n\ndef efficientnet_b1(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> EfficientNet:\n    \"\"\"\n    Constructs a EfficientNet B1 architecture from\n    `\"EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks\" <https://arxiv.org/abs/1905.11946>`_.\n\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    return _efficientnet(\"efficientnet_b1\", 1.0, 1.1, 0.2, pretrained, progress, **kwargs)\n\n\ndef efficientnet_b2(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> EfficientNet:\n    \"\"\"\n    Constructs a EfficientNet B2 architecture from\n    `\"EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks\" <https://arxiv.org/abs/1905.11946>`_.\n\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    return _efficientnet(\"efficientnet_b2\", 1.1, 1.2, 0.3, pretrained, progress, **kwargs)\n\n\ndef efficientnet_b3(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> EfficientNet:\n    \"\"\"\n    Constructs a EfficientNet B3 architecture from\n    `\"EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks\" <https://arxiv.org/abs/1905.11946>`_.\n\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    return _efficientnet(\"efficientnet_b3\", 1.2, 1.4, 0.3, pretrained, progress, **kwargs)\n\n\ndef efficientnet_b4(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> EfficientNet:\n    \"\"\"\n    Constructs a EfficientNet B4 architecture from\n    `\"EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks\" <https://arxiv.org/abs/1905.11946>`_.\n\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    return _efficientnet(\"efficientnet_b4\", 1.4, 1.8, 0.4, pretrained, progress, **kwargs)\n\n\ndef efficientnet_b5(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> EfficientNet:\n    \"\"\"\n    Constructs a EfficientNet B5 architecture from\n    `\"EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks\" <https://arxiv.org/abs/1905.11946>`_.\n\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    return _efficientnet(\n        \"efficientnet_b5\",\n        1.6,\n        2.2,\n        0.4,\n        pretrained,\n        progress,\n        norm_layer=partial(nn.BatchNorm2d, eps=0.001, momentum=0.01),\n        **kwargs,\n    )\n\n\n\ndef efficientnet_b6(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> EfficientNet:\n    \"\"\"\n    Constructs a EfficientNet B6 architecture from\n    `\"EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks\" <https://arxiv.org/abs/1905.11946>`_.\n\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    return _efficientnet(\n        \"efficientnet_b6\",\n        1.8,\n        2.6,\n        0.5,\n        pretrained,\n        progress,\n        norm_layer=partial(nn.BatchNorm2d, eps=0.001, momentum=0.01),\n        **kwargs,\n    )\n\n\n\ndef efficientnet_b7(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> EfficientNet:\n    \"\"\"\n    Constructs a EfficientNet B7 architecture from\n    `\"EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks\" <https://arxiv.org/abs/1905.11946>`_.\n\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    return _efficientnet(\n        \"efficientnet_b7\",\n        2.0,\n        3.1,\n        0.5,\n        pretrained,\n        progress,\n        norm_layer=partial(nn.BatchNorm2d, eps=0.001, momentum=0.01),\n        **kwargs,\n    )","metadata":{"execution":{"iopub.status.busy":"2022-02-16T11:19:04.094503Z","iopub.execute_input":"2022-02-16T11:19:04.094778Z","iopub.status.idle":"2022-02-16T11:19:04.172404Z","shell.execute_reply.started":"2022-02-16T11:19:04.094742Z","shell.execute_reply":"2022-02-16T11:19:04.171792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LayerNorm(nn.Module):\n    r\"\"\" LayerNorm that supports two data formats: channels_last (default) or channels_first. \n    The ordering of the dimensions in the inputs. channels_last corresponds to inputs with \n    shape (batch_size, height, width, channels) while channels_first corresponds to inputs \n    with shape (batch_size, channels, height, width).\n    \"\"\"\n    def __init__(self, normalized_shape, eps=1e-6, data_format=\"channels_last\"):\n        super().__init__()\n        self.weight = nn.Parameter(torch.ones(normalized_shape))\n        self.bias = nn.Parameter(torch.zeros(normalized_shape))\n        self.eps = eps\n        self.data_format = data_format\n        if self.data_format not in [\"channels_last\", \"channels_first\"]:\n            raise NotImplementedError \n        self.normalized_shape = (normalized_shape, )\n    \n    def forward(self, x):\n        if self.data_format == \"channels_last\":\n            return F.layer_norm(x, self.normalized_shape, self.weight, self.bias, self.eps)\n        elif self.data_format == \"channels_first\":\n            u = x.mean(1, keepdim=True)\n            s = (x - u).pow(2).mean(1, keepdim=True)\n            x = (x - u) / torch.sqrt(s + self.eps)\n            x = self.weight[:, None, None] * x + self.bias[:, None, None]\n            return x\n\nclass Block(nn.Module):\n    r\"\"\" ConvNeXt Block. There are two equivalent implementations:\n    (1) DwConv -> LayerNorm (channels_first) -> 1x1 Conv -> GELU -> 1x1 Conv; all in (N, C, H, W)\n    (2) DwConv -> Permute to (N, H, W, C); LayerNorm (channels_last) -> Linear -> GELU -> Linear; Permute back\n    We use (2) as we find it slightly faster in PyTorch\n    \n    Args:\n        dim (int): Number of input channels.\n        drop_path (float): Stochastic depth rate. Default: 0.0\n        layer_scale_init_value (float): Init value for Layer Scale. Default: 1e-6.\n    \"\"\"\n    def __init__(self, dim, drop_path=0., layer_scale_init_value=1e-6):\n        super().__init__()\n        self.dwconv = nn.Conv2d(dim, dim, kernel_size=7, padding=3, groups=dim) # depthwise conv\n        self.norm = LayerNorm(dim, eps=1e-6)\n        self.pwconv1 = nn.Linear(dim, 4 * dim) # pointwise/1x1 convs, implemented with linear layers\n        self.act = nn.GELU()\n        self.pwconv2 = nn.Linear(4 * dim, dim)\n        self.gamma = nn.Parameter(layer_scale_init_value * torch.ones((dim)), \n                                    requires_grad=True) if layer_scale_init_value > 0 else None\n        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()\n\n    def forward(self, x):\n        input = x\n        x = self.dwconv(x)\n        x = x.permute(0, 2, 3, 1) # (N, C, H, W) -> (N, H, W, C)\n        x = self.norm(x)\n        x = self.pwconv1(x)\n        x = self.act(x)\n        x = self.pwconv2(x)\n        if self.gamma is not None:\n            x = self.gamma * x\n        x = x.permute(0, 3, 1, 2) # (N, H, W, C) -> (N, C, H, W)\n\n        x = input + self.drop_path(x)\n        return x\n\nclass ConvNeXt(nn.Module):\n    r\"\"\" ConvNeXt\n        A PyTorch impl of : `A ConvNet for the 2020s`  -\n          https://arxiv.org/pdf/2201.03545.pdf\n    Args:\n        in_chans (int): Number of input image channels. Default: 3\n        num_classes (int): Number of classes for classification head. Default: 1000\n        depths (tuple(int)): Number of blocks at each stage. Default: [3, 3, 9, 3]\n        dims (int): Feature dimension at each stage. Default: [96, 192, 384, 768]\n        drop_path_rate (float): Stochastic depth rate. Default: 0.\n        layer_scale_init_value (float): Init value for Layer Scale. Default: 1e-6.\n        head_init_scale (float): Init scaling value for classifier weights and biases. Default: 1.\n    \"\"\"\n    def __init__(self, in_chans=3, num_classes=1000, \n                 depths=[3, 3, 9, 3], dims=[96, 192, 384, 768], drop_path_rate=0., \n                 layer_scale_init_value=1e-6, head_init_scale=1.,\n                 ):\n        super().__init__()\n\n        self.downsample_layers = nn.ModuleList() # stem and 3 intermediate downsampling conv layers\n        stem = nn.Sequential(\n            nn.Conv2d(in_chans, dims[0], kernel_size=4, stride=4),\n            LayerNorm(dims[0], eps=1e-6, data_format=\"channels_first\")\n        )\n        self.downsample_layers.append(stem)\n        for i in range(3):\n            downsample_layer = nn.Sequential(\n                    LayerNorm(dims[i], eps=1e-6, data_format=\"channels_first\"),\n                    nn.Conv2d(dims[i], dims[i+1], kernel_size=2, stride=2),\n            )\n            self.downsample_layers.append(downsample_layer)\n\n        self.stages = nn.ModuleList() # 4 feature resolution stages, each consisting of multiple residual blocks\n        dp_rates=[x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))] \n        cur = 0\n        for i in range(4):\n            stage = nn.Sequential(\n                *[Block(dim=dims[i], drop_path=dp_rates[cur + j], \n                layer_scale_init_value=layer_scale_init_value) for j in range(depths[i])]\n            )\n            self.stages.append(stage)\n            cur += depths[i]\n\n        self.norm = nn.LayerNorm(dims[-1], eps=1e-6) # final norm layer\n        self.head = nn.Linear(dims[-1], num_classes)\n\n        self.apply(self._init_weights)\n        self.head.weight.data.mul_(head_init_scale)\n        self.head.bias.data.mul_(head_init_scale)\n\n    def _init_weights(self, m):\n        if isinstance(m, (nn.Conv2d, nn.Linear)):\n            trunc_normal_(m.weight, std=.02)\n            nn.init.constant_(m.bias, 0)\n\n    def forward_features(self, x):\n        for i in range(4):\n            x = self.downsample_layers[i](x)\n            x = self.stages[i](x)\n        return self.norm(x.mean([-2, -1])) # global average pooling, (N, C, H, W) -> (N, C)\n\n    def forward(self, x):\n        x = self.forward_features(x)\n        x = self.head(x)\n        return x\n\n\nclass WDModel(LightningModule):\n    def __init__(self, weights):\n        super().__init__()\n        #self.save_hyperparameters()\n        \"\"\"\n        self.model = ConvNeXt(\n            depths=[3, 3, 9, 3],\n            dims=[96, 192, 384, 768],\n            in_chans=3,\n            num_classes=N_CLASSES, \n            drop_path_rate=0., \n            layer_scale_init_value=1e-6,\n            head_init_scale=1.\n        )\n        \"\"\"\n        self.model = efficientnet_b0(True)\n        self.classification_head = nn.Linear(1280, N_CLASSES)\n        #self.criterion = LabelSmoothingCrossEntropy(0.0)\n        self.criterion = nn.CrossEntropyLoss(weight=torch.tensor(weights))\n        \n        acc = torchmetrics.Accuracy()\n        # use .clone so that each metric can maintain its own state\n        self.train_acc = acc.clone()\n        # assign all metrics as attributes of module so they are detected as children\n        self.val_acc = acc.clone()\n        \n        self.validation_outputs = []\n    \n    def forward(self, x):\n        _, x = self.model(x)\n        x = self.classification_head(x)\n        return x\n    \n    def training_step(self, batch, batch_idx):\n        x, y = batch\n        logits = self(x)\n        loss = self.criterion(logits, y)\n        self.train_acc(logits, y)\n        self.log(\"train/loss\", loss)\n        self.log(\"train/acc\", self.train_acc)\n        return {\"loss\": loss, \"preds\": logits.detach(), \"targets\": y.detach()}\n    \n    def validation_step(self, batch, batch_idx):\n        x, y = batch\n        logits = self(x)\n        loss = self.criterion(logits, y)\n        self.val_acc(logits, y)\n        self.log(\"val/loss\", loss)\n        self.log(\"val/acc\", self.val_acc)\n        return {\"val_loss\": loss, \"preds\": logits.detach(), \"targets\": y.detach()}\n    \n    def validation_epoch_end(self, outputs):\n        outputs = list(map(lambda x: x[\"preds\"], outputs))\n        outputs = torch.cat(outputs)\n        self.validation_outputs.append(outputs)\n\n    def configure_optimizers(self):\n        opt = torch.optim.AdamW(self.parameters(), lr=CFG.learning_rate)\n        return {\n            \"optimizer\": opt,\n            \"lr_scheduler\": {\n                \"scheduler\": torch.optim.lr_scheduler.ReduceLROnPlateau(opt, patience=4),\n                \"monitor\": \"val/loss\",\n            }\n        }","metadata":{"execution":{"iopub.status.busy":"2022-02-16T11:19:04.173789Z","iopub.execute_input":"2022-02-16T11:19:04.174024Z","iopub.status.idle":"2022-02-16T11:19:04.211333Z","shell.execute_reply.started":"2022-02-16T11:19:04.173992Z","shell.execute_reply":"2022-02-16T11:19:04.210578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=CFG.FOLDS)\nfolds = np.zeros(len(data), dtype=np.uint8)\nfor fold, ( _, val_) in enumerate(skf.split(X=data, y=data.species)):\n    folds[val_] = fold\ndata[\"fold\"] = folds\n\nspecies_pred = np.zeros((len(data), N_CLASSES))\nfor i in range(CFG.FOLDS):\n    train = data[data[\"fold\"] != i]\n    val = data[data[\"fold\"] == i]\n\n    train_dataset = WandDLoader(train, True)\n    val_dataset = WandDLoader(val, False)\n\n    train_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        batch_size=CFG.batch_size,\n        num_workers=CFG.num_workers,\n        pin_memory=True,\n        shuffle=True\n    )\n    valid_loader = torch.utils.data.DataLoader(\n        val_dataset,\n        batch_size=CFG.batch_size * 4,\n        num_workers=CFG.num_workers,\n        pin_memory=True,\n        shuffle=False\n    )\n\n    class_counts = train['species'].value_counts()\n    weights = class_counts.max() / class_counts\n    weights.index = weights.index.map(class_map)\n    weights = weights.sort_index().values.tolist()\n\n    model = WDModel(weights)\n    wandb_logger = WandbLogger(project=\"W&D - classification\")\n\n    checkpoint_callback = pl.callbacks.model_checkpoint.ModelCheckpoint(\n        monitor=\"val/loss\",\n        dirpath=\"./\",\n        filename=\"model-wandd-species-fold\" + str(i) + \"-{val/loss:.4f}-{epoch:02d}\",\n        save_top_k=3,\n        mode=\"min\",\n        every_n_epochs=1\n    )\n\n    trainer = Trainer(\n        gpus=1,\n        max_epochs=CFG.epochs,\n        logger=wandb_logger,\n        log_every_n_steps=10,\n        callbacks=[\n            checkpoint_callback\n        ]\n    )\n    \n    trainer.fit(model, train_loader, valid_loader)\n    \n    n = int(checkpoint_callback.best_model_path[-7:-5]) + 1\n    val_preds = model.validation_outputs[n].cpu().numpy()\n    species_pred[data[\"fold\"] == i] = val_preds\n    \n    artifact = wandb.log_artifact(checkpoint_callback.best_model_path, name='w_and_d-species-fold' + str(i), type='model')\n    \n    wandb_logger.finalize(\"success\")\n    wandb.finish()\n    \n    del model, train_dataset, val_dataset, train_loader, valid_loader, trainer, wandb_logger\n\ndata[\"species_pred\"] = species_pred.tolist()\ndata","metadata":{"execution":{"iopub.status.busy":"2022-02-16T11:19:04.212738Z","iopub.execute_input":"2022-02-16T11:19:04.213293Z","iopub.status.idle":"2022-02-16T11:25:04.087551Z","shell.execute_reply.started":"2022-02-16T11:19:04.213256Z","shell.execute_reply":"2022-02-16T11:25:04.086406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.to_csv(\"preds.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T11:25:04.088775Z","iopub.status.idle":"2022-02-16T11:25:04.089488Z","shell.execute_reply.started":"2022-02-16T11:25:04.089243Z","shell.execute_reply":"2022-02-16T11:25:04.089271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_csv(\"preds.csv\")\nnp.mean(data[\"species_pred\"].map(eval).map(np.argmax) == data[\"species_id\"])","metadata":{"execution":{"iopub.status.busy":"2022-02-16T11:25:04.090690Z","iopub.status.idle":"2022-02-16T11:25:04.091443Z","shell.execute_reply.started":"2022-02-16T11:25:04.091201Z","shell.execute_reply":"2022-02-16T11:25:04.091228Z"},"trusted":true},"execution_count":null,"outputs":[]}]}