{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import math\n\nimport cv2\nimport audioread\nimport collections\nimport logging\nimport os\nimport random\nimport re\nimport time\nimport warnings\n\nimport librosa\nimport numpy as np\nimport pandas as pd\nimport soundfile as sf\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data as data\n\nfrom contextlib import contextmanager\nfrom functools import partial\nfrom pathlib import Path\nfrom typing import Optional\n\nfrom fastprogress import progress_bar\nfrom sklearn.metrics import f1_score\nfrom torch.nn import Conv2d, Module, Linear, BatchNorm2d, ReLU\nfrom torch.nn.modules.utils import _pair","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"markdown","source":"## Utilities"},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_seed(seed: int = 42):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)  # type: ignore\n    torch.backends.cudnn.deterministic = True  # type: ignore\n    torch.backends.cudnn.benchmark = True  # type: ignore\n    \n    \ndef get_logger(out_file=None):\n    logger = logging.getLogger()\n    formatter = logging.Formatter(\"%(asctime)s - %(levelname)s - %(message)s\")\n    logger.handlers = []\n    logger.setLevel(logging.INFO)\n\n    handler = logging.StreamHandler()\n    handler.setFormatter(formatter)\n    handler.setLevel(logging.INFO)\n    logger.addHandler(handler)\n\n    if out_file is not None:\n        fh = logging.FileHandler(out_file)\n        fh.setFormatter(formatter)\n        fh.setLevel(logging.INFO)\n        logger.addHandler(fh)\n    logger.info(\"logger set up\")\n    return logger\n    \n    \n@contextmanager\ndef timer(name: str, logger: Optional[logging.Logger] = None):\n    t0 = time.time()\n    msg = f\"[{name}] start\"\n    if logger is None:\n        print(msg)\n    else:\n        logger.info(msg)\n    yield\n\n    msg = f\"[{name}] done in {time.time() - t0:.2f} s\"\n    if logger is None:\n        print(msg)\n    else:\n        logger.info(msg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"logger = get_logger(\"main.log\")\nset_seed(1213)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Loading"},{"metadata":{"trusted":true},"cell_type":"code","source":"TARGET_SR = 32000\nTEST = Path(\"../input/birdsong-recognition/test_audio\").exists()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if TEST:\n    DATA_DIR = Path(\"../input/birdsong-recognition/\")\nelse:\n    # dataset created by @shonenkov, thanks!\n    DATA_DIR = Path(\"../input/birdcall-check/\")\n    \n\ntest = pd.read_csv(DATA_DIR / \"test.csv\")\ntest_audio = DATA_DIR / \"test_audio\"\n\n\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv(\"../input/birdsong-recognition/sample_submission.csv\")\nsub.to_csv(\"submission.csv\", index=False)  # this will be overwritten if everything goes well","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define Model"},{"metadata":{},"cell_type":"markdown","source":"### Building blocks"},{"metadata":{"trusted":true},"cell_type":"code","source":"def init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n\n\ndef interpolate(x: torch.Tensor, ratio: int):\n    \"\"\"Interpolate data in time domain. This is used to compensate the\n    resolution reduction in downsampling of a CNN.\n\n    Args:\n      x: (batch_size, time_steps, classes_num)\n      ratio: int, ratio to interpolate\n    Returns:\n      upsampled: (batch_size, time_steps * ratio, classes_num)\n    \"\"\"\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):\n    \"\"\"Pad framewise_output to the same length as input frames. The pad value\n    is the same as the value of the last frame.\n    Args:\n      framewise_output: (batch_size, frames_num, classes_num)\n      frames_num: int, number of frames to pad\n    Outputs:\n      output: (batch_size, frames_num, classes_num)\n    \"\"\"\n    pad = framewise_output[:, -1:, :].repeat(\n        1, frames_num - framewise_output.shape[1], 1)\n    \"\"\"tensor for padding\"\"\"\n\n    output = torch.cat((framewise_output, pad), dim=1)\n    \"\"\"(batch_size, frames_num, classes_num)\"\"\"\n\n    return output\n\n\nclass ConvBlock(nn.Module):\n    def __init__(self, in_channels: int, out_channels: int):\n        super().__init__()\n\n        self.conv1 = nn.Conv2d(\n            in_channels=in_channels,\n            out_channels=out_channels,\n            kernel_size=(3, 3),\n            stride=(1, 1),\n            padding=(1, 1),\n            bias=False)\n\n        self.conv2 = nn.Conv2d(\n            in_channels=out_channels,\n            out_channels=out_channels,\n            kernel_size=(3, 3),\n            stride=(1, 1),\n            padding=(1, 1),\n            bias=False)\n\n        self.bn1 = nn.BatchNorm2d(out_channels)\n        self.bn2 = nn.BatchNorm2d(out_channels)\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_layer(self.conv1)\n        init_layer(self.conv2)\n        init_bn(self.bn1)\n        init_bn(self.bn2)\n\n    def forward(self, input, pool_size=(2, 2), pool_type='avg'):\n\n        x = input\n        x = F.relu_(self.bn1(self.conv1(x)))\n        x = F.relu_(self.bn2(self.conv2(x)))\n        if pool_type == 'max':\n            x = F.max_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg':\n            x = F.avg_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg+max':\n            x1 = F.avg_pool2d(x, kernel_size=pool_size)\n            x2 = F.max_pool2d(x, kernel_size=pool_size)\n            x = x1 + x2\n        else:\n            raise Exception('Incorrect argument!')\n\n        return x\n\n\nclass AttBlock(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"linear\",\n                 temperature=1.0):\n        super().__init__()\n\n        self.activation = activation\n        self.temperature = temperature\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n\n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.tanh(self.att(x)), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SplAtConv2d(Module):\n    \"\"\"Split-Attention Conv2d\n    \"\"\"\n    def __init__(self, in_channels, channels, kernel_size, stride=(1, 1), padding=(0, 0),\n                 dilation=(1, 1), groups=1, bias=True,\n                 radix=2, reduction_factor=4,\n                 rectify=False, rectify_avg=False, norm_layer=None,\n                 dropblock_prob=0.0, **kwargs):\n        super(SplAtConv2d, self).__init__()\n        padding = _pair(padding)\n        self.rectify = rectify and (padding[0] > 0 or padding[1] > 0)\n        self.rectify_avg = rectify_avg\n        inter_channels = max(in_channels*radix//reduction_factor, 32)\n        self.radix = radix\n        self.cardinality = groups\n        self.channels = channels\n        self.dropblock_prob = dropblock_prob\n        if self.rectify:\n            from rfconv import RFConv2d\n            self.conv = RFConv2d(in_channels, channels*radix, kernel_size, stride, padding, dilation,\n                                 groups=groups*radix, bias=bias, average_mode=rectify_avg, **kwargs)\n        else:\n            self.conv = Conv2d(in_channels, channels*radix, kernel_size, stride, padding, dilation,\n                               groups=groups*radix, bias=bias, **kwargs)\n        self.use_bn = norm_layer is not None\n        if self.use_bn:\n            self.bn0 = norm_layer(channels*radix)\n        self.relu = ReLU(inplace=True)\n        self.fc1 = Conv2d(channels, inter_channels, 1, groups=self.cardinality)\n        if self.use_bn:\n            self.bn1 = norm_layer(inter_channels)\n        self.fc2 = Conv2d(inter_channels, channels*radix, 1, groups=self.cardinality)\n        if dropblock_prob > 0.0:\n            self.dropblock = DropBlock2D(dropblock_prob, 3)\n        self.rsoftmax = rSoftMax(radix, groups)\n\n    def forward(self, x):\n        x = self.conv(x)\n        if self.use_bn:\n            x = self.bn0(x)\n        if self.dropblock_prob > 0.0:\n            x = self.dropblock(x)\n        x = self.relu(x)\n\n        batch, rchannel = x.shape[:2]\n        if self.radix > 1:\n            if torch.__version__ < '1.5':\n                splited = torch.split(x, int(rchannel//self.radix), dim=1)\n            else:\n                splited = torch.split(x, rchannel//self.radix, dim=1)\n            gap = sum(splited) \n        else:\n            gap = x\n        gap = F.adaptive_avg_pool2d(gap, 1)\n        gap = self.fc1(gap)\n\n        if self.use_bn:\n            gap = self.bn1(gap)\n        gap = self.relu(gap)\n\n        atten = self.fc2(gap)\n        atten = self.rsoftmax(atten).view(batch, -1, 1, 1)\n\n        if self.radix > 1:\n            if torch.__version__ < '1.5':\n                attens = torch.split(atten, int(rchannel//self.radix), dim=1)\n            else:\n                attens = torch.split(atten, rchannel//self.radix, dim=1)\n            out = sum([att*split for (att, split) in zip(attens, splited)])\n        else:\n            out = atten * x\n        return out.contiguous()\n\nclass rSoftMax(nn.Module):\n    def __init__(self, radix, cardinality):\n        super().__init__()\n        self.radix = radix\n        self.cardinality = cardinality\n\n    def forward(self, x):\n        batch = x.size(0)\n        if self.radix > 1:\n            x = x.view(batch, self.cardinality, self.radix, -1).transpose(1, 2)\n            x = F.softmax(x, dim=1)\n            x = x.reshape(batch, -1)\n        else:\n            x = torch.sigmoid(x)\n        return x\n    \n    \nclass DropBlock2D(object):\n    def __init__(self, *args, **kwargs):\n        raise NotImplementedError\n\nclass GlobalAvgPool2d(nn.Module):\n    def __init__(self):\n        \"\"\"Global average pooling over the input's spatial dimensions\"\"\"\n        super(GlobalAvgPool2d, self).__init__()\n\n    def forward(self, inputs):\n        return nn.functional.adaptive_avg_pool2d(inputs, 1).view(inputs.size(0), -1)\n\nclass Bottleneck(nn.Module):\n    \"\"\"ResNet Bottleneck\n    \"\"\"\n    # pylint: disable=unused-argument\n    expansion = 4\n    def __init__(self, inplanes, planes, stride=1, downsample=None,\n                 radix=1, cardinality=1, bottleneck_width=64,\n                 avd=False, avd_first=False, dilation=1, is_first=False,\n                 rectified_conv=False, rectify_avg=False,\n                 norm_layer=None, dropblock_prob=0.0, last_gamma=False):\n        super(Bottleneck, self).__init__()\n        group_width = int(planes * (bottleneck_width / 64.)) * cardinality\n        self.conv1 = nn.Conv2d(inplanes, group_width, kernel_size=1, bias=False)\n        self.bn1 = norm_layer(group_width)\n        self.dropblock_prob = dropblock_prob\n        self.radix = radix\n        self.avd = avd and (stride > 1 or is_first)\n        self.avd_first = avd_first\n\n        if self.avd:\n            self.avd_layer = nn.AvgPool2d(3, stride, padding=1)\n            stride = 1\n\n        if dropblock_prob > 0.0:\n            self.dropblock1 = DropBlock2D(dropblock_prob, 3)\n            if radix == 1:\n                self.dropblock2 = DropBlock2D(dropblock_prob, 3)\n            self.dropblock3 = DropBlock2D(dropblock_prob, 3)\n\n        if radix >= 1:\n            self.conv2 = SplAtConv2d(\n                group_width, group_width, kernel_size=3,\n                stride=stride, padding=dilation,\n                dilation=dilation, groups=cardinality, bias=False,\n                radix=radix, rectify=rectified_conv,\n                rectify_avg=rectify_avg,\n                norm_layer=norm_layer,\n                dropblock_prob=dropblock_prob)\n        elif rectified_conv:\n            from rfconv import RFConv2d\n            self.conv2 = RFConv2d(\n                group_width, group_width, kernel_size=3, stride=stride,\n                padding=dilation, dilation=dilation,\n                groups=cardinality, bias=False,\n                average_mode=rectify_avg)\n            self.bn2 = norm_layer(group_width)\n        else:\n            self.conv2 = nn.Conv2d(\n                group_width, group_width, kernel_size=3, stride=stride,\n                padding=dilation, dilation=dilation,\n                groups=cardinality, bias=False)\n            self.bn2 = norm_layer(group_width)\n\n        self.conv3 = nn.Conv2d(\n            group_width, planes * 4, kernel_size=1, bias=False)\n        self.bn3 = norm_layer(planes*4)\n\n        if last_gamma:\n            from torch.nn.init import zeros_\n            zeros_(self.bn3.weight)\n        self.relu = nn.ReLU(inplace=True)\n        self.downsample = downsample\n        self.dilation = dilation\n        self.stride = stride\n\n    def forward(self, x):\n        residual = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        if self.dropblock_prob > 0.0:\n            out = self.dropblock1(out)\n        out = self.relu(out)\n\n        if self.avd and self.avd_first:\n            out = self.avd_layer(out)\n\n        out = self.conv2(out)\n        if self.radix == 0:\n            out = self.bn2(out)\n            if self.dropblock_prob > 0.0:\n                out = self.dropblock2(out)\n            out = self.relu(out)\n\n        if self.avd and not self.avd_first:\n            out = self.avd_layer(out)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n        if self.dropblock_prob > 0.0:\n            out = self.dropblock3(out)\n\n        if self.downsample is not None:\n            residual = self.downsample(x)\n\n        out += residual\n        out = self.relu(out)\n\n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ResNest(nn.Module):\n    \"\"\"ResNet Variants\n    Parameters\n    ----------\n    block : Block\n        Class for the residual block. Options are BasicBlockV1, BottleneckV1.\n    layers : list of int\n        Numbers of layers in each block\n    classes : int, default 1000\n        Number of classification classes.\n    dilated : bool, default False\n        Applying dilation strategy to pretrained ResNet yielding a stride-8 model,\n        typically used in Semantic Segmentation.\n    norm_layer : object\n        Normalization layer used in backbone network (default: :class:`mxnet.gluon.nn.BatchNorm`;\n        for Synchronized Cross-GPU BachNormalization).\n    Reference:\n        - He, Kaiming, et al. \"Deep residual learning for image recognition.\" Proceedings of the IEEE conference on computer vision and pattern recognition. 2016.\n        - Yu, Fisher, and Vladlen Koltun. \"Multi-scale context aggregation by dilated convolutions.\"\n    \"\"\"\n    # pylint: disable=unused-variable\n    def __init__(self, block, layers, radix=1, groups=1, bottleneck_width=64,\n                 num_classes=1000, dilated=False, dilation=1,\n                 deep_stem=False, stem_width=64, avg_down=False,\n                 rectified_conv=False, rectify_avg=False,\n                 avd=False, avd_first=False,\n                 final_drop=0.0, dropblock_prob=0,\n                 last_gamma=False, norm_layer=nn.BatchNorm2d):\n        self.cardinality = groups\n        self.bottleneck_width = bottleneck_width\n        # ResNet-D params\n        self.inplanes = stem_width*2 if deep_stem else 64\n        self.avg_down = avg_down\n        self.last_gamma = last_gamma\n        # ResNeSt params\n        self.radix = radix\n        self.avd = avd\n        self.avd_first = avd_first\n\n        super(ResNest, self).__init__()\n        self.rectified_conv = rectified_conv\n        self.rectify_avg = rectify_avg\n        if rectified_conv:\n            from rfconv import RFConv2d\n            conv_layer = RFConv2d\n        else:\n            conv_layer = nn.Conv2d\n        conv_kwargs = {'average_mode': rectify_avg} if rectified_conv else {}\n        if deep_stem:\n            self.conv1 = nn.Sequential(\n                conv_layer(3, stem_width, kernel_size=3, stride=2, padding=1, bias=False, **conv_kwargs),\n                norm_layer(stem_width),\n                nn.ReLU(inplace=True),\n                conv_layer(stem_width, stem_width, kernel_size=3, stride=1, padding=1, bias=False, **conv_kwargs),\n                norm_layer(stem_width),\n                nn.ReLU(inplace=True),\n                conv_layer(stem_width, stem_width*2, kernel_size=3, stride=1, padding=1, bias=False, **conv_kwargs),\n            )\n        else:\n            self.conv1 = conv_layer(3, 64, kernel_size=7, stride=2, padding=3,\n                                   bias=False, **conv_kwargs)\n        self.bn1 = norm_layer(self.inplanes)\n        self.relu = nn.ReLU(inplace=True)\n        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        self.layer1 = self._make_layer(block, 64, layers[0], norm_layer=norm_layer, is_first=False)\n        self.layer2 = self._make_layer(block, 128, layers[1], stride=2, norm_layer=norm_layer)\n        if dilated or dilation == 4:\n            self.layer3 = self._make_layer(block, 256, layers[2], stride=1,\n                                           dilation=2, norm_layer=norm_layer,\n                                           dropblock_prob=dropblock_prob)\n            self.layer4 = self._make_layer(block, 512, layers[3], stride=1,\n                                           dilation=4, norm_layer=norm_layer,\n                                           dropblock_prob=dropblock_prob)\n        elif dilation==2:\n            self.layer3 = self._make_layer(block, 256, layers[2], stride=2,\n                                           dilation=1, norm_layer=norm_layer,\n                                           dropblock_prob=dropblock_prob)\n            self.layer4 = self._make_layer(block, 512, layers[3], stride=1,\n                                           dilation=2, norm_layer=norm_layer,\n                                           dropblock_prob=dropblock_prob)\n        else:\n            self.layer3 = self._make_layer(block, 256, layers[2], stride=2,\n                                           norm_layer=norm_layer,\n                                           dropblock_prob=dropblock_prob)\n            self.layer4 = self._make_layer(block, 512, layers[3], stride=2,\n                                           norm_layer=norm_layer,\n                                           dropblock_prob=dropblock_prob)\n        self.avgpool = GlobalAvgPool2d()\n        self.drop = nn.Dropout(final_drop) if final_drop > 0.0 else None\n        self.fc = nn.Linear(512 * block.expansion, num_classes)\n\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n                m.weight.data.normal_(0, math.sqrt(2. / n))\n            elif isinstance(m, norm_layer):\n                m.weight.data.fill_(1)\n                m.bias.data.zero_()\n\n    def _make_layer(self, block, planes, blocks, stride=1, dilation=1, norm_layer=None,\n                    dropblock_prob=0.0, is_first=True):\n        downsample = None\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            down_layers = []\n            if self.avg_down:\n                if dilation == 1:\n                    down_layers.append(nn.AvgPool2d(kernel_size=stride, stride=stride,\n                                                    ceil_mode=True, count_include_pad=False))\n                else:\n                    down_layers.append(nn.AvgPool2d(kernel_size=1, stride=1,\n                                                    ceil_mode=True, count_include_pad=False))\n                down_layers.append(nn.Conv2d(self.inplanes, planes * block.expansion,\n                                             kernel_size=1, stride=1, bias=False))\n            else:\n                down_layers.append(nn.Conv2d(self.inplanes, planes * block.expansion,\n                                             kernel_size=1, stride=stride, bias=False))\n            down_layers.append(norm_layer(planes * block.expansion))\n            downsample = nn.Sequential(*down_layers)\n\n        layers = []\n        if dilation == 1 or dilation == 2:\n            layers.append(block(self.inplanes, planes, stride, downsample=downsample,\n                                radix=self.radix, cardinality=self.cardinality,\n                                bottleneck_width=self.bottleneck_width,\n                                avd=self.avd, avd_first=self.avd_first,\n                                dilation=1, is_first=is_first, rectified_conv=self.rectified_conv,\n                                rectify_avg=self.rectify_avg,\n                                norm_layer=norm_layer, dropblock_prob=dropblock_prob,\n                                last_gamma=self.last_gamma))\n        elif dilation == 4:\n            layers.append(block(self.inplanes, planes, stride, downsample=downsample,\n                                radix=self.radix, cardinality=self.cardinality,\n                                bottleneck_width=self.bottleneck_width,\n                                avd=self.avd, avd_first=self.avd_first,\n                                dilation=2, is_first=is_first, rectified_conv=self.rectified_conv,\n                                rectify_avg=self.rectify_avg,\n                                norm_layer=norm_layer, dropblock_prob=dropblock_prob,\n                                last_gamma=self.last_gamma))\n        else:\n            raise RuntimeError(\"=> unknown dilation size: {}\".format(dilation))\n\n        self.inplanes = planes * block.expansion\n        for i in range(1, blocks):\n            layers.append(block(self.inplanes, planes,\n                                radix=self.radix, cardinality=self.cardinality,\n                                bottleneck_width=self.bottleneck_width,\n                                avd=self.avd, avd_first=self.avd_first,\n                                dilation=dilation, rectified_conv=self.rectified_conv,\n                                rectify_avg=self.rectify_avg,\n                                norm_layer=norm_layer, dropblock_prob=dropblock_prob,\n                                last_gamma=self.last_gamma))\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.maxpool(x)\n\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n\n        x = self.avgpool(x)\n        #x = x.view(x.size(0), -1)\n        x = torch.flatten(x, 1)\n        if self.drop:\n            x = self.drop(x)\n        x = self.fc(x)\n\n        multilabel_proba = torch.sigmoid(x)\n        multiclass_proba = torch.softmax(x, dim=1)\n        return {\n            \"logits\": x,\n            \"multilabel_proba\": multilabel_proba,\n            \"multiclass_proba\": multiclass_proba\n        }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# An ordinary implementation of Swish function\nclass Swish(nn.Module):\n    def forward(self, x):\n        return x * torch.sigmoid(x)\n    \n    \n# A memory-efficient implementation of Swish function\nclass SwishImplementation(torch.autograd.Function):\n    @staticmethod\n    def forward(ctx, i):\n        result = i * torch.sigmoid(i)\n        ctx.save_for_backward(i)\n        return result\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        i = ctx.saved_tensors[0]\n        sigmoid_i = torch.sigmoid(i)\n        return grad_output * (sigmoid_i * (1 + i * (1 - sigmoid_i)))\n\n\nclass MemoryEfficientSwish(nn.Module):\n    def forward(self, x):\n        return SwishImplementation.apply(x)\n\n    \ndef round_filters(filters, global_params):\n    \"\"\"Calculate and round number of filters based on width multiplier.\n       Use width_coefficient, depth_divisor and min_depth of global_params.\n    Args:\n        filters (int): Filters number to be calculated.\n        global_params (namedtuple): Global params of the model.\n    Returns:\n        new_filters: New filters number after calculating.\n    \"\"\"\n    multiplier = global_params.width_coefficient\n    if not multiplier:\n        return filters\n    # TODO: modify the params names.\n    #       maybe the names (width_divisor,min_width)\n    #       are more suitable than (depth_divisor,min_depth).\n    divisor = global_params.depth_divisor\n    min_depth = global_params.min_depth\n    filters *= multiplier\n    min_depth = min_depth or divisor # pay attention to this line when using min_depth\n    # follow the formula transferred from official TensorFlow implementation\n    new_filters = max(min_depth, int(filters + divisor / 2) // divisor * divisor)\n    if new_filters < 0.9 * filters: # prevent rounding by more than 10%\n        new_filters += divisor\n    return int(new_filters)\n\n\ndef round_repeats(repeats, global_params):\n    \"\"\"Calculate module's repeat number of a block based on depth multiplier.\n       Use depth_coefficient of global_params.\n    Args:\n        repeats (int): num_repeat to be calculated.\n        global_params (namedtuple): Global params of the model.\n    Returns:\n        new repeat: New repeat number after calculating.\n    \"\"\"\n    multiplier = global_params.depth_coefficient\n    if not multiplier:\n        return repeats\n    # follow the formula transferred from official TensorFlow implementation\n    return int(math.ceil(multiplier * repeats))\n\n\ndef drop_connect(inputs, p, training):\n    \"\"\"Drop connect.\n       \n    Args:\n        input (tensor: BCWH): Input of this structure.\n        p (float: 0.0~1.0): Probability of drop connection.\n        training (bool): The running mode.\n    Returns:\n        output: Output after drop connection.\n    \"\"\"\n    assert p >= 0 and p <= 1, 'p must be in range of [0,1]'\n\n    if not training:\n        return inputs\n\n    batch_size = inputs.shape[0]\n    keep_prob = 1 - p\n\n    # generate binary_tensor mask according to probability (p for 0, 1-p for 1)\n    random_tensor = keep_prob\n    random_tensor += torch.rand([batch_size, 1, 1, 1], dtype=inputs.dtype, device=inputs.device)\n    binary_tensor = torch.floor(random_tensor)\n\n    output = inputs / keep_prob * binary_tensor\n    return output\n\n\ndef get_width_and_height_from_size(x):\n    \"\"\"Obtain height and width from x.\n    Args:\n        x (int, tuple or list): Data size.\n    Returns:\n        size: A tuple or list (H,W).\n    \"\"\"\n    if isinstance(x, int):\n        return x, x\n    if isinstance(x, list) or isinstance(x, tuple):\n        return x\n    else:\n        raise TypeError()\n\n\ndef calculate_output_image_size(input_image_size, stride):\n    \"\"\"Calculates the output image size when using Conv2dSamePadding with a stride.\n       Necessary for static padding. Thanks to mannatsingh for pointing this out.\n    Args:\n        input_image_size (int, tuple or list): Size of input image.\n        stride (int, tuple or list): Conv2d operation's stride.\n    Returns:\n        output_image_size: A list [H,W].\n    \"\"\"\n    if input_image_size is None:\n        return None\n    image_height, image_width = get_width_and_height_from_size(input_image_size)\n    stride = stride if isinstance(stride, int) else stride[0]\n    image_height = int(math.ceil(image_height / stride))\n    image_width = int(math.ceil(image_width / stride))\n    return [image_height, image_width]\n\n\n# Note: \n# The following 'SamePadding' functions make output size equal ceil(input size/stride).\n# Only when stride equals 1, can the output size be the same as input size.\n# Don't be confused by their function names ! ! !\n\ndef get_same_padding_conv2d(image_size=None):\n    \"\"\"Chooses static padding if you have specified an image size, and dynamic padding otherwise.\n       Static padding is necessary for ONNX exporting of models.\n    Args:\n        image_size (int or tuple): Size of the image.\n    Returns:\n        Conv2dDynamicSamePadding or Conv2dStaticSamePadding.\n    \"\"\"\n    if image_size is None:\n        return Conv2dDynamicSamePadding\n    else:\n        return partial(Conv2dStaticSamePadding, image_size=image_size)\n\n\nclass Conv2dDynamicSamePadding(nn.Conv2d):\n    \"\"\"2D Convolutions like TensorFlow, for a dynamic image size.\n       The padding is operated in forward function by calculating dynamically.\n    \"\"\"\n\n    # Tips for 'SAME' mode padding.\n    #     Given the following:\n    #         i: width or height\n    #         s: stride\n    #         k: kernel size\n    #         d: dilation\n    #         p: padding\n    #     Output after Conv2d:\n    #         o = floor((i+p-((k-1)*d+1))/s+1)\n    # If o equals i, i = floor((i+p-((k-1)*d+1))/s+1),\n    # => p = (i-1)*s+((k-1)*d+1)-i\n\n    def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, groups=1, bias=True):\n        super().__init__(in_channels, out_channels, kernel_size, stride, 0, dilation, groups, bias)\n        self.stride = self.stride if len(self.stride) == 2 else [self.stride[0]] * 2\n\n    def forward(self, x):\n        ih, iw = x.size()[-2:]\n        kh, kw = self.weight.size()[-2:]\n        sh, sw = self.stride\n        oh, ow = math.ceil(ih / sh), math.ceil(iw / sw) # change the output size according to stride ! ! !\n        pad_h = max((oh - 1) * self.stride[0] + (kh - 1) * self.dilation[0] + 1 - ih, 0)\n        pad_w = max((ow - 1) * self.stride[1] + (kw - 1) * self.dilation[1] + 1 - iw, 0)\n        if pad_h > 0 or pad_w > 0:\n            x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2])\n        return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)\n\n\nclass Conv2dStaticSamePadding(nn.Conv2d):\n    \"\"\"2D Convolutions like TensorFlow's 'SAME' mode, with the given input image size.\n       The padding mudule is calculated in construction function, then used in forward.\n    \"\"\"\n\n    # With the same calculation as Conv2dDynamicSamePadding\n\n    def __init__(self, in_channels, out_channels, kernel_size, stride=1, image_size=None, **kwargs):\n        super().__init__(in_channels, out_channels, kernel_size, stride, **kwargs)\n        self.stride = self.stride if len(self.stride) == 2 else [self.stride[0]] * 2\n\n        # Calculate padding based on image size and save it\n        assert image_size is not None\n        ih, iw = (image_size, image_size) if isinstance(image_size, int) else image_size\n        kh, kw = self.weight.size()[-2:]\n        sh, sw = self.stride\n        oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)\n        pad_h = max((oh - 1) * self.stride[0] + (kh - 1) * self.dilation[0] + 1 - ih, 0)\n        pad_w = max((ow - 1) * self.stride[1] + (kw - 1) * self.dilation[1] + 1 - iw, 0)\n        if pad_h > 0 or pad_w > 0:\n            self.static_padding = nn.ZeroPad2d((pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2))\n        else:\n            self.static_padding = Identity()\n\n    def forward(self, x):\n        x = self.static_padding(x)\n        x = F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)\n        return x\n\n\ndef get_same_padding_maxPool2d(image_size=None):\n    \"\"\"Chooses static padding if you have specified an image size, and dynamic padding otherwise.\n       Static padding is necessary for ONNX exporting of models.\n    Args:\n        image_size (int or tuple): Size of the image.\n    Returns:\n        MaxPool2dDynamicSamePadding or MaxPool2dStaticSamePadding.\n    \"\"\"\n    if image_size is None:\n        return MaxPool2dDynamicSamePadding\n    else:\n        return partial(MaxPool2dStaticSamePadding, image_size=image_size)\n\n\nclass MaxPool2dDynamicSamePadding(nn.MaxPool2d):\n    \"\"\"2D MaxPooling like TensorFlow's 'SAME' mode, with a dynamic image size.\n       The padding is operated in forward function by calculating dynamically.\n    \"\"\"\n\n    def __init__(self, kernel_size, stride, padding=0, dilation=1, return_indices=False, ceil_mode=False):\n        super().__init__(kernel_size, stride, padding, dilation, return_indices, ceil_mode)\n        self.stride = [self.stride] * 2 if isinstance(self.stride, int) else self.stride\n        self.kernel_size = [self.kernel_size] * 2 if isinstance(self.kernel_size, int) else self.kernel_size\n        self.dilation = [self.dilation] * 2 if isinstance(self.dilation, int) else self.dilation\n\n    def forward(self, x):\n        ih, iw = x.size()[-2:]\n        kh, kw = self.kernel_size\n        sh, sw = self.stride\n        oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)\n        pad_h = max((oh - 1) * self.stride[0] + (kh - 1) * self.dilation[0] + 1 - ih, 0)\n        pad_w = max((ow - 1) * self.stride[1] + (kw - 1) * self.dilation[1] + 1 - iw, 0)\n        if pad_h > 0 or pad_w > 0:\n            x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2])\n        return F.max_pool2d(x, self.kernel_size, self.stride, self.padding,\n                            self.dilation, self.ceil_mode, self.return_indices)\n\nclass MaxPool2dStaticSamePadding(nn.MaxPool2d):\n    \"\"\"2D MaxPooling like TensorFlow's 'SAME' mode, with the given input image size.\n       The padding mudule is calculated in construction function, then used in forward.\n    \"\"\"\n\n    def __init__(self, kernel_size, stride, image_size=None, **kwargs):\n        super().__init__(kernel_size, stride, **kwargs)\n        self.stride = [self.stride] * 2 if isinstance(self.stride, int) else self.stride\n        self.kernel_size = [self.kernel_size] * 2 if isinstance(self.kernel_size, int) else self.kernel_size\n        self.dilation = [self.dilation] * 2 if isinstance(self.dilation, int) else self.dilation\n\n        # Calculate padding based on image size and save it\n        assert image_size is not None\n        ih, iw = (image_size, image_size) if isinstance(image_size, int) else image_size\n        kh, kw = self.kernel_size\n        sh, sw = self.stride\n        oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)\n        pad_h = max((oh - 1) * self.stride[0] + (kh - 1) * self.dilation[0] + 1 - ih, 0)\n        pad_w = max((ow - 1) * self.stride[1] + (kw - 1) * self.dilation[1] + 1 - iw, 0)\n        if pad_h > 0 or pad_w > 0:\n            self.static_padding = nn.ZeroPad2d((pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2))\n        else:\n            self.static_padding = Identity()\n\n    def forward(self, x):\n        x = self.static_padding(x)\n        x = F.max_pool2d(x, self.kernel_size, self.stride, self.padding,\n                         self.dilation, self.ceil_mode, self.return_indices)\n        return x\n    \n    \nclass Identity(nn.Module):\n    \"\"\"Identity mapping.\n       Send input to output directly.\n    \"\"\"\n\n    def __init__(self):\n        super(Identity, self).__init__()\n\n    def forward(self, input):\n        return input\n    \n    \ndef efficientnet_params(model_name):\n    \"\"\"Map EfficientNet model name to parameter coefficients.\n    Args:\n        model_name (str): Model name to be queried.\n    Returns:\n        params_dict[model_name]: A (width,depth,res,dropout) tuple.\n    \"\"\"\n    params_dict = {\n        # Coefficients:   width,depth,res,dropout\n        'efficientnet-b0': (1.0, 1.0, 224, 0.2),\n        'efficientnet-b1': (1.0, 1.1, 240, 0.2),\n        'efficientnet-b2': (1.1, 1.2, 260, 0.3),\n        'efficientnet-b3': (1.2, 1.4, 300, 0.3),\n        'efficientnet-b4': (1.4, 1.8, 380, 0.4),\n        'efficientnet-b5': (1.6, 2.2, 456, 0.4),\n        'efficientnet-b6': (1.8, 2.6, 528, 0.5),\n        'efficientnet-b7': (2.0, 3.1, 600, 0.5),\n        'efficientnet-b8': (2.2, 3.6, 672, 0.5),\n        'efficientnet-l2': (4.3, 5.3, 800, 0.5),\n    }\n    return params_dict[model_name]\n\n\ndef get_model_params(model_name, override_params):\n    \"\"\"Get the block args and global params for a given model name.\n    Args:\n        model_name (str): Model's name.\n        override_params (dict): A dict to modify global_params.\n    Returns:\n        blocks_args, global_params\n    \"\"\"\n    if model_name.startswith('efficientnet'):\n        w, d, s, p = efficientnet_params(model_name)\n        # note: all models have drop connect rate = 0.2\n        blocks_args, global_params = efficientnet(\n            width_coefficient=w, depth_coefficient=d, dropout_rate=p, image_size=s)\n    else:\n        raise NotImplementedError('model name is not pre-defined: %s' % model_name)\n    if override_params:\n        # ValueError will be raised here if override_params has fields not included in global_params.\n        global_params = global_params._replace(**override_params)\n    return blocks_args, global_params\n\n\nclass BlockDecoder(object):\n    \"\"\"Block Decoder for readability,\n       straight from the official TensorFlow repository.\n    \"\"\"\n\n    @staticmethod\n    def _decode_block_string(block_string):\n        \"\"\"Get a block through a string notation of arguments.\n        Args:\n            block_string (str): A string notation of arguments.\n                                Examples: 'r1_k3_s11_e1_i32_o16_se0.25_noskip'.\n        Returns:\n            BlockArgs: The namedtuple defined at the top of this file.\n        \"\"\"\n        assert isinstance(block_string, str)\n\n        ops = block_string.split('_')\n        options = {}\n        for op in ops:\n            splits = re.split(r'(\\d.*)', op)\n            if len(splits) >= 2:\n                key, value = splits[:2]\n                options[key] = value\n\n        # Check stride\n        assert (('s' in options and len(options['s']) == 1) or\n                (len(options['s']) == 2 and options['s'][0] == options['s'][1]))\n\n        return BlockArgs(\n            num_repeat=int(options['r']),\n            kernel_size=int(options['k']),\n            stride=[int(options['s'][0])],\n            expand_ratio=int(options['e']),\n            input_filters=int(options['i']),\n            output_filters=int(options['o']),\n            se_ratio=float(options['se']) if 'se' in options else None,\n            id_skip=('noskip' not in block_string))\n\n    @staticmethod\n    def _encode_block_string(block):\n        \"\"\"Encode a block to a string.\n        Args:\n            block (namedtuple): A BlockArgs type argument.\n        Returns:\n            block_string: A String form of BlockArgs.\n        \"\"\"\n        args = [\n            'r%d' % block.num_repeat,\n            'k%d' % block.kernel_size,\n            's%d%d' % (block.strides[0], block.strides[1]),\n            'e%s' % block.expand_ratio,\n            'i%d' % block.input_filters,\n            'o%d' % block.output_filters\n        ]\n        if 0 < block.se_ratio <= 1:\n            args.append('se%s' % block.se_ratio)\n        if block.id_skip is False:\n            args.append('noskip')\n        return '_'.join(args)\n\n    @staticmethod\n    def decode(string_list):\n        \"\"\"Decode a list of string notations to specify blocks inside the network.\n        Args:\n            string_list (list[str]): A list of strings, each string is a notation of block.\n        Returns:\n            blocks_args: A list of BlockArgs namedtuples of block args.\n        \"\"\"\n        assert isinstance(string_list, list)\n        blocks_args = []\n        for block_string in string_list:\n            blocks_args.append(BlockDecoder._decode_block_string(block_string))\n        return blocks_args\n\n    @staticmethod\n    def encode(blocks_args):\n        \"\"\"Encode a list of BlockArgs to a list of strings.\n        Args:\n            blocks_args (list[namedtuples]): A list of BlockArgs namedtuples of block args.\n        Returns:\n            block_strings: A list of strings, each string is a notation of block.\n        \"\"\"\n        block_strings = []\n        for block in blocks_args:\n            block_strings.append(BlockDecoder._encode_block_string(block))\n        return block_strings\n    \n    \ndef efficientnet(width_coefficient=None, depth_coefficient=None, image_size=None,\n                 dropout_rate=0.2, drop_connect_rate=0.2, num_classes=1000):\n    \"\"\"Create BlockArgs and GlobalParams for efficientnet model.\n    Args:\n        width_coefficient (float)\n        depth_coefficient (float)\n        image_size (int)\n        dropout_rate (float)\n        drop_connect_rate (float)\n        num_classes (int)\n        Meaning as the name suggests.\n    Returns:\n        blocks_args, global_params.\n    \"\"\"\n\n    # Blocks args for the whole model(efficientnet-b0 by default)\n    # It will be modified in the construction of EfficientNet Class according to model\n    blocks_args = [\n        'r1_k3_s11_e1_i32_o16_se0.25',\n        'r2_k3_s22_e6_i16_o24_se0.25',\n        'r2_k5_s22_e6_i24_o40_se0.25',\n        'r3_k3_s22_e6_i40_o80_se0.25',\n        'r3_k5_s11_e6_i80_o112_se0.25',\n        'r4_k5_s22_e6_i112_o192_se0.25',\n        'r1_k3_s11_e6_i192_o320_se0.25',\n    ]\n    blocks_args = BlockDecoder.decode(blocks_args)\n\n    global_params = GlobalParams(\n        width_coefficient=width_coefficient,\n        depth_coefficient=depth_coefficient,\n        image_size=image_size,\n        dropout_rate=dropout_rate,\n\n        num_classes=num_classes,\n        batch_norm_momentum=0.99,\n        batch_norm_epsilon=1e-3,\n        drop_connect_rate=drop_connect_rate,\n        depth_divisor=8,\n        min_depth=None,\n    )\n\n    return blocks_args, global_params\n\n\nGlobalParams = collections.namedtuple('GlobalParams', [\n    'width_coefficient', 'depth_coefficient', 'image_size', 'dropout_rate',\n    'num_classes', 'batch_norm_momentum', 'batch_norm_epsilon',\n    'drop_connect_rate', 'depth_divisor', 'min_depth'])\n\n# Parameters for an individual model block\nBlockArgs = collections.namedtuple('BlockArgs', [\n    'num_repeat', 'kernel_size', 'stride', 'expand_ratio',\n    'input_filters', 'output_filters', 'se_ratio', 'id_skip'])\n\n# Set GlobalParams and BlockArgs's defaults\nGlobalParams.__new__.__defaults__ = (None,) * len(GlobalParams._fields)\nBlockArgs.__new__.__defaults__ = (None,) * len(BlockArgs._fields)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"VALID_MODELS = (\n    'efficientnet-b0', 'efficientnet-b1', 'efficientnet-b2', 'efficientnet-b3',\n    'efficientnet-b4', 'efficientnet-b5', 'efficientnet-b6', 'efficientnet-b7',\n    'efficientnet-b8',\n\n    # Support the construction of 'efficientnet-l2' without pretrained weights\n    'efficientnet-l2'\n)\n\n\nclass MBConvBlock(nn.Module):\n    \"\"\"Mobile Inverted Residual Bottleneck Block.\n    Args:\n        block_args (namedtuple): BlockArgs, defined in utils.py.\n        global_params (namedtuple): GlobalParam, defined in utils.py.\n        image_size (tuple or list): [image_height, image_width].\n    References:\n        [1] https://arxiv.org/abs/1704.04861 (MobileNet v1)\n        [2] https://arxiv.org/abs/1801.04381 (MobileNet v2)\n        [3] https://arxiv.org/abs/1905.02244 (MobileNet v3)\n    \"\"\"\n\n    def __init__(self, block_args, global_params, image_size=None):\n        super().__init__()\n        self._block_args = block_args\n        self._bn_mom = 1 - global_params.batch_norm_momentum # pytorch's difference from tensorflow\n        self._bn_eps = global_params.batch_norm_epsilon\n        self.has_se = (self._block_args.se_ratio is not None) and (0 < self._block_args.se_ratio <= 1)\n        self.id_skip = block_args.id_skip  # whether to use skip connection and drop connect\n\n        # Expansion phase (Inverted Bottleneck)\n        inp = self._block_args.input_filters  # number of input channels\n        oup = self._block_args.input_filters * self._block_args.expand_ratio  # number of output channels\n        if self._block_args.expand_ratio != 1:\n            Conv2d = get_same_padding_conv2d(image_size=image_size)\n            self._expand_conv = Conv2d(in_channels=inp, out_channels=oup, kernel_size=1, bias=False)\n            self._bn0 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)\n            # image_size = calculate_output_image_size(image_size, 1) <-- this wouldn't modify image_size\n\n        # Depthwise convolution phase\n        k = self._block_args.kernel_size\n        s = self._block_args.stride\n        Conv2d = get_same_padding_conv2d(image_size=image_size)\n        self._depthwise_conv = Conv2d(\n            in_channels=oup, out_channels=oup, groups=oup,  # groups makes it depthwise\n            kernel_size=k, stride=s, bias=False)\n        self._bn1 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)\n        image_size = calculate_output_image_size(image_size, s)\n\n        # Squeeze and Excitation layer, if desired\n        if self.has_se:\n            Conv2d = get_same_padding_conv2d(image_size=(1,1))\n            num_squeezed_channels = max(1, int(self._block_args.input_filters * self._block_args.se_ratio))\n            self._se_reduce = Conv2d(in_channels=oup, out_channels=num_squeezed_channels, kernel_size=1)\n            self._se_expand = Conv2d(in_channels=num_squeezed_channels, out_channels=oup, kernel_size=1)\n\n        # Pointwise convolution phase\n        final_oup = self._block_args.output_filters\n        Conv2d = get_same_padding_conv2d(image_size=image_size)\n        self._project_conv = Conv2d(in_channels=oup, out_channels=final_oup, kernel_size=1, bias=False)\n        self._bn2 = nn.BatchNorm2d(num_features=final_oup, momentum=self._bn_mom, eps=self._bn_eps)\n        self._swish = MemoryEfficientSwish()\n\n    def forward(self, inputs, drop_connect_rate=None):\n        \"\"\"MBConvBlock's forward function.\n        Args:\n            inputs (tensor): Input tensor.\n            drop_connect_rate (bool): Drop connect rate (float, between 0 and 1).\n        Returns:\n            Output of this block after processing.\n        \"\"\"\n\n        # Expansion and Depthwise Convolution\n        x = inputs\n        if self._block_args.expand_ratio != 1:\n            x = self._expand_conv(inputs)\n            x = self._bn0(x)\n            x = self._swish(x)\n\n        x = self._depthwise_conv(x)\n        x = self._bn1(x)\n        x = self._swish(x)\n\n        # Squeeze and Excitation\n        if self.has_se:\n            x_squeezed = F.adaptive_avg_pool2d(x, 1)\n            x_squeezed = self._se_reduce(x_squeezed)\n            x_squeezed = self._swish(x_squeezed)\n            x_squeezed = self._se_expand(x_squeezed)\n            x = torch.sigmoid(x_squeezed) * x\n\n        # Pointwise Convolution\n        x = self._project_conv(x)\n        x = self._bn2(x)\n\n        # Skip connection and drop connect\n        input_filters, output_filters = self._block_args.input_filters, self._block_args.output_filters\n        if self.id_skip and self._block_args.stride == 1 and input_filters == output_filters:\n            # The combination of skip connection and drop connect brings about stochastic depth.\n            if drop_connect_rate:\n                x = drop_connect(x, p=drop_connect_rate, training=self.training)\n            x = x + inputs  # skip connection\n        return x\n\n    def set_swish(self, memory_efficient=True):\n        \"\"\"Sets swish function as memory efficient (for training) or standard (for export).\n        Args:\n            memory_efficient (bool): Whether to use memory-efficient version of swish.\n        \"\"\"\n        self._swish = MemoryEfficientSwish() if memory_efficient else Swish()\n        \n        \n        \nclass EfficientNet(nn.Module):\n    \"\"\"EfficientNet model.\n       Most easily loaded with the .from_name or .from_pretrained methods.\n    Args:\n        blocks_args (list[namedtuple]): A list of BlockArgs to construct blocks.\n        global_params (namedtuple): A set of GlobalParams shared between blocks.\n    \n    References:\n        [1] https://arxiv.org/abs/1905.11946 (EfficientNet)\n    Example:\n        >>> import torch\n        >>> from efficientnet.model import EfficientNet\n        >>> inputs = torch.rand(1, 3, 224, 224)\n        >>> model = EfficientNet.from_pretrained('efficientnet-b0')\n        >>> model.eval()\n        >>> outputs = model(inputs)\n    \"\"\"\n\n    def __init__(self, blocks_args=None, global_params=None):\n        super().__init__()\n        assert isinstance(blocks_args, list), 'blocks_args should be a list'\n        assert len(blocks_args) > 0, 'block args must be greater than 0'\n        self._global_params = global_params\n        self._blocks_args = blocks_args\n\n        # Batch norm parameters\n        bn_mom = 1 - self._global_params.batch_norm_momentum\n        bn_eps = self._global_params.batch_norm_epsilon\n\n        # Get stem static or dynamic convolution depending on image size\n        image_size = global_params.image_size\n        Conv2d = get_same_padding_conv2d(image_size=image_size)\n\n        # Stem\n        in_channels = 3  # rgb\n        out_channels = round_filters(32, self._global_params)  # number of output channels\n        self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)\n        self._bn0 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)\n        image_size = calculate_output_image_size(image_size, 2)\n\n        # Build blocks\n        self._blocks = nn.ModuleList([])\n        for block_args in self._blocks_args:\n\n            # Update block input and output filters based on depth multiplier.\n            block_args = block_args._replace(\n                input_filters=round_filters(block_args.input_filters, self._global_params),\n                output_filters=round_filters(block_args.output_filters, self._global_params),\n                num_repeat=round_repeats(block_args.num_repeat, self._global_params)\n            )\n\n            # The first block needs to take care of stride and filter size increase.\n            self._blocks.append(MBConvBlock(block_args, self._global_params, image_size=image_size))\n            image_size = calculate_output_image_size(image_size, block_args.stride)\n            if block_args.num_repeat > 1: # modify block_args to keep same output size\n                block_args = block_args._replace(input_filters=block_args.output_filters, stride=1)\n            for _ in range(block_args.num_repeat - 1):\n                self._blocks.append(MBConvBlock(block_args, self._global_params, image_size=image_size))\n                # image_size = calculate_output_image_size(image_size, block_args.stride)  # stride = 1\n\n        # Head\n        in_channels = block_args.output_filters  # output of final block\n        out_channels = round_filters(1280, self._global_params)\n        Conv2d = get_same_padding_conv2d(image_size=image_size)\n        self._conv_head = Conv2d(in_channels, out_channels, kernel_size=1, bias=False)\n        self._bn1 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)\n\n        # Final linear layer\n        self._avg_pooling = nn.AdaptiveAvgPool2d(1)\n        self._dropout = nn.Dropout(self._global_params.dropout_rate)\n        self._fc = nn.Linear(out_channels, self._global_params.num_classes)\n        self._swish = MemoryEfficientSwish()\n\n    def set_swish(self, memory_efficient=True):\n        \"\"\"Sets swish function as memory efficient (for training) or standard (for export).\n        Args:\n            memory_efficient (bool): Whether to use memory-efficient version of swish.\n        \"\"\"\n        self._swish = MemoryEfficientSwish() if memory_efficient else Swish()\n        for block in self._blocks:\n            block.set_swish(memory_efficient)\n\n    def extract_endpoints(self, inputs):\n        \"\"\"Use convolution layer to extract features\n        from reduction levels i in [1, 2, 3, 4, 5].\n        Args:\n            inputs (tensor): Input tensor.\n        Returns:\n            Dictionary of last intermediate features\n            with reduction levels i in [1, 2, 3, 4, 5].\n            Example:\n                >>> import torch\n                >>> from efficientnet.model import EfficientNet\n                >>> inputs = torch.rand(1, 3, 224, 224)\n                >>> model = EfficientNet.from_pretrained('efficientnet-b0')\n                >>> endpoints = model.extract_endpoints(inputs)\n                >>> print(endpoints['reduction_1'].shape)  # torch.Size([1, 16, 112, 112])\n                >>> print(endpoints['reduction_2'].shape)  # torch.Size([1, 24, 56, 56])\n                >>> print(endpoints['reduction_3'].shape)  # torch.Size([1, 40, 28, 28])\n                >>> print(endpoints['reduction_4'].shape)  # torch.Size([1, 112, 14, 14])\n                >>> print(endpoints['reduction_5'].shape)  # torch.Size([1, 1280, 7, 7])\n        \"\"\"\n        endpoints = dict()\n\n        # Stem\n        x = self._swish(self._bn0(self._conv_stem(inputs)))\n        prev_x = x\n\n        # Blocks\n        for idx, block in enumerate(self._blocks):\n            drop_connect_rate = self._global_params.drop_connect_rate\n            if drop_connect_rate:\n                drop_connect_rate *= float(idx) / len(self._blocks) # scale drop connect_rate\n            x = block(x, drop_connect_rate=drop_connect_rate)\n            if prev_x.size(2) > x.size(2):\n                endpoints[f'reduction_{len(endpoints)+1}'] = prev_x\n            prev_x = x\n\n        # Head\n        x = self._swish(self._bn1(self._conv_head(x)))\n        endpoints[f'reduction_{len(endpoints)+1}'] = x\n\n        return endpoints\n\n    def extract_features(self, inputs):\n        \"\"\"use convolution layer to extract feature .\n        Args:\n            inputs (tensor): Input tensor.\n        Returns:\n            Output of the final convolution \n            layer in the efficientnet model.\n        \"\"\"\n        # Stem\n        x = self._swish(self._bn0(self._conv_stem(inputs)))\n\n        # Blocks\n        for idx, block in enumerate(self._blocks):\n            drop_connect_rate = self._global_params.drop_connect_rate\n            if drop_connect_rate:\n                drop_connect_rate *= float(idx) / len(self._blocks) # scale drop connect_rate\n            x = block(x, drop_connect_rate=drop_connect_rate)\n        \n        # Head\n        x = self._swish(self._bn1(self._conv_head(x)))\n\n        return x\n\n    def forward(self, inputs):\n        \"\"\"EfficientNet's forward function.\n           Calls extract_features to extract features, applies final linear layer, and returns logits.\n        Args:\n            inputs (tensor): Input tensor.\n        Returns:\n            Output of this model after processing.\n        \"\"\"\n        # Convolution layers\n        x = self.extract_features(inputs)\n\n        # Pooling and final linear layer\n        x = self._avg_pooling(x)\n        x = x.flatten(start_dim=1)\n        x = self._dropout(x)\n        x = self._fc(x)\n\n        return x\n\n    @classmethod\n    def from_name(cls, model_name, in_channels=3, **override_params):\n        \"\"\"create an efficientnet model according to name.\n        Args:\n            model_name (str): Name for efficientnet.\n            in_channels (int): Input data's channel number.\n            override_params (other key word params): \n                Params to override model's global_params.\n                Optional key:\n                    'width_coefficient', 'depth_coefficient',\n                    'image_size', 'dropout_rate',\n                    'num_classes', 'batch_norm_momentum',\n                    'batch_norm_epsilon', 'drop_connect_rate',\n                    'depth_divisor', 'min_depth'\n        Returns:\n            An efficientnet model.\n        \"\"\"\n        cls._check_model_name_is_valid(model_name)\n        blocks_args, global_params = get_model_params(model_name, override_params)\n        model = cls(blocks_args, global_params)\n        model._change_in_channels(in_channels)\n        return model\n\n    @classmethod\n    def from_pretrained(cls, model_name, weights_path=None, advprop=False, \n                        in_channels=3, num_classes=1000, **override_params):\n        \"\"\"create an efficientnet model according to name.\n        Args:\n            model_name (str): Name for efficientnet.\n            weights_path (None or str): \n                str: path to pretrained weights file on the local disk.\n                None: use pretrained weights downloaded from the Internet.\n            advprop (bool): \n                Whether to load pretrained weights\n                trained with advprop (valid when weights_path is None).\n            in_channels (int): Input data's channel number.\n            num_classes (int): \n                Number of categories for classification.\n                It controls the output size for final linear layer.\n            override_params (other key word params): \n                Params to override model's global_params.\n                Optional key:\n                    'width_coefficient', 'depth_coefficient',\n                    'image_size', 'dropout_rate',\n                    'num_classes', 'batch_norm_momentum',\n                    'batch_norm_epsilon', 'drop_connect_rate',\n                    'depth_divisor', 'min_depth'\n        Returns:\n            A pretrained efficientnet model.\n        \"\"\"\n        model = cls.from_name(model_name, num_classes = num_classes, **override_params)\n        load_pretrained_weights(model, model_name, weights_path=weights_path, load_fc=(num_classes == 1000), advprop=advprop)\n        model._change_in_channels(in_channels)\n        return model\n\n    @classmethod\n    def get_image_size(cls, model_name):\n        \"\"\"Get the input image size for a given efficientnet model.\n        Args:\n            model_name (str): Name for efficientnet.\n        Returns:\n            Input image size (resolution).\n        \"\"\"\n        cls._check_model_name_is_valid(model_name)\n        _, _, res, _ = efficientnet_params(model_name)\n        return res\n\n    @classmethod\n    def _check_model_name_is_valid(cls, model_name):\n        \"\"\"Validates model name. \n        Args:\n            model_name (str): Name for efficientnet.\n        Returns:\n            bool: Is a valid name or not.\n        \"\"\"\n        if model_name not in VALID_MODELS:\n            raise ValueError('model_name should be one of: ' + ', '.join(VALID_MODELS))\n\n    def _change_in_channels(self, in_channels):\n        \"\"\"Adjust model's first convolution layer to in_channels, if in_channels not equals 3.\n        Args:\n            in_channels (int): Input data's channel number.\n        \"\"\"\n        if in_channels != 3:\n            Conv2d = get_same_padding_conv2d(image_size = self._global_params.image_size)\n            out_channels = round_filters(32, self._global_params)\n            self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class ResNestSED(nn.Module):\n    def __init__(self, num_classes=264):\n        super().__init__()\n        self.interpolate_ratio = 30  # Downsampled ratio\n        base_model = ResNest(\n                Bottleneck, [3, 4, 6, 3],\n                radix=1, groups=1, bottleneck_width=64,\n                deep_stem=True, stem_width=32, avg_down=True,\n                avd=True, avd_first=True)\n        layers = list(base_model.children())[:-2]\n        self.encoder = nn.Sequential(*layers)\n\n        in_features = base_model.fc.in_features\n\n        self.fc1 = nn.Linear(in_features, in_features, bias=True)\n        self.att_block = AttBlock(in_features, num_classes, activation=\"sigmoid\")\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_layer(self.fc1)\n\n    def forward(self, input):\n        frames_num = input.size(3)\n\n        # (batch_size, channels, freq, frames)\n        x = self.encoder(input)\n\n        # (batch_size, channels, frames)\n        x = torch.mean(x, dim=2)\n\n        # channel smoothing\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.5, training=self.training)\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n        logit = torch.sum(norm_att * self.att_block.cla(x), dim=2)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        # Get framewise output\n        framewise_output = interpolate(segmentwise_output,\n                                       self.interpolate_ratio)\n        framewise_output = pad_framewise_output(framewise_output, frames_num)\n\n        output_dict = {\n            \"framewise_output\": framewise_output,\n            \"logit\": logit,\n            \"clipwise_output\": clipwise_output\n        }\n\n        return output_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class EfficientNetSED(nn.Module):\n    def __init__(self, base_model_name: str, pretrained=False,\n                 num_classes=264):\n        super().__init__()\n        self.interpolate_ratio = 32  # Downsampled ratio\n        if pretrained:\n            self.base_model = EfficientNet.from_pretrained(base_model_name)\n        else:\n            self.base_model = EfficientNet.from_name(base_model_name)\n\n        in_features = self.base_model._fc.in_features\n\n        self.fc1 = nn.Linear(in_features, in_features, bias=True)\n        self.att_block = AttBlock(in_features, num_classes, activation=\"sigmoid\")\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_layer(self.fc1)\n\n    def forward(self, input):\n        frames_num = input.size(3)\n\n        # (batch_size, channels, freq, frames)\n        x = self.base_model.extract_features(input)\n\n        # (batch_size, channels, frames)\n        x = torch.mean(x, dim=2)\n\n        # channel smoothing\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.5, training=self.training)\n        (clipwise_output, _, segmentwise_output) = self.att_block(x)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        # Get framewise output\n        framewise_output = interpolate(segmentwise_output,\n                                       self.interpolate_ratio)\n        framewise_output = pad_framewise_output(framewise_output, frames_num)\n\n        output_dict = {\n            \"framewise_output\": framewise_output,\n            \"segmentwise_output\": segmentwise_output,\n            \"clipwise_output\": clipwise_output\n        }\n\n        return output_dict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"resnest_model_config = {\n    \"num_classes\": 264\n}\n\neffnet_model_config = {\n    \"num_classes\": 264,\n    \"base_model_name\": \"efficientnet-b0\",\n    \"pretrained\": False\n}\n\nweights_path = {\n    \"resnest\": {\n        \"base\": \"../input/birdcall-resnest-ema-all/ema.pth\",\n        \"th04\": \"../input/birdcall-resnest-ema-all-th04/ema.pth\",\n        \"th06\": \"../input/birdcall-resnest-ema-all-th06/ema.pth\",\n        \"ref2\": \"../input/birdcall-resnest-ema-all-refine2/ema.pth\",\n        \"ref2_th03\": \"../input/birdcall-resnest-ema-all-ref2-th03/ema.pth\",\n        \"ref2_th04\": \"../input/birdcall-resnest-ema-all-ref2-th04/ema.pth\",\n        \"ref2_th06\": \"../input/birdcall-resnest-ema-all-ref2-th06/ema.pth\",\n        \"ref2_th07\": \"../input/birdcall-resnest-ema-all-ref2-th07/ema.pth\",\n        \"ext\": \"../input/birdcall-resnest-emta-all-ext-ref2-th04/ema.pth\"\n    },\n    \"effnet\": {\n        \"eff_th04\": \"../input/birdcall-effnet-ema-all-focal-ref2-th04/ema.pth\",\n        \"eff_th05\": \"../input/birdcall-effnet-b0-ema-all-ref2/ema.pth\"\n    }\n}\n\nratio = {\n    \"base\": 0.05,\n    \"th06\": 0.05,\n    \"th04\": 0.01,\n    \"ref2\": 0.05,\n    \"ref2_th03\": 0.25,\n    \"ref2_th04\": 0.14,\n    \"ref2_th06\": 0.05,\n    \"ref2_th07\": 0.01,\n    \"eff_th04\": 0.13,\n    \"eff_th05\": 0.01,\n    \"ext\": 0.25\n}\n\nmelspectrogram_parameters = {\n    \"n_mels\": 128,\n    \"fmin\": 20,\n    \"fmax\": 16000\n}\n\npcen_parameters = {\n    \"gain\": 0.98,\n    \"bias\": 2,\n    \"power\": 0.5,\n    \"time_constant\": 0.4,\n    \"eps\": 0.000001\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BIRD_CODE = {\n    'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,\n    'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,\n    'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,\n    'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,\n    'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,\n    'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,\n    'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,\n    'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,\n    'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,\n    'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,\n    'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,\n    'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,\n    'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,\n    'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,\n    'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,\n    'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,\n    'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,\n    'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,\n    'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,\n    'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,\n    'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,\n    'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,\n    'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,\n    'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,\n    'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,\n    'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,\n    'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,\n    'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,\n    'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,\n    'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,\n    'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,\n    'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,\n    'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,\n    'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,\n    'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,\n    'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,\n    'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,\n    'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,\n    'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,\n    'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,\n    'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,\n    'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,\n    'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,\n    'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,\n    'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,\n    'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,\n    'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,\n    'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,\n    'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,\n    'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,\n    'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,\n    'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,\n    'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263\n}\n\nINV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items()}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"thresholds = {\n    'aldfly': 0.8300000000000001, 'ameavo': 0.35000000000000003, 'amebit': 0.73, 'amecro': 0.35000000000000003, 'amegfi': 0.53, \n    'amekes': 0.47000000000000003, 'amepip': 0.78, 'amered': 0.87, 'amerob': 0.26, 'amewig': 0.98, \n    'amewoo': 0.81, 'amtspa': 0.74, 'annhum': 0.78, 'astfly': 0.9500000000000001, 'baisan': 0.9400000000000001, \n    'baleag': 0.61, 'balori': 0.74, 'banswa': 0.85, 'barswa': 0.6, 'bawwar': 0.4, \n    'belkin1': 0.41000000000000003, 'belspa2': 0.79, 'bewwre': 0.9, 'bkbcuc': 0.76, 'bkbmag1': 0.67, \n    'bkbwar': 0.54, 'bkcchi': 0.58, 'bkchum': 0.68, 'bkhgro': 0.52, 'bkpwar': 0.5700000000000001, \n    'bktspa': 0.87, 'blkpho': 0.7000000000000001, 'blugrb1': 0.4, 'blujay': 0.66, 'bnhcow': 0.56, \n    'boboli': 0.6, 'bongul': 0.7000000000000001, 'brdowl': 0.98, 'brebla': 0.77, 'brespa': 0.9, \n    'brncre': 0.6900000000000001, 'brnthr': 0.97, 'brthum': 0.86, 'brwhaw': 0.9500000000000001, 'btbwar': 0.89, \n    'btnwar': 0.9, 'btywar': 0.98, 'buffle': 0.9400000000000001, 'buggna': 0.67, 'buhvir': 0.34, \n    'bulori': 0.68, 'bushti': 0.67, 'buwtea': 0.3, 'buwwar': 0.61, 'cacwre': 0.59, \n    'calgul': 0.89, 'calqua': 0.86, 'camwar': 0.67, 'cangoo': 0.46, 'canwar': 0.86, \n    'canwre': 0.76, 'carwre': 0.53, 'casfin': 0.6900000000000001, 'caster1': 0.48, 'casvir': 0.79, \n    'cedwax': 0.5, 'chispa': 0.43, 'chiswi': 0.6, 'chswar': 0.9500000000000001, 'chukar': 0.47000000000000003, \n    'clanut': 0.99, 'cliswa': 0.51, 'comgol': 0.5, 'comgra': 0.63, 'comloo': 0.29, \n    'commer': 0.79, 'comnig': 0.81, 'comrav': 0.42, 'comred': 0.76, 'comter': 0.67, \n    'comyel': 0.73, 'coohaw': 0.46, 'coshum': 0.64, 'cowscj1': 0.56, 'daejun': 0.65, \n    'doccor': 0.72, 'dowwoo': 0.68, 'dusfly': 0.8300000000000001, 'eargre': 0.9, 'easblu': 0.71, \n    'easkin': 0.97, 'easmea': 0.81, 'easpho': 0.96, 'eastow': 0.5, 'eawpew': 0.6900000000000001, \n    'eucdov': 0.65, 'eursta': 0.47000000000000003, 'evegro': 0.81, 'fiespa': 0.63, 'fiscro': 0.91, \n    'foxspa': 0.55, 'gadwal': 0.77, 'gcrfin': 0.6900000000000001, 'gnttow': 0.6900000000000001, 'gnwtea': 0.67, \n    'gockin': 0.66, 'gocspa': 0.86, 'goleag': 0.48, 'grbher3': 0.59, 'grcfly': 0.91, \n    'greegr': 0.6900000000000001, 'greroa': 0.81, 'greyel': 0.91, 'grhowl': 0.8200000000000001, 'grnher': 0.59, \n    'grtgra': 0.52, 'grycat': 0.65, 'gryfly': 0.5, 'haiwoo': 0.76, 'hamfly': 0.51, \n    'hergul': 0.92, 'herthr': 0.88, 'hoomer': 0.64, 'hoowar': 0.85, 'horgre': 0.47000000000000003, \n    'horlar': 0.78, 'houfin': 0.36, 'houspa': 0.44, 'houwre': 0.38, 'indbun': 0.71, \n    'juntit1': 0.96, 'killde': 0.6900000000000001, 'labwoo': 0.64, 'larspa': 0.67, 'lazbun': 0.6900000000000001, \n    'leabit': 0.88, 'leafly': 0.35000000000000003, 'leasan': 0.62, 'lecthr': 0.9400000000000001, 'lesgol': 0.72, \n    'lesnig': 0.6, 'lesyel': 0.37, 'lewwoo': 0.62, 'linspa': 0.88, 'lobcur': 0.73, \n    'lobdow': 0.49, 'logshr': 0.93, 'lotduc': 0.43, 'louwat': 0.6900000000000001, 'macwar': 0.76, \n    'magwar': 0.99, 'mallar3': 0.68, 'marwre': 0.81, 'merlin': 0.77, 'moublu': 0.79, \n    'mouchi': 0.85, 'moudov': 0.59, 'norcar': 0.38, 'norfli': 0.5, 'norhar2': 0.48, \n    'normoc': 0.42, 'norpar': 0.51, 'norpin': 0.84, 'norsho': 0.48, 'norwat': 0.56, \n    'nrwswa': 0.36, 'nutwoo': 0.38, 'olsfly': 0.96, 'orcwar': 0.78, 'osprey': 0.97, \n    'ovenbi1': 0.56, 'palwar': 0.48, 'pasfly': 0.9, 'pecsan': 0.41000000000000003, 'perfal': 0.6900000000000001, \n    'phaino': 0.91, 'pibgre': 0.48, 'pilwoo': 0.64, 'pingro': 0.84, 'pinjay': 0.92, \n    'pinsis': 0.39, 'pinwar': 0.88, 'plsvir': 0.6, 'prawar': 0.9, 'purfin': 0.55, \n    'pygnut': 0.9400000000000001, 'rebmer': 0.76, 'rebnut': 0.33, 'rebsap': 0.42, 'rebwoo': 0.38, \n    'redcro': 0.77, 'redhea': 0.86, 'reevir1': 0.72, 'renpha': 0.48, 'reshaw': 0.5700000000000001, \n    'rethaw': 0.79, 'rewbla': 0.43, 'ribgul': 0.3, 'rinduc': 0.47000000000000003, 'robgro': 0.92, \n    'rocpig': 0.71, 'rocwre': 0.58, 'rthhum': 0.44, 'ruckin': 0.85, 'rudduc': 0.67, \n    'rufgro': 0.27, 'rufhum': 0.86, 'rusbla': 0.8200000000000001, 'sagspa1': 0.75, 'sagthr': 0.86, \n    'savspa': 0.81, 'saypho': 0.85, 'scatan': 0.96, 'scoori': 0.93, 'semplo': 0.46, \n    'semsan': 0.96, 'sheowl': 0.43, 'shshaw': 0.44, 'snobun': 0.6, 'snogoo': 0.97, \n    'solsan': 0.73, 'sonspa': 0.59, 'sora': 0.72, 'sposan': 0.5700000000000001, 'spotow': 0.55, \n    'stejay': 0.6900000000000001, 'swahaw': 0.58, 'swaspa': 0.6, 'swathr': 0.67, 'treswa': 0.98, \n    'truswa': 0.93, 'tuftit': 0.71, 'tunswa': 0.55, 'veery': 0.91, 'vesspa': 0.96, \n    'vigswa': 0.45, 'warvir': 0.86, 'wesblu': 0.65, 'wesgre': 0.6, 'weskin': 0.8200000000000001, \n    'wesmea': 0.58, 'wessan': 0.63, 'westan': 0.47000000000000003, 'wewpew': 0.78, 'whbnut': 0.48, \n    'whcspa': 0.72, 'whfibi': 0.66, 'whtspa': 0.76, 'whtswi': 0.96, 'wilfly': 0.68, \n    'wilsni1': 0.36, 'wiltur': 0.41000000000000003, 'winwre3': 0.68, 'wlswar': 0.77, 'wooduc': 0.61, \n    'wooscj2': 0.93, 'woothr': 0.8200000000000001, 'y00475': 0.45, 'yebfly': 0.52, 'yebsap': 0.39, \n    'yehbla': 0.6900000000000001, 'yelwar': 0.66, 'yerwar': 0.64, 'yetvir': 0.9\n}\n\nfor key in thresholds:\n    thresholds[key] = 0.4","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prediction loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model(resnest_config: dict, effnet_config: dict, weights_path: dict):\n    models = {}\n    device = torch.device(\"cuda\")\n    for model_key in weights_path:\n        path_dict = weights_path[model_key]\n        for path_key in path_dict:\n            if model_key == \"resnest\":\n                model = ResNestSED(**resnest_config)\n            else:\n                model = EfficientNetSED(**effnet_config)\n            checkpoint = torch.load(path_dict[path_key])\n            model_state_dict = {}\n            for key in checkpoint[\"model_state_dict\"]:\n                if key == \"n_averaged\":\n                    continue\n                new_key = key.replace(\"module.\", \"\")\n                model_state_dict[new_key] = checkpoint[\"model_state_dict\"][key]\n            model.load_state_dict(model_state_dict)\n            model.to(device)\n            model.eval()\n            models[path_key] = model\n    return models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def normalize_melspec(X: np.ndarray):\n    eps = 1e-6\n    mean = X.mean()\n    X = X - mean\n    std = X.std()\n    Xstd = X / (std + eps)\n    norm_min, norm_max = Xstd.min(), Xstd.max()\n    if (norm_max - norm_min) > eps:\n        V = Xstd\n        V[V < norm_min] = norm_min\n        V[V > norm_max] = norm_max\n        V = 255 * (V - norm_min) / (norm_max - norm_min)\n        V = V.astype(np.uint8)\n    else:\n        # Just zero\n        V = np.zeros_like(Xstd, dtype=np.uint8)\n    return V","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prediction_for_clip(test_df: pd.DataFrame,\n                        clip: np.ndarray, \n                        models,\n                        melspectrogram_parameters={},\n                        pcen_parameters={}):\n    PERIOD = 30\n    images = []\n    y = clip.astype(np.float32)\n    len_y = len(y)\n    start = 0\n    end = PERIOD * TARGET_SR\n    while True:\n        y_batch = y[start:end].astype(np.float32)\n        if len(y_batch) > 0:\n            max_vol = np.abs(y_batch).max()\n            if max_vol > 0:\n                y_batch = np.asfortranarray(y_batch * 1 / max_vol)\n        if len(y_batch) != PERIOD * TARGET_SR:\n            y_pad = np.zeros(PERIOD * TARGET_SR, dtype=np.float32)\n            y_pad[:len(y_batch)] = y_batch\n            \n            melspec = librosa.feature.melspectrogram(y_pad,\n                                                     sr=TARGET_SR,\n                                                     **melspectrogram_parameters)\n            pcen = librosa.pcen(melspec, sr=TARGET_SR, **pcen_parameters)\n            clean_mel = librosa.power_to_db(melspec ** 1.5)\n            melspec = librosa.power_to_db(melspec).astype(np.float32)\n                    \n            norm_melspec = normalize_melspec(melspec)\n            norm_pcen = normalize_melspec(pcen)\n            norm_clean_mel = normalize_melspec(clean_mel)\n            image = np.stack([norm_melspec, norm_pcen, norm_clean_mel], axis=-1)\n            height, width, _ = image.shape\n            image = cv2.resize(image, (int(width * 224 / height), 224))\n            image = np.moveaxis(image, 2, 0)\n            image = (image / 255.0).astype(np.float32)\n            images.append(image)\n            break\n        start = end\n        end += PERIOD * TARGET_SR\n        \n        melspec = librosa.feature.melspectrogram(y_batch,\n                                                 sr=TARGET_SR,\n                                                 **melspectrogram_parameters)\n        pcen = librosa.pcen(melspec, sr=TARGET_SR, **pcen_parameters)\n        clean_mel = librosa.power_to_db(melspec ** 1.5)\n        melspec = librosa.power_to_db(melspec).astype(np.float32)\n\n        norm_melspec = normalize_melspec(melspec)\n        norm_pcen = normalize_melspec(pcen)\n        norm_clean_mel = normalize_melspec(clean_mel)\n        image = np.stack([norm_melspec, norm_pcen, norm_clean_mel], axis=-1)\n\n        height, width, _ = image.shape\n        image = cv2.resize(image, (int(width * 224 / height), 224))\n        image = np.moveaxis(image, 2, 0)\n        image = (image / 255.0).astype(np.float32)\n        images.append(image)\n        \n    array = np.asarray(images)\n    tensors = torch.from_numpy(array)\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n    estimated_event_list = []\n    global_time = 0.0\n    site = test_df[\"site\"].values[0]\n    audio_id = test_df[\"audio_id\"].values[0]\n    for image in progress_bar(tensors):\n        image = image.view(1, image.size(0), image.size(1), image.size(2))\n        image = image.to(device)\n\n        outputs = {}\n        with torch.no_grad():\n            for key in models:\n                prediction = models[key](image)\n                framewise_outputs = prediction[\"framewise_output\"].detach(\n                    ).cpu().numpy()[0]\n                outputs[key] = framewise_outputs\n                \n        key = list(outputs.keys())[0]\n        framewise_outputs = np.zeros_like(outputs[key], dtype=np.float32)\n        for key in outputs:\n            framewise_outputs += ratio[key] * outputs[key]\n                \n        thresholded = np.zeros_like(framewise_outputs)\n        for i in range(len(thresholds)):\n            thresholded[:, i] = framewise_outputs[:, i] >= thresholds[INV_BIRD_CODE[i]]\n        sec_per_frame = PERIOD / thresholded.shape[0]\n\n        for target_idx in range(thresholded.shape[1]):\n            if thresholded[:, target_idx].mean() == 0:\n                pass\n            else:\n                detected = np.argwhere(thresholded[:, target_idx]).reshape(-1)\n                head_idx = 0\n                tail_idx = 0\n                while True:\n                    if (tail_idx + 1 == len(detected)) or (\n                            detected[tail_idx + 1] - \n                            detected[tail_idx] != 1):\n                        onset = sec_per_frame * detected[\n                            head_idx] + global_time\n                        offset = sec_per_frame * detected[\n                            tail_idx] + global_time\n                        onset_idx = detected[head_idx]\n                        offset_idx = detected[tail_idx]\n                        max_confidence = framewise_outputs[\n                            onset_idx:offset_idx, target_idx].max()\n                        mean_confidence = framewise_outputs[\n                            onset_idx:offset_idx, target_idx].mean()\n                        estimated_event = {\n                            \"site\": site,\n                            \"audio_id\": audio_id,\n                            \"ebird_code\": INV_BIRD_CODE[target_idx],\n                            \"onset\": onset,\n                            \"offset\": offset,\n                            \"max_confidence\": max_confidence,\n                            \"mean_confidence\": mean_confidence\n                        }\n                        estimated_event_list.append(estimated_event)\n                        head_idx = tail_idx + 1\n                        tail_idx = tail_idx + 1\n                        if head_idx >= len(detected):\n                            break\n                    else:\n                        tail_idx += 1\n        global_time += PERIOD\n        \n    prediction_df = pd.DataFrame(estimated_event_list)\n    return prediction_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prediction(test_df: pd.DataFrame,\n               test_audio: Path,\n               resnest_config: dict,\n               effnet_config: dict,\n               weights_path: dict,\n               melspectrogram_parameters={},\n               pcen_parameters={}):\n    models = get_model(resnest_config, effnet_config, weights_path)\n    unique_audio_id = test_df.audio_id.unique()\n\n    warnings.filterwarnings(\"ignore\")\n    prediction_dfs = []\n    for audio_id in unique_audio_id:\n        with timer(f\"Loading {audio_id}\"):\n            clip, _ = librosa.load(test_audio / (audio_id + \".mp3\"),\n                                   sr=TARGET_SR,\n                                   mono=True,\n                                   res_type=\"kaiser_fast\")\n        \n        test_df_for_audio_id = test_df.query(\n            f\"audio_id == '{audio_id}'\").reset_index(drop=True)\n        with timer(f\"Prediction on {audio_id}\"):\n            prediction_df = prediction_for_clip(test_df_for_audio_id,\n                                                clip=clip,\n                                                models=models,\n                                                melspectrogram_parameters=melspectrogram_parameters,\n                                                pcen_parameters=pcen_parameters)\n\n        prediction_dfs.append(prediction_df)\n    \n    prediction_df = pd.concat(prediction_dfs, axis=0, sort=False).reset_index(drop=True)\n    return prediction_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction_df = prediction(test_df=test,\n                           test_audio=test_audio,\n                           resnest_config=resnest_model_config,\n                           effnet_config=effnet_model_config,\n                           weights_path=weights_path,\n                           melspectrogram_parameters=melspectrogram_parameters,\n                           pcen_parameters=pcen_parameters)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Postprocess"},{"metadata":{"trusted":true},"cell_type":"code","source":"# prediction_df[\"duration\"] = prediction_df[\"offset\"] - prediction_df[\"onset\"]\n# prediction_df = prediction_df.query(\"duration > 0.1\").reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = {}\n\nfor audio_id, sub_df in prediction_df.groupby(\"audio_id\"):\n    events = sub_df[[\"ebird_code\", \"onset\", \"offset\", \"max_confidence\", \"site\"]].values\n    n_events = len(events)\n\n    site = events[0][4]\n    for i in range(n_events):\n        event = events[i][0]\n        onset = events[i][1]\n        offset = events[i][2]\n        if site in {\"site_1\", \"site_2\"}:\n            start_section = int((onset // 5) * 5) + 5\n            end_section = int((offset // 5) * 5) + 5\n            cur_section = start_section\n\n            row_id = f\"{site}_{audio_id}_{start_section}\"\n            if labels.get(row_id) is not None:\n                labels[row_id].add(event)\n            else:\n                labels[row_id] = set()\n                labels[row_id].add(event)\n\n            while cur_section != end_section:\n                cur_section += 5\n                row_id = f\"{site}_{audio_id}_{cur_section}\"\n                if labels.get(row_id) is not None:\n                    labels[row_id].add(event)\n                else:\n                    labels[row_id] = set()\n                    labels[row_id].add(event)\n        else:\n            row_id = f\"{site}_{audio_id}\"\n            if labels.get(row_id) is not None:\n                labels[row_id].add(event)\n            else:\n                labels[row_id] = set()\n                labels[row_id].add(event)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for key in labels:\n    labels[key] = \" \".join(sorted(list(labels[key])))\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"row_ids = list(labels.keys())\nbirds = list(labels.values())\npost_processed = pd.DataFrame({\n    \"row_id\": row_ids,\n    \"birds\": birds\n})\npost_processed.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_row_id = test[[\"row_id\"]]\nsubmission = all_row_id.merge(post_processed, on=\"row_id\", how=\"left\")\nsubmission = submission.fillna(\"nocall\")\nsubmission.to_csv(\"submission.csv\", index=False)\nsubmission.head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## EOF"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}