{"cells":[{"metadata":{"_uuid":"00be38c7-f8b3-455d-95d3-0a35284aa3df","_cell_guid":"521d3373-7a32-4e8c-a5e9-6e47e4d173c3","trusted":true},"cell_type":"code","source":"!pip install /kaggle/input/torchlibrosa/torchlibrosa-0.0.4-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d8ac3aeb-6401-4758-8be0-034019d8f50b","_cell_guid":"fe36c203-44d7-4bb5-9c49-92cd530e91aa","trusted":true},"cell_type":"code","source":"import sys\nsys.path.insert(0, '/kaggle/input/geffnet-blend2/')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0a149b22-e90e-4ea4-9d49-4902b67f9cf6","_cell_guid":"df164511-d6c0-4244-a924-7c39602dad00","trusted":true},"cell_type":"code","source":"from geffnet import tf_efficientnet_b4_ns, tf_efficientnet_b3_ns, tf_efficientnet_b0_ns, tf_efficientnet_b2_ns","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bbf7c99c-a0bc-41a6-b982-1bac3b02fea0","_cell_guid":"06895b66-dc23-4cf2-a3b4-c199c8fb148a","trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport time\nimport math\nimport shutil\nimport random\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport typing as tp\nfrom pathlib import Path\n\nimport cv2\nimport librosa\nimport tqdm\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.nn import Conv2d, Module, Linear, BatchNorm2d, ReLU\nfrom torch.nn.modules.utils import _pair\nimport torch.utils.data as data\n\nfrom sklearn.preprocessing import LabelEncoder\nimport torchlibrosa\nfrom argparse import Namespace","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e01ebf9c-e9ae-4b74-b906-e16463af1824","_cell_guid":"5f8cdb1f-3b93-4c86-9923-56accab73e71","trusted":true},"cell_type":"code","source":"def set_seed(seed: int = 42):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)  # type: ignore\n    torch.cuda.manual_seed_all(seed)\n    #torch.backends.cudnn.deterministic = True  # type: ignore\n    torch.backends.cudnn.benchmark = False  # type: ignore\n    \nset_seed(1213)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b6df4cfc-b878-4c4e-9e6c-9adec48a94df","_cell_guid":"17ef9481-645e-4059-bad9-d9a3b7adec60","trusted":true},"cell_type":"code","source":"ROOT = Path.cwd().parent\nINPUT_ROOT = ROOT / \"input\"\nRAW_DATA = INPUT_ROOT / \"birdsong-recognition\"\nTRAIN_AUDIO_DIR = RAW_DATA / \"train_audio\"\nTEST_AUDIO_DIR = RAW_DATA / \"test_audio\"\n\nif not TEST_AUDIO_DIR.exists():\n    TEST_AUDIO_DIR = INPUT_ROOT / \"birdcall-check\" / \"test_audio\"\n    test = pd.read_csv(INPUT_ROOT / \"birdcall-check\" / \"test.csv\")\nelse:\n    test = pd.read_csv(RAW_DATA / \"test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"70a2a2e1-a8da-4cfb-91ca-e41b818ad94b","_cell_guid":"e58cb79b-1fd5-4429-bfc4-85cfab3c92c5","trusted":true},"cell_type":"code","source":"train = pd.read_csv(RAW_DATA / \"train.csv\")\nlabel_encoder = LabelEncoder().fit(train.ebird_code.values)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"161950d2-007f-4def-a827-5beb139b4a82","_cell_guid":"118eb221-92fc-456c-a584-0f163f317c70","trusted":true},"cell_type":"code","source":"class model_config:\n    ROOT_PATH = \"../input/train_audio\"\n    num_classes = 264\n    max_duration = 5\n    sample_rate = 32000\n    sigmoid = True\n\n    batch_size = 32\n    num_workers = 4\n    nmels = 128\n\n    melspectrogram_parameters = {\n        \"n_mels\": 128, \n        \"fmin\": 20, \n        \"fmax\": 16000, \n        \"hop_length\": 320, \n        \"n_fft\": 1024\n    }\n    threshold = 0.3\n    augm_spec_prob = 0.\n    res_type = \"kaiser_best\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f614791f-9966-4131-8ba4-353d9611b2fd","_cell_guid":"12ff0beb-cc17-4dee-a7ac-768c45f1811c","trusted":true},"cell_type":"code","source":"def init_layer(layer):\n    \"\"\"Initialize a Linear or Convolutional layer. \"\"\"\n    nn.init.xavier_uniform_(layer.weight)\n\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.0)\n\n\ndef init_bn(bn):\n    \"\"\"Initialize a Batchnorm layer. \"\"\"\n    bn.bias.data.fill_(0.0)\n    bn.weight.data.fill_(1.0)\n\n    \nclass LogMel(nn.Module):\n    def __init__(self, params):\n        super(LogMel, self).__init__()\n        params = Namespace(**params)\n        self.spectrogram_extractor = torchlibrosa.stft.Spectrogram(n_fft=params.n_fft, hop_length=params.hop_length)\n        self.logmel_extractor = torchlibrosa.stft.LogmelFilterBank(\n            sr=32000, \n            n_fft=params.n_fft, \n            n_mels=params.n_mels, \n            top_db=None, \n            fmin=params.fmin, \n            fmax=params.fmax, \n            is_log=False)\n\n    def forward(self, sound):\n        spec = self.spectrogram_extractor(sound)\n        spec = self.logmel_extractor(spec)\n        spec = torch.log(1e-8 + spec)\n\n        return spec / 10.\n\nclass AttBlock(nn.Module):\n    def __init__(self, n_in, n_out, activation=\"linear\", temperature=1.0):\n        super(AttBlock, self).__init__()\n\n        self.activation = activation\n        self.temperature = temperature\n        self.att = nn.Conv1d(\n            in_channels=n_in,\n            out_channels=n_out,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True,\n        )\n        self.cla = nn.Conv1d(\n            in_channels=n_in,\n            out_channels=n_out,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True,\n        )\n\n        self.bn_att = nn.BatchNorm1d(n_out)\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n        init_bn(self.bn_att)\n\n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.clamp(self.att(x), -10, 10), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == \"linear\":\n            return x\n        elif self.activation == \"sigmoid\":\n            return torch.sigmoid(x)\n\n\nclass ConvBlock(nn.Module):\n    def __init__(self, in_channels, out_channels):\n\n        super(ConvBlock, self).__init__()\n\n        self.conv1 = nn.Conv2d(\n            in_channels=in_channels,\n            out_channels=out_channels,\n            kernel_size=(3, 3),\n            stride=(1, 1),\n            padding=(1, 1),\n            bias=False,\n        )\n\n        self.conv2 = nn.Conv2d(\n            in_channels=out_channels,\n            out_channels=out_channels,\n            kernel_size=(3, 3),\n            stride=(1, 1),\n            padding=(1, 1),\n            bias=False,\n        )\n\n        self.bn1 = nn.BatchNorm2d(out_channels)\n        self.bn2 = nn.BatchNorm2d(out_channels)\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_layer(self.conv1)\n        init_layer(self.conv2)\n        init_bn(self.bn1)\n        init_bn(self.bn2)\n\n    def forward(self, input, pool_size=(2, 2), pool_type=\"avg\"):\n\n        x = input\n        x = F.relu_(self.bn1(self.conv1(x)))\n        x = F.relu_(self.bn2(self.conv2(x)))\n        if pool_type == \"max\":\n            x = F.max_pool2d(x, kernel_size=pool_size)\n        elif pool_type == \"avg\":\n            x = F.avg_pool2d(x, kernel_size=pool_size)\n        elif pool_type == \"avg+max\":\n            x1 = F.avg_pool2d(x, kernel_size=pool_size)\n            x2 = F.max_pool2d(x, kernel_size=pool_size)\n            x = x1 + x2\n        else:\n            raise Exception(\"Incorrect argument!\")\n\n        return x\n\n\ndef interpolate(x, ratio):\n    \"\"\"Interpolate data in time domain. This is used to compensate the \n    resolution reduction in downsampling of a CNN.\n    \n    Args:\n      x: (batch_size, time_steps, classes_num)\n      ratio: int, ratio to interpolate\n    Returns:\n      upsampled: (batch_size, time_steps * ratio, classes_num)\n    \"\"\"\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output, frames_num):\n    \"\"\"Pad framewise_output to the same length as input frames. The pad value \n    is the same as the value of the last frame.\n    Args:\n      framewise_output: (batch_size, frames_num, classes_num)\n      frames_num: int, number of frames to pad\n    Outputs:\n      output: (batch_size, frames_num, classes_num)\n    \"\"\"\n    pad = framewise_output[:, -1:, :].repeat(\n        1, frames_num - framewise_output.shape[1], 1\n    )\n    \"\"\"tensor for padding\"\"\"\n\n    output = torch.cat((framewise_output, pad), dim=1)\n    \"\"\"(batch_size, frames_num, classes_num)\"\"\"\n\n    return output\n\n\nclass Cnn14_DecisionLevelAtt(nn.Module):\n    def __init__(self, classes_num, config):\n        super(Cnn14_DecisionLevelAtt, self).__init__()\n        self.interpolate_ratio = 32  # Downsampled ratio\n\n        self.bn0 = nn.BatchNorm2d(config.melspectrogram_parameters[\"n_mels\"])\n\n        self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)\n        self.conv_block2 = ConvBlock(in_channels=64, out_channels=128)\n        self.conv_block3 = ConvBlock(in_channels=128, out_channels=256)\n        self.conv_block4 = ConvBlock(in_channels=256, out_channels=512)\n        self.conv_block5 = ConvBlock(in_channels=512, out_channels=1024)\n        self.conv_block6 = ConvBlock(in_channels=1024, out_channels=2048)\n\n        self.fc1 = nn.Linear(2048, 2048, bias=True)\n        self.att_block = AttBlock(2048, classes_num, activation=\"sigmoid\")\n        self.logmel = LogMel(config.melspectrogram_parameters)\n        \n        self.init_weight()\n\n    def init_weight(self):\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n\n    def forward(self, x):\n        #print(x.shape)\n        x = self.logmel(x)\n        #print(x.shape)\n        #x = x.transpose(2, 3)\n        frames_num = x.shape[2]\n\n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n\n        x = self.conv_block1(x, pool_size=(2, 2), pool_type=\"avg\")\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block2(x, pool_size=(2, 2), pool_type=\"avg\")\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block3(x, pool_size=(2, 2), pool_type=\"avg\")\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block4(x, pool_size=(2, 2), pool_type=\"avg\")\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block5(x, pool_size=(2, 2), pool_type=\"avg\")\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block6(x, pool_size=(1, 1), pool_type=\"avg\")\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = torch.mean(x, dim=3)\n        \n\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.5, training=self.training)\n        (clipwise_output, _, segmentwise_output) = self.att_block(x)\n#         segmentwise_output = segmentwise_output.transpose(1, 2)\n\n#         # Get framewise output\n#         framewise_output = interpolate(segmentwise_output, self.interpolate_ratio)\n#         framewise_output = pad_framewise_output(framewise_output, frames_num)\n\n#         output_dict = {\n#             \"framewise_output\": framewise_output,\n#             \"clipwise_output\": clipwise_output,\n#         }\n\n        # print(clipwise_output.min(), clipwise_output.max())\n        return clipwise_output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _resnet_conv3x3(in_planes, out_planes):\n    #3x3 convolution with padding\n    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=1,\n                     padding=1, groups=1, bias=False, dilation=1)\n\n\ndef _resnet_conv1x1(in_planes, out_planes):\n    #1x1 convolution\n    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, bias=False)\n\n\nclass _ResnetBasicBlock(nn.Module):\n    expansion = 1\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,\n                 base_width=64, dilation=1, norm_layer=None):\n        super(_ResnetBasicBlock, self).__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        if groups != 1 or base_width != 64:\n            raise ValueError('_ResnetBasicBlock only supports groups=1 and base_width=64')\n        if dilation > 1:\n            raise NotImplementedError(\"Dilation > 1 not supported in _ResnetBasicBlock\")\n        # Both self.conv1 and self.downsample layers downsample the input when stride != 1\n\n        self.stride = stride\n\n        self.conv1 = _resnet_conv3x3(inplanes, planes)\n        self.bn1 = norm_layer(planes)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = _resnet_conv3x3(planes, planes)\n        self.bn2 = norm_layer(planes)\n        self.downsample = downsample\n        self.stride = stride\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.conv1)\n        init_bn(self.bn1)\n        init_layer(self.conv2)\n        init_bn(self.bn2)\n        nn.init.constant_(self.bn2.weight, 0)\n\n    def forward(self, x):\n        identity = x\n\n        if self.stride == 2:\n            out = F.avg_pool2d(x, kernel_size=(2, 2))\n        else:\n            out = x\n\n        out = self.conv1(out)\n        out = self.bn1(out)\n        out = self.relu(out)\n        out = F.dropout(out, p=0.1, training=self.training)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        \n        if self.downsample is not None:\n            identity = self.downsample(identity)\n\n        out += identity\n        out = self.relu(out)\n\n        return out\n\n\nclass _ResnetBottleneck(nn.Module):\n    expansion = 4\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,\n                 base_width=64, dilation=1, norm_layer=None):\n        super(_ResnetBottleneck, self).__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        width = int(planes * (base_width / 64.)) * groups\n        self.stride = stride\n        # Both self.conv2 and self.downsample layers downsample the input when stride != 1\n        self.conv1 = _resnet_conv1x1(inplanes, width)\n        self.bn1 = norm_layer(width)\n        self.conv2 = _resnet_conv3x3(width, width)\n        self.bn2 = norm_layer(width)\n        self.conv3 = _resnet_conv1x1(width, planes * self.expansion)\n        self.bn3 = norm_layer(planes * self.expansion)\n        self.relu = nn.ReLU(inplace=True)\n        self.downsample = downsample\n        self.stride = stride\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.conv1)\n        init_bn(self.bn1)\n        init_layer(self.conv2)\n        init_bn(self.bn2)\n        init_layer(self.conv3)\n        init_bn(self.bn3)\n        nn.init.constant_(self.bn3.weight, 0)\n\n    def forward(self, x):\n        identity = x\n\n        if self.stride == 2:\n            x = F.avg_pool2d(x, kernel_size=(2, 2))\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n        out = F.dropout(out, p=0.1, training=self.training)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n\n        if self.downsample is not None:\n            identity = self.downsample(identity)\n\n        out += identity\n        out = self.relu(out)\n\n        return out\n\n\nclass _ResNet(nn.Module):\n    def __init__(self, block, layers, zero_init_residual=False,\n                 groups=1, width_per_group=64, replace_stride_with_dilation=None,\n                 norm_layer=None):\n        super(_ResNet, self).__init__()\n\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        self._norm_layer = norm_layer\n\n        self.inplanes = 64\n        self.dilation = 1\n        if replace_stride_with_dilation is None:\n            # each element in the tuple indicates if we should replace\n            # the 2x2 stride with a dilated convolution instead\n            replace_stride_with_dilation = [False, False, False]\n        if len(replace_stride_with_dilation) != 3:\n            raise ValueError(\"replace_stride_with_dilation should be None \"\n                             \"or a 3-element tuple, got {}\".format(replace_stride_with_dilation))\n        self.groups = groups\n        self.base_width = width_per_group\n\n        self.layer1 = self._make_layer(block, 64, layers[0], stride=1)\n        self.layer2 = self._make_layer(block, 128, layers[1], stride=2,\n                                       dilate=replace_stride_with_dilation[0])\n        self.layer3 = self._make_layer(block, 256, layers[2], stride=2,\n                                       dilate=replace_stride_with_dilation[1])\n        self.layer4 = self._make_layer(block, 512, layers[3], stride=2,\n                                       dilate=replace_stride_with_dilation[2])\n\n    def _make_layer(self, block, planes, blocks, stride=1, dilate=False):\n        norm_layer = self._norm_layer\n        downsample = None\n        previous_dilation = self.dilation\n        if dilate:\n            self.dilation *= stride\n            stride = 1\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            if stride == 1:\n                downsample = nn.Sequential(\n                    _resnet_conv1x1(self.inplanes, planes * block.expansion),\n                    norm_layer(planes * block.expansion),\n                )\n                init_layer(downsample[0])\n                init_bn(downsample[1])\n            elif stride == 2:\n                downsample = nn.Sequential(\n                    nn.AvgPool2d(kernel_size=2), \n                    _resnet_conv1x1(self.inplanes, planes * block.expansion),\n                    norm_layer(planes * block.expansion),\n                )\n                init_layer(downsample[1])\n                init_bn(downsample[2])\n\n        layers = []\n        layers.append(block(self.inplanes, planes, stride, downsample, self.groups,\n                            self.base_width, previous_dilation, norm_layer))\n        self.inplanes = planes * block.expansion\n        for _ in range(1, blocks):\n            layers.append(block(self.inplanes, planes, groups=self.groups,\n                                base_width=self.base_width, dilation=self.dilation,\n                                norm_layer=norm_layer))\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n\n        return x\n\nclass ResNet38(nn.Module):\n    def __init__(self, classes_num, config):\n        super(ResNet38, self).__init__()\n\n        self.bn0 = nn.BatchNorm2d(config.nmels)\n\n        self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)\n\n        self.resnet = _ResNet(block=_ResnetBasicBlock, layers=[3, 4, 6, 3], zero_init_residual=True)\n\n        self.conv_block_after1 = ConvBlock(in_channels=512, out_channels=2048)\n\n        self.fc1 = nn.Linear(2048, 2048)\n        self.classifier = nn.Linear(2048, classes_num, bias=True)\n\n        self.init_weights()\n\n        self.logmel = LogMel(config.melspectrogram_parameters)\n        self.spec_augm = torchlibrosa.augmentation.SpecAugmentation(\n            time_drop_width=64,\n            time_stripes_num=2 * (config.max_duration // 5),\n            freq_drop_width=8,\n            freq_stripes_num=2,\n        )\n        self.spec_augm_prob = config.augm_spec_prob\n\n    def init_weights(self):\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n        init_layer(self.classifier)\n\n\n    def forward(self, input):\n        x = self.logmel(input)\n        if self.training:\n            mask = (torch.rand(x.size(0)) > 0.33).to(device)\n            x = torch.where(\n                torch.repeat_interleave(mask, x.size(2) * x.size(3)).reshape(x.size()),\n                x, self.spec_augm(x))\n        \n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n    \n        \n        x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training, inplace=True)\n        x = self.resnet(x)\n        x = F.avg_pool2d(x, kernel_size=(2, 2))\n        x = F.dropout(x, p=0.2, training=self.training, inplace=True)\n        x = self.conv_block_after1(x, pool_size=(1, 1), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training, inplace=True)\n        x = torch.mean(x, dim=3)\n        \n        (x1, _) = torch.max(x, dim=2)\n        x2 = torch.mean(x, dim=2)\n        x = x1 + x2\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = F.relu_(self.fc1(x))\n        embedding = F.dropout(x, p=0.5, training=self.training)\n        clipwise_output = self.classifier(x)\n\n        return clipwise_output","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5e2a488a-d1a6-4018-a069-96488732a748","_cell_guid":"878e2f62-f6b7-43c9-8ad5-f05420eadea5","trusted":true},"cell_type":"code","source":"def get_model(config):\n    models = []\n    sigmoids = []\n    weights = []\n    \n    for fold in range(5):\n        # b4_128_xeno_pshift_bg\n        config.model = \"b4\"\n        config.vertical_encoding = True\n        config.batch_norm = True\n        \n        model = tf_efficientnet_b4_ns(\n            config, pretrained=False, num_classes=config.num_classes\n        ).cuda()\n        \n        state_dict = torch.load(f\"../input/blend5-b4-best/fold_{fold}_test_1_05.pth\")\n        model.load_state_dict(state_dict, strict=False)\n        model.eval()\n        models.append(model)  \n        sigmoids.append(True) \n        weights.append(1)   \n        \n    for fold in range(5):\n        config.model = \"b3\"\n        config.vertical_encoding = True\n        config.batch_norm = True\n        \n        model = tf_efficientnet_b3_ns(\n            config, pretrained=False, num_classes=config.num_classes\n        ).cuda()\n        state_dict = torch.load(f\"../input/blend5-b3-london/fold_{fold}_test_1_05.pth\")\n        model.load_state_dict(state_dict, strict=False)\n        model.eval()\n        models.append(model) \n        sigmoids.append(True)  \n        weights.append(1)   \n        \n    for fold in range(5):\n        config.model = \"b2\"\n        config.vertical_encoding = True\n        config.batch_norm = True\n        \n        model = tf_efficientnet_b2_ns(\n            config, pretrained=False, num_classes=config.num_classes\n        ).cuda()\n        state_dict = torch.load(f\"../input/blend5-b2-xbet/fold_{fold}_test_1_05.pth\")\n        model.load_state_dict(state_dict, strict=False)\n        model.eval()\n        models.append(model)  \n        sigmoids.append(True) \n        weights.append(1)   \n        \n    for fold in range(5):\n        config.model = \"b0\"\n        config.vertical_encoding = True\n        config.batch_norm = True\n        \n        model = tf_efficientnet_b0_ns(\n            config, pretrained=False, num_classes=config.num_classes\n        ).cuda()\n        state_dict = torch.load(f\"../input/blend5-b0-detka/fold_{fold}_test_1_05.pth\")\n        model.load_state_dict(state_dict, strict=False)\n        model.eval()\n        models.append(model)  \n        sigmoids.append(True) \n        weights.append(1)   \n        \n    for fold in range(5):\n        config.model = \"cnn14_att\"        \n        model = Cnn14_DecisionLevelAtt(config.num_classes, config).cuda()\n        state_dict = torch.load(f\"../input/blend5-cnn14-mega/fold_{fold}_test_1_05.pth\")\n        model.load_state_dict(state_dict, strict=False)\n        model.eval()\n        models.append(model)  \n        sigmoids.append(False) \n        weights.append(2)   \n        \n    for fold in range(5):\n        config.model = \"r38\"        \n        model = ResNet38(config.num_classes, config).cuda()\n        state_dict = torch.load(f\"../input/blend5-r38-oxford/fold_{fold}_test_1_05.pth\")\n        model.load_state_dict(state_dict, strict=False)\n        model.eval()\n        models.append(model)  \n        sigmoids.append(True) \n        weights.append(2)   \n    \n    return models, sigmoids, weights","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9ac9fb66-f0d2-49ba-ac82-d81d27248bc3","_cell_guid":"afcd75da-57b6-49f1-bd86-c7350e945c3f","trusted":true},"cell_type":"code","source":"from librosa.core.audio import __audioread_load as audioread_load","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"86967c81-e0d4-4d15-8b03-c96ee5c4269c","_cell_guid":"53bda7d0-876f-493f-80d1-a9d6f53ec8bc","trusted":true},"cell_type":"code","source":"class TestItem:\n    def __init__(self, audio_id, site, df, spectrograms):\n        self.audio_id = audio_id\n        self.site = site\n        self.df = df\n        self.spectrograms = torch.from_numpy(spectrograms)\n        \n    def pin_memory(self):\n        self.spectrograms = self.spectrograms.pin_memory()\n        return self\n    \n    def __len__(self):\n        return len(self.df)\n\n\nclass TestDataset(data.Dataset):\n    def __init__(self, test, config):\n        self.test = test\n        self.config = config\n        self.unique_audio_id = test.audio_id.unique()\n\n    def __len__(self):\n        return len(self.unique_audio_id)\n\n    def _load_audio(self, filename):\n        clip, _ = librosa.load(\n            TEST_AUDIO_DIR / (filename + \".mp3\"), sr=model_config.sample_rate, mono=True#, res_type=\"kaiser_best\"\n        )\n\n        return clip.astype(np.float32)\n    \n    def _load_audio_faster(self, filename):\n        path = TEST_AUDIO_DIR / (filename + '.mp3')\n        clip, sr_native = audioread_load(path, offset=0.0, duration=None, dtype=np.float32)\n        clip = librosa.to_mono(clip)\n        if sr_native > 0:\n            clip = librosa.resample(clip, sr_native, model_config.sample_rate, res_type=model_config.res_type)\n            \n        return clip.astype(np.float32)\n\n    def _transform(self, audio, max_chunks):\n        audio = audio[:self.config.sample_rate * self.config.max_duration * max_chunks]\n        return audio.reshape(max_chunks, self.config.sample_rate * self.config.max_duration)\n\n\n    def __getitem__(self, idx: int):\n        audio_id = self.unique_audio_id[idx]\n        test_df_for_audio_id = test.query(f\"audio_id == '{audio_id}'\").reset_index(\n            drop=True\n        )\n        site = test_df_for_audio_id.iloc[0].site\n        audio = self._load_audio_faster(audio_id)\n\n        if site != \"site_3\":\n            specs = self._transform(audio, len(test_df_for_audio_id))\n        else:\n            num_chunks = (len(audio) / self.config.sample_rate) // 5\n            specs = self._transform(audio, int(num_chunks))\n\n        return TestItem(audio_id, site, test_df_for_audio_id, specs)\n\ndef mo_(output):\n    if type(output) == tuple:\n        return output[0]\n    else:\n        return output\n    \ndef prediction_for_clip(model, spectrograms, config):\n    model.eval()\n    with torch.no_grad():\n        predictions = []\n        batches = math.ceil(spectrograms.shape[0] / config.batch_size)\n\n        for i in range(batches):\n            batch = spectrograms[i * config.batch_size : (i + 1) * config.batch_size]\n            if batch.ndim == 3:\n                batch = batch.unsqueeze(0)\n                \n            pred = mo_(model(batch))\n            predictions.append(pred)\n\n        predictions = torch.cat(predictions, dim=0)\n\n        if config.sigmoid:\n            predictions = predictions.sigmoid()\n\n        return predictions","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e4ab9b6b-686a-4148-9efc-195bc2ff1543","_cell_guid":"f83d6fb1-8b46-45d3-b980-ea58ce1c09a0","trusted":true},"cell_type":"code","source":"models, sigmoids, weights = get_model(model_config)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0ef3f385-3cba-4add-9bf7-9e5ace4e668e","_cell_guid":"76a760c2-2e38-4756-868e-12b5533a60a7","trusted":true},"cell_type":"code","source":"predictions_top = []\ntest_items_top = []\n\nfor test_item in tqdm.tqdm(\n    torch.utils.data.DataLoader(\n        TestDataset(test, model_config), collate_fn=lambda x: x, num_workers=4, shuffle=False, pin_memory=True\n    ),\n    disable=False\n):\n    test_item = test_item[0]\n    spectrograms = test_item.spectrograms.cuda()\n    \n    predictions_all = []\n    for model, to_sigmoid, weight in zip(models, sigmoids, weights):\n        model_config.sigmoid = to_sigmoid\n        predictions = prediction_for_clip(model, spectrograms, model_config)\n        predictions_all.append(predictions)\n        \n        # ugly code I know\n        if weight == 2:\n            predictions_all.append(predictions)\n        \n    predictions = torch.stack(predictions_all, dim=0).mean(dim=0)\n    predictions = predictions.detach().cpu().numpy()\n    \n    if test_item.site == \"site_3\":\n        predictions = predictions.max(axis=0)\n        predictions = np.expand_dims(predictions, 0)\n        \n    assert len(predictions) == len(test_item)\n    \n    predictions_top.extend(list(predictions))\n    test_items_top.extend(test_item.df.row_id.values)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f1a7a721-553d-4557-b82e-b87f9a6d7402","_cell_guid":"232b7952-aae6-44d2-b32e-5688f523a404","trusted":true},"cell_type":"code","source":"prediction_dict = {}\nfor i, row_id in enumerate(test_items_top):\n    events = (predictions_top[i] >= model_config.threshold)\n    labels = np.argwhere(events).reshape(-1).tolist()\n\n    if len(labels) == 0:\n        prediction_dict[row_id] = \"nocall\"\n    else:\n        labels_str_list = label_encoder.inverse_transform(labels)\n        label_string = \" \".join(labels_str_list)\n        prediction_dict[row_id] = label_string","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ff0291fd-8edb-4885-af86-f7f2ff9b0f35","_cell_guid":"7309cf66-1406-4382-a0f6-3ebfca7d6d5c","trusted":true},"cell_type":"code","source":"row_id = list(prediction_dict.keys())\nbirds = list(prediction_dict.values())\nsubmission = pd.DataFrame({\n    \"row_id\": row_id,\n    \"birds\": birds\n})\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a063a700-5ba7-4d31-bb73-0ccdadf6d725","_cell_guid":"b421bdac-9d72-4f1f-a042-8fb7f6153f4b","trusted":true},"cell_type":"code","source":"submission.head(50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}