{"cells":[{"metadata":{},"cell_type":"markdown","source":"Updated: \n+ Deeper Network: Resnet38 + GRU -> SED\n+ Trainer with high control\n+ Mixup add\n+ Add new customer loss: NLL, BCE, MAP(option)\n+ Augmetation","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"import cv2\nimport audioread\nimport logging\nimport os\nimport random\nimport time\nimport warnings\n\nimport librosa\nimport librosa.display as display\nimport numpy as np\nimport pandas as pd\nimport soundfile as sf\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.utils.data as data\nfrom contextlib import contextmanager\nfrom IPython.display import Audio\nfrom pathlib import Path\nfrom typing import Optional, List\n\nfrom catalyst.dl import SupervisedRunner, State, CallbackOrder, Callback, CheckpointCallback\nfrom fastprogress import progress_bar\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score, average_precision_score","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def set_seed(seed: int = 42):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)  # type: ignore\n    torch.backends.cudnn.deterministic = True  # type: ignore\n    torch.backends.cudnn.benchmark = True  # type: ignore\n    \n    \ndef get_logger(out_file=None):\n    logger = logging.getLogger()\n    formatter = logging.Formatter(\"%(asctime)s - %(levelname)s - %(message)s\")\n    logger.handlers = []\n    logger.setLevel(logging.INFO)\n\n    handler = logging.StreamHandler()\n    handler.setFormatter(formatter)\n    handler.setLevel(logging.INFO)\n    logger.addHandler(handler)\n\n    if out_file is not None:\n        fh = logging.FileHandler(out_file)\n        fh.setFormatter(formatter)\n        fh.setLevel(logging.INFO)\n        logger.addHandler(fh)\n    logger.info(\"logger set up\")\n    return logger\n    \n    \n@contextmanager\ndef timer(name: str, logger: Optional[logging.Logger] = None):\n    t0 = time.time()\n    msg = f\"[{name}] start\"\n    if logger is None:\n        print(msg)\n    else:\n        logger.info(msg)\n    yield\n\n    msg = f\"[{name}] done in {time.time() - t0:.2f} s\"\n    if logger is None:\n        print(msg)\n    else:\n        logger.info(msg)\n    \n    \nset_seed(1213)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"ROOT = Path.cwd().parent\nINPUT_ROOT = ROOT / \"input\"\nRAW_DATA = INPUT_ROOT / \"birdsong-recognition\"\nTRAIN_AUDIO_DIR = RAW_DATA / \"train_audio\"\nTRAIN_RESAMPLED_AUDIO_DIRS = [\n  INPUT_ROOT / \"birdsong-resampled-train-audio-{:0>2}\".format(i)  for i in range(5)\n]\nTEST_AUDIO_DIR = RAW_DATA / \"test_audio\"","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"train = pd.read_csv(TRAIN_RESAMPLED_AUDIO_DIRS[0] / \"train_mod.csv\")\n\nif not TEST_AUDIO_DIR.exists():\n    TEST_AUDIO_DIR = INPUT_ROOT / \"birdcall-check\" / \"test_audio\"\n    test = pd.read_csv(INPUT_ROOT / \"birdcall-check\" / \"test.csv\")\nelse:\n    test = pd.read_csv(RAW_DATA / \"test.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### torchlibrosa\n\n\nIn PANNs, `torchlibrosa`, a PyTorch based implementation are used to replace some of the `librosa`'s functions. Here I use some functions of `torchlibrosa`.\n\nRef: https://github.com/qiuqiangkong/torchlibrosa","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"#### LICENSE\n\n```\nISC License\nCopyright (c) 2013--2017, librosa development team.\n\nPermission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.\n```","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class DFTBase(nn.Module):\n    def __init__(self):\n        \"\"\"Base class for DFT and IDFT matrix\"\"\"\n        super(DFTBase, self).__init__()\n\n    def dft_matrix(self, n):\n        (x, y) = np.meshgrid(np.arange(n), np.arange(n))\n        omega = np.exp(-2 * np.pi * 1j / n)\n        W = np.power(omega, x * y)\n        return W\n\n    def idft_matrix(self, n):\n        (x, y) = np.meshgrid(np.arange(n), np.arange(n))\n        omega = np.exp(2 * np.pi * 1j / n)\n        W = np.power(omega, x * y)\n        return W\n    \n    \nclass STFT(DFTBase):\n    def __init__(self, n_fft=2048, hop_length=None, win_length=None, \n        window='hann', center=True, pad_mode='reflect', freeze_parameters=True):\n        \"\"\"Implementation of STFT with Conv1d. The function has the same output \n        of librosa.core.stft\n        \"\"\"\n        super(STFT, self).__init__()\n\n        assert pad_mode in ['constant', 'reflect']\n\n        self.n_fft = n_fft\n        self.center = center\n        self.pad_mode = pad_mode\n\n        # By default, use the entire frame\n        if win_length is None:\n            win_length = n_fft\n\n        # Set the default hop, if it's not already specified\n        if hop_length is None:\n            hop_length = int(win_length // 4)\n\n        fft_window = librosa.filters.get_window(window, win_length, fftbins=True)\n\n        # Pad the window out to n_fft size\n        fft_window = librosa.util.pad_center(fft_window, n_fft)\n\n        # DFT & IDFT matrix\n        self.W = self.dft_matrix(n_fft)\n\n        out_channels = n_fft // 2 + 1\n\n        self.conv_real = nn.Conv1d(in_channels=1, out_channels=out_channels, \n            kernel_size=n_fft, stride=hop_length, padding=0, dilation=1, \n            groups=1, bias=False)\n\n        self.conv_imag = nn.Conv1d(in_channels=1, out_channels=out_channels, \n            kernel_size=n_fft, stride=hop_length, padding=0, dilation=1, \n            groups=1, bias=False)\n\n        self.conv_real.weight.data = torch.Tensor(\n            np.real(self.W[:, 0 : out_channels] * fft_window[:, None]).T)[:, None, :]\n        # (n_fft // 2 + 1, 1, n_fft)\n\n        self.conv_imag.weight.data = torch.Tensor(\n            np.imag(self.W[:, 0 : out_channels] * fft_window[:, None]).T)[:, None, :]\n        # (n_fft // 2 + 1, 1, n_fft)\n\n        if freeze_parameters:\n            for param in self.parameters():\n                param.requires_grad = False\n\n    def forward(self, input):\n        \"\"\"input: (batch_size, data_length)\n        Returns:\n          real: (batch_size, n_fft // 2 + 1, time_steps)\n          imag: (batch_size, n_fft // 2 + 1, time_steps)\n        \"\"\"\n\n        x = input[:, None, :]   # (batch_size, channels_num, data_length)\n\n        if self.center:\n            x = F.pad(x, pad=(self.n_fft // 2, self.n_fft // 2), mode=self.pad_mode)\n\n        real = self.conv_real(x)\n        imag = self.conv_imag(x)\n        # (batch_size, n_fft // 2 + 1, time_steps)\n\n        real = real[:, None, :, :].transpose(2, 3)\n        imag = imag[:, None, :, :].transpose(2, 3)\n        # (batch_size, 1, time_steps, n_fft // 2 + 1)\n\n        return real, imag\n    \n    \nclass Spectrogram(nn.Module):\n    def __init__(self, n_fft=2048, hop_length=None, win_length=None, \n        window='hann', center=True, pad_mode='reflect', power=2.0, \n        freeze_parameters=True):\n        \"\"\"Calculate spectrogram using pytorch. The STFT is implemented with \n        Conv1d. The function has the same output of librosa.core.stft\n        \"\"\"\n        super(Spectrogram, self).__init__()\n\n        self.power = power\n\n        self.stft = STFT(n_fft=n_fft, hop_length=hop_length, \n            win_length=win_length, window=window, center=center, \n            pad_mode=pad_mode, freeze_parameters=True)\n\n    def forward(self, input):\n        \"\"\"input: (batch_size, 1, time_steps, n_fft // 2 + 1)\n        Returns:\n          spectrogram: (batch_size, 1, time_steps, n_fft // 2 + 1)\n        \"\"\"\n\n        (real, imag) = self.stft.forward(input)\n        # (batch_size, n_fft // 2 + 1, time_steps)\n\n        spectrogram = real ** 2 + imag ** 2\n\n        if self.power == 2.0:\n            pass\n        else:\n            spectrogram = spectrogram ** (power / 2.0)\n\n        return spectrogram\n\n    \nclass LogmelFilterBank(nn.Module):\n    def __init__(self, sr=32000, n_fft=2048, n_mels=64, fmin=50, fmax=14000, is_log=True, \n        ref=1.0, amin=1e-10, top_db=80.0, freeze_parameters=True):\n        \"\"\"Calculate logmel spectrogram using pytorch. The mel filter bank is \n        the pytorch implementation of as librosa.filters.mel \n        \"\"\"\n        super(LogmelFilterBank, self).__init__()\n\n        self.is_log = is_log\n        self.ref = ref\n        self.amin = amin\n        self.top_db = top_db\n\n        self.melW = librosa.filters.mel(sr=sr, n_fft=n_fft, n_mels=n_mels,\n            fmin=fmin, fmax=fmax).T\n        # (n_fft // 2 + 1, mel_bins)\n\n        self.melW = nn.Parameter(torch.Tensor(self.melW))\n\n        if freeze_parameters:\n            for param in self.parameters():\n                param.requires_grad = False\n\n    def forward(self, input):\n        \"\"\"input: (batch_size, channels, time_steps)\n        \n        Output: (batch_size, time_steps, mel_bins)\n        \"\"\"\n\n        # Mel spectrogram\n        mel_spectrogram = torch.matmul(input, self.melW)\n\n        # Logmel spectrogram\n        if self.is_log:\n            output = self.power_to_db(mel_spectrogram)\n        else:\n            output = mel_spectrogram\n\n        return output\n\n\n    def power_to_db(self, input):\n        \"\"\"Power to db, this function is the pytorch implementation of \n        librosa.core.power_to_lb\n        \"\"\"\n        ref_value = self.ref\n        log_spec = 10.0 * torch.log10(torch.clamp(input, min=self.amin, max=np.inf))\n        log_spec -= 10.0 * np.log10(np.maximum(self.amin, ref_value))\n\n        if self.top_db is not None:\n            if self.top_db < 0:\n                raise ParameterError('top_db must be non-negative')\n            log_spec = torch.clamp(log_spec, min=log_spec.max().item() - self.top_db, max=np.inf)\n\n        return log_spec","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class DropStripes(nn.Module):\n    def __init__(self, dim, drop_width, stripes_num):\n        \"\"\"Drop stripes. \n        Args:\n          dim: int, dimension along which to drop\n          drop_width: int, maximum width of stripes to drop\n          stripes_num: int, how many stripes to drop\n        \"\"\"\n        super(DropStripes, self).__init__()\n\n        assert dim in [2, 3]    # dim 2: time; dim 3: frequency\n\n        self.dim = dim\n        self.drop_width = drop_width\n        self.stripes_num = stripes_num\n\n    def forward(self, input):\n        \"\"\"input: (batch_size, channels, time_steps, freq_bins)\"\"\"\n\n        assert input.ndimension() == 4\n\n        if self.training is False:\n            return input\n\n        else:\n            batch_size = input.shape[0]\n            total_width = input.shape[self.dim]\n\n            for n in range(batch_size):\n                self.transform_slice(input[n], total_width)\n\n            return input\n\n\n    def transform_slice(self, e, total_width):\n        \"\"\"e: (channels, time_steps, freq_bins)\"\"\"\n\n        for _ in range(self.stripes_num):\n            distance = torch.randint(low=0, high=self.drop_width, size=(1,))[0]\n            bgn = torch.randint(low=0, high=total_width - distance, size=(1,))[0]\n\n            if self.dim == 2:\n                e[:, bgn : bgn + distance, :] = 0\n            elif self.dim == 3:\n                e[:, :, bgn : bgn + distance] = 0\n\n\nclass SpecAugmentation(nn.Module):\n    def __init__(self, time_drop_width, time_stripes_num, freq_drop_width, \n        freq_stripes_num):\n        \"\"\"Spec augmetation. \n        [ref] Park, D.S., Chan, W., Zhang, Y., Chiu, C.C., Zoph, B., Cubuk, E.D. \n        and Le, Q.V., 2019. Specaugment: A simple data augmentation method \n        for automatic speech recognition. arXiv preprint arXiv:1904.08779.\n        Args:\n          time_drop_width: int\n          time_stripes_num: int\n          freq_drop_width: int\n          freq_stripes_num: int\n        \"\"\"\n\n        super(SpecAugmentation, self).__init__()\n\n        self.time_dropper = DropStripes(dim=2, drop_width=time_drop_width, \n            stripes_num=time_stripes_num)\n\n        self.freq_dropper = DropStripes(dim=3, drop_width=freq_drop_width, \n            stripes_num=freq_stripes_num)\n\n    def forward(self, input):\n        x = self.time_dropper(input)\n        x = self.freq_dropper(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### audioset_tagging_cnn\n\nI also use `Cnn14_DecisionLevelAtt` model from [PANNs models](https://github.com/qiuqiangkong/audioset_tagging_cnn/blob/master/pytorch/models.py), which is a SED model.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"#### LICENSE\n\n```\nThe MIT License\n  \nCopyright (c) 2010-2017 Google, Inc. http://angularjs.org\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE.\n```","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Building blocks","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n\n\ndef interpolate(x: torch.Tensor, ratio: int):\n    \"\"\"Interpolate data in time domain. This is used to compensate the\n    resolution reduction in downsampling of a CNN.\n\n    Args:\n      x: (batch_size, time_steps, classes_num)\n      ratio: int, ratio to interpolate\n    Returns:\n      upsampled: (batch_size, time_steps * ratio, classes_num)\n    \"\"\"\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):\n    \"\"\"Pad framewise_output to the same length as input frames. The pad value\n    is the same as the value of the last frame.\n    Args:\n      framewise_output: (batch_size, frames_num, classes_num)\n      frames_num: int, number of frames to pad\n    Outputs:\n      output: (batch_size, frames_num, classes_num)\n    \"\"\"\n    pad = framewise_output[:, -1:, :].repeat(\n        1, frames_num - framewise_output.shape[1], 1)\n    \"\"\"tensor for padding\"\"\"\n\n    output = torch.cat((framewise_output, pad), dim=1)\n    \"\"\"(batch_size, frames_num, classes_num)\"\"\"\n\n    return output\n\n\nclass ConvBlock(nn.Module):\n    def __init__(self, in_channels: int, out_channels: int):\n        super().__init__()\n\n        self.conv1 = nn.Conv2d(\n            in_channels=in_channels,\n            out_channels=out_channels,\n            kernel_size=(3, 3),\n            stride=(1, 1),\n            padding=(1, 1),\n            bias=False)\n\n        self.conv2 = nn.Conv2d(\n            in_channels=out_channels,\n            out_channels=out_channels,\n            kernel_size=(3, 3),\n            stride=(1, 1),\n            padding=(1, 1),\n            bias=False)\n\n        self.bn1 = nn.BatchNorm2d(out_channels)\n        self.bn2 = nn.BatchNorm2d(out_channels)\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_layer(self.conv1)\n        init_layer(self.conv2)\n        init_bn(self.bn1)\n        init_bn(self.bn2)\n\n    def forward(self, input, pool_size=(2, 2), pool_type='avg'):\n\n        x = input\n        x = F.relu_(self.bn1(self.conv1(x)))\n        x = F.relu_(self.bn2(self.conv2(x)))\n        if pool_type == 'max':\n            x = F.max_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg':\n            x = F.avg_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg+max':\n            x1 = F.avg_pool2d(x, kernel_size=pool_size)\n            x2 = F.max_pool2d(x, kernel_size=pool_size)\n            x = x1 + x2\n        else:\n            raise Exception('Incorrect argument!')\n\n        return x\n\n\nclass AttBlock(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"linear\",\n                 temperature=1.0):\n        super().__init__()\n\n        self.activation = activation\n        self.temperature = temperature\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.bn_att = nn.BatchNorm1d(out_features)\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n        init_bn(self.bn_att)\n\n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n#         norm_att = torch.softmax(torch.clamp(self.att(x), -10, 10), dim=-1)\n        norm_att = 10*torch.tanh(self.att(x))\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)\n        \n\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Mixup(object):\n    def __init__(self, mixup_alpha, random_seed=1234):\n        \"\"\"Mixup coefficient generator.\n        \"\"\"\n        self.mixup_alpha = mixup_alpha\n        self.random_state = np.random.RandomState(random_seed)\n\n    def get_lambda(self, batch_size):\n        \"\"\"Get mixup random coefficients.\n        Args:\n          batch_size: int\n        Returns:\n          mixup_lambdas: (batch_size,)\n        \"\"\"\n        mixup_lambdas = []\n        for n in range(0, batch_size, 2):\n            lam = self.random_state.beta(self.mixup_alpha, self.mixup_alpha, 1)[0]\n            mixup_lambdas.append(lam)\n            mixup_lambdas.append(1. - lam)\n\n        return np.array(mixup_lambdas)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PANNsCNN14Att(nn.Module):\n    def __init__(self, sample_rate: int, window_size: int, hop_size: int,\n                 mel_bins: int, fmin: int, fmax: int, classes_num: int):\n        super().__init__()\n\n        window = 'hann'\n        center = True\n        pad_mode = 'reflect'\n        ref = 1.0\n        amin = 1e-10\n        top_db = None\n        self.interpolate_ratio = 32  # Downsampled ratio\n\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(\n            n_fft=window_size,\n            hop_length=hop_size,\n            win_length=window_size,\n            window=window,\n            center=center,\n            pad_mode=pad_mode,\n            freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(\n            sr=sample_rate,\n            n_fft=window_size,\n            n_mels=mel_bins,\n            fmin=fmin,\n            fmax=fmax,\n            ref=ref,\n            amin=amin,\n            top_db=top_db,\n            freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(\n            time_drop_width=64,\n            time_stripes_num=2,\n            freq_drop_width=8,\n            freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(mel_bins)\n\n        self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)\n        self.conv_block2 = ConvBlock(in_channels=64, out_channels=128)\n        self.conv_block3 = ConvBlock(in_channels=128, out_channels=256)\n        self.conv_block4 = ConvBlock(in_channels=256, out_channels=512)\n        self.conv_block5 = ConvBlock(in_channels=512, out_channels=1024)\n        self.conv_block6 = ConvBlock(in_channels=1024, out_channels=2048)\n\n        self.fc1 = nn.Linear(2048, 2048, bias=True)\n        self.att_block = AttBlock(2048, classes_num, activation='sigmoid')\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n        \n    def cnn_feature_extractor(self, x):\n        x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block2(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block3(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block4(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block5(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block6(x, pool_size=(1, 1), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        return x\n    \n    def preprocess(self, input, mixup_lambda=None):\n        # t1 = time.time()\n        x = self.spectrogram_extractor(input)  # (batch_size, 1, time_steps, freq_bins)\n        x = self.logmel_extractor(x)  # (batch_size, 1, time_steps, mel_bins)\n\n        frames_num = x.shape[2]\n\n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n\n        if self.training:\n            x = self.spec_augmenter(x)\n\n        # Mixup on spectrogram\n        if self.training and mixup_lambda is not None:\n            x = do_mixup(x, mixup_lambda)\n        return x, frames_num\n        \n\n    def forward(self, input, mixup_lambda=None):\n        \"\"\"\n        Input: (batch_size, data_length)\"\"\"\n        x, frames_num = self.preprocess(input, mixup_lambda=mixup_lambda)\n\n        # Output shape (batch size, channels, time, frequency)\n        x = self.cnn_feature_extractor(x)\n        \n        # Aggregate in frequency axis\n        x = torch.mean(x, dim=3)\n\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.5, training=self.training)\n\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        # Get framewise output\n        framewise_output = interpolate(segmentwise_output,\n                                       self.interpolate_ratio)\n        framewise_output = pad_framewise_output(framewise_output, frames_num)\n\n        output_dict = {\n            'framewise_output': framewise_output,\n            'clipwise_output': clipwise_output\n        }\n\n        return output_dict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"What is good in PANNs models is that they accept raw audio clip as input. Let's put a chunk into the CNN feature extractor of the model above.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"SR = 32000\n\ny, _ = librosa.load(TRAIN_RESAMPLED_AUDIO_DIRS[0] / \"aldfly\" / \"XC134874.wav\",\n                    sr=SR,\n                    res_type=\"kaiser_fast\",\n                    mono=True)\n\nAudio(y, rate=SR)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display.waveplot(y, sr=SR);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_config = {\n    \"sample_rate\": 32000,\n    \"window_size\": 1024,\n    \"hop_size\": 320,\n    \"mel_bins\": 64,\n    \"fmin\": 50,\n    \"fmax\": 14000,\n    \"classes_num\": 264\n}\n\nmodel = PANNsCNN14Att(**model_config)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"In `PANNsCNN14Att`, input raw waveform will be converted into log-melspectrogram using `torchlibrosa`'s utilities. I put this functionality in `PANNsCNN14Att.preprocess()` method. Let's check the output.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"chunk = torch.from_numpy(y[:SR * 5]).unsqueeze(0)\nmelspec, _ = model.preprocess(chunk)\nmelspec.size()\nmelspec_numpy = melspec.detach().numpy()[0, 0].transpose(1, 0)\ndisplay.specshow(melspec_numpy, sr=SR, y_axis=\"mel\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"`PANNsCNN14Att.cnn_feature_extractor()` method will take this as input and output feature map. Let's check the output of the feature extractor.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_map = model.cnn_feature_extractor(melspec)\nfeature_map.size()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Although it's downsized through several convolution and pooling layers, the size of it's third dimension is 15 and it still contains time information. Each element of this dimension is *segment*. In SED model, we provide prediction for each of this.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Dataset","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"BIRD_CODE = {\n    'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,\n    'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,\n    'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,\n    'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,\n    'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,\n    'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,\n    'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,\n    'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,\n    'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,\n    'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,\n    'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,\n    'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,\n    'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,\n    'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,\n    'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,\n    'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,\n    'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,\n    'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,\n    'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,\n    'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,\n    'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,\n    'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,\n    'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,\n    'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,\n    'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,\n    'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,\n    'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,\n    'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,\n    'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,\n    'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,\n    'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,\n    'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,\n    'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,\n    'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,\n    'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,\n    'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,\n    'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,\n    'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,\n    'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,\n    'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,\n    'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,\n    'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,\n    'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,\n    'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,\n    'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,\n    'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,\n    'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,\n    'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,\n    'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,\n    'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,\n    'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,\n    'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,\n    'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263\n}\n\nINV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items()}","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"PERIOD = 5\n\nclass PANNsDataset(data.Dataset):\n    def __init__(\n            self,\n            file_list: List[List[str]],\n            waveform_transforms=None):\n        self.file_list = file_list  # list of list: [file_path, ebird_code]\n        self.waveform_transforms = waveform_transforms\n\n    def __len__(self):\n        return len(self.file_list)\n\n    def __getitem__(self, idx: int):\n        wav_path, ebird_code = self.file_list[idx]\n\n        y, sr = sf.read(wav_path)\n\n        if self.waveform_transforms:\n            y = self.waveform_transforms(y)\n        else:\n            len_y = len(y)\n            effective_length = sr * PERIOD\n            if len_y < effective_length:\n                new_y = np.zeros(effective_length, dtype=y.dtype)\n                start = np.random.randint(effective_length - len_y)\n                new_y[start:start + len_y] = y\n                y = new_y.astype(np.float32)\n            elif len_y > effective_length:\n                start = np.random.randint(len_y - effective_length)\n                y = y[start:start + effective_length].astype(np.float32)\n            else:\n                y = y.astype(np.float32)\n\n        labels = np.zeros(len(BIRD_CODE), dtype=\"f\")\n        labels[BIRD_CODE[ebird_code]] = 1\n\n        return {\"waveform\": y, \"targets\": labels}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train\n\nSome code are taken from https://www.kaggle.com/ttahara/training-birdsong-baseline-resnest50-fast .\nThanks @ttahara!","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp_list = []\nfor audio_d in TRAIN_RESAMPLED_AUDIO_DIRS:\n    if not audio_d.exists():\n        continue\n    for ebird_d in audio_d.iterdir():\n        if ebird_d.is_file():\n            continue\n        for wav_f in ebird_d.iterdir():\n            tmp_list.append([ebird_d.name, wav_f.name, wav_f.as_posix()])\n            \ntrain_wav_path_exist = pd.DataFrame(\n    tmp_list, columns=[\"ebird_code\", \"resampled_filename\", \"file_path\"])\n\ndel tmp_list\n\ntrain_all = pd.merge(\n    train, train_wav_path_exist, on=[\"ebird_code\", \"resampled_filename\"], how=\"inner\")\n\nprint(train.shape)\nprint(train_wav_path_exist.shape)\nprint(train_all.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n\ntrain_all[\"fold\"] = -1\nfor fold_id, (train_index, val_index) in enumerate(skf.split(train_all, train_all[\"ebird_code\"])):\n    train_all.iloc[val_index, -1] = fold_id\n    \n# # check the propotion\nfold_proportion = pd.pivot_table(train_all, index=\"ebird_code\", columns=\"fold\", values=\"xc_id\", aggfunc=len)\nprint(fold_proportion.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"use_fold = 0\ntrain_file_list = train_all.query(\"fold != @use_fold\")[[\"file_path\", \"ebird_code\"]].values.tolist()\nval_file_list = train_all.query(\"fold == @use_fold\")[[\"file_path\", \"ebird_code\"]].values.tolist()\n\nprint(\"[fold {}] train: {}, val: {}\".format(use_fold, len(train_file_list), len(val_file_list)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda:0\")\n# loaders\nloaders = {\n    \"train\": data.DataLoader(PANNsDataset(train_file_list, None), \n                             batch_size=64, \n                             shuffle=True, \n                             num_workers=2, \n                             pin_memory=True, \n                             drop_last=True),\n    \"valid\": data.DataLoader(PANNsDataset(val_file_list, None), \n                             batch_size=64, \n                             shuffle=False,\n                             num_workers=2,\n                             pin_memory=True,\n                             drop_last=False)\n}\n\n# model\nmodel_config[\"classes_num\"] = 527\n# model = PANNsResNet38Att(**model_config) \nmodel = PANNsCNN14Att(**model_config)\nweights = torch.load(\"../input/pannscnn14-decisionlevelatt-weight/Cnn14_DecisionLevelAtt_mAP0.425.pth\")\nmodel.load_state_dict(weights['model'])\nmodel.att_block = AttBlock(2048, 264, activation='sigmoid')\nmodel.att_block.init_weights()\nmodel.to(device)\n\n# weights=torch.load(\"../input/birdcallresnet38atl/fold0/checkpoints/best.pth\")\n# model.load_state_dict(weights['model_state_dict'])\nparam_optimizer = list(model.named_parameters())\nno_decay = [\"bias\", \"LayerNorm.bias\", \"LayerNorm.weight\"]\noptimizer_parameters = [\n            {\n                \"params\": [\n                    p for n, p in param_optimizer if not any(nd in n for nd in no_decay)\n                ],\n                \"weight_decay\": 0.001,\n            },\n            {\n                \"params\": [\n                    p for n, p in param_optimizer if any(nd in n for nd in no_decay)\n                ],\n                \"weight_decay\": 0.0,\n            },\n        ]\n# Optimizer\noptimizer = optim.AdamW(optimizer_parameters, lr=0.0001)\n\n# Scheduler\nscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min',\n                                                 verbose=False,\n                                                 threshold=0.0001, \n                                                 min_lr=0.00001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter:\n    def __init__(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\ndef do_mixup(x, mixup_lambda):\n    out = (x[0 :: 2].transpose(0, -1) * mixup_lambda[0 :: 2] + \\\n        x[1 :: 2].transpose(0, -1) * mixup_lambda[1 :: 2]).transpose(0, -1)\n    return out.float()   \n\nclass PANNsLoss(nn.Module):\n    def __init__(self):\n        super().__init__()\n\n        self.bce = nn.BCELoss()\n\n    def forward(self, input, target):\n        input_ = input[\"clipwise_output\"]\n        input_ = torch.where(torch.isnan(input_),\n                             torch.zeros_like(input_),\n                             input_)\n        input_ = torch.where(torch.isinf(input_),\n                             torch.zeros_like(input_),\n                             input_)\n\n        target = target['targets']\n\n        return self.bce(input_, target)\n    \ndef clip_nll(output_dict, target_dict):\n    loss = - torch.mean(target_dict['targets'] * output_dict['clipwise_output'])\n    return loss\n\nclass EarlyStopping:\n    def __init__(self, patience=7, mode=\"max\", delta=0.0001, tpu=False):\n        self.patience = patience\n        self.counter = 0\n        self.mode = mode\n        self.best_score = None\n        self.early_stop = False\n        self.tpu = tpu\n        self.delta = delta\n        if self.mode == \"min\":\n            self.val_score = np.Inf\n        else:\n            self.val_score = -np.Inf\n\n       \n\n    def __call__(self, epoch_score, model, model_path):\n        if self.mode == \"min\":\n            score = -1.0 * epoch_score\n        else:\n            score = np.copy(epoch_score)\n\n        if self.best_score is None:\n            self.best_score = score\n            self.save_checkpoint(epoch_score, model, model_path)\n        elif score < self.best_score + self.delta:\n            self.counter += 1\n            if self.tpu:\n                xm.master_print(\n                    \"EarlyStopping counter: {} out of {}\".format(\n                        self.counter, self.patience\n                    )\n                )\n            else:\n                print(\n                    \"EarlyStopping counter: {} out of {}\".format(\n                        self.counter, self.patience\n                    )\n                )\n            if self.counter >= self.patience:\n                self.early_stop = True\n        else:\n            self.best_score = score\n            self.save_checkpoint(epoch_score, model, model_path)\n            self.counter = 0\n\n    def save_checkpoint(self, epoch_score, model, model_path):\n        if epoch_score not in [-np.inf, np.inf, -np.nan, np.nan]:\n            if self.tpu:\n                xm.master_print(\n                    \"Validation score improved ({} --> {}). Saving model!\".format(\n                        self.val_score, epoch_score\n                    )\n                )\n            else:\n                print(\n                    \"Validation score improved ({} --> {}). Saving model!\".format(\n                        self.val_score, epoch_score\n                    )\n                )\n            if self.tpu:\n                xm.save(model.state_dict(), model_path)\n            else:\n                torch.save(model.state_dict(), model_path)\n        self.val_score = epoch_score\n        \n# from sklearn.metrics import f1_score   \n# def f1_score_func(output_dict, target_dict):\n#     y_pred = output_dict['clipwise_output'].detach().cpu().numpy().argmax(axis=1)\n#     y_true = target_dict['targets'].detach().cpu().numpy().argmax(axis=1)\n#     return f1_score(y_true, y_pred, average=\"macro\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm.notebook import tqdm\n\n\npanns_loss = PANNsLoss()\n\ndef model_fn(batch_data_dict, device, model, mixup=True):\n        if mixup:\n            batch_output_dict = model(batch_data_dict['waveform'], \n                batch_data_dict['mixup_lambda'].float())\n\n            batch_target_dict = {'targets': do_mixup(batch_data_dict['targets'], \n                batch_data_dict['mixup_lambda'].float())}\n        else:\n            batch_output_dict = model(batch_data_dict['waveform'], None)\n            batch_target_dict = {'targets': batch_data_dict['targets']}\n        # loss\n        nll_loss = 10*clip_nll(batch_output_dict, batch_target_dict)\n        pan_loss = panns_loss(batch_output_dict, batch_target_dict)\n        f1_score_loss = f1_score_func(batch_output_dict, batch_target_dict)\n        return nll_loss,pan_loss,f1_score_loss\nclass Engine:\n    def __init__(\n        self,\n        model,\n        optimizer,\n        device,\n        scheduler=None,\n        accumulation_steps=1,\n        use_tpu=False,\n        tpu_print=10,\n        fp16=False,\n        model_fn=None,\n        use_mean_loss=False,\n    ):\n        self.model = model\n        self.optimizer = optimizer\n        self.device = device\n        self.scheduler = scheduler\n        self.accumulation_steps = accumulation_steps\n        self.use_tpu = use_tpu\n        self.tpu_print = tpu_print\n        self.model_fn = model_fn\n        self.fp16 = fp16\n        self.scaler = None\n        self.mixup = True\n        if self.fp16:\n            self.scaler = amp.GradScaler()\n        self.model = self.model.to(self.device)\n\n    def train(self, data_loader):\n        losses = AverageMeter()\n        self.model.train()\n        print_idx = int(len(data_loader) * self.tpu_print / 100)\n        augmentation = ['mixup']\n        if self.mixup:\n            mixup_augmenter = Mixup(mixup_alpha=1.)\n            \n        if self.accumulation_steps > 1:\n            self.optimizer.zero_grad()\n        \n        tk0 = tqdm(data_loader, total=len(data_loader))\n\n        for b_idx, batch_data_dict in enumerate(tk0):\n            \n            if self.mixup:\n                batch_data_dict['mixup_lambda'] = torch.tensor(mixup_augmenter.get_lambda(len(batch_data_dict['waveform'])))\n            for key, value in batch_data_dict.items():\n                    batch_data_dict[key] = value.to(self.device)\n            if self.accumulation_steps == 1 and b_idx == 0:\n                self.optimizer.zero_grad()\n\n            if self.model_fn is None:\n                _, loss = self.model(**batch_data_dict)\n            else:\n                if self.fp16:\n                    with amp.autocast():\n                        nll_loss,pan_loss,f1_score_loss = self.model_fn(batch_data_dict, self.device, self.model)\n                else:\n                    nll_loss,pan_loss,f1_score_loss = self.model_fn(batch_data_dict, self.device, self.model)\n                    loss = nll_loss \n\n            if not self.use_tpu:\n                with torch.set_grad_enabled(True):\n                    \n                    if self.fp16:\n                        self.scaler.scale(loss).backward()\n                    else:\n                        loss.backward()\n\n                    if (b_idx + 1) % self.accumulation_steps == 0:\n                        if self.fp16:\n                            self.scaler.step(self.optimizer)\n                        else:\n                            self.optimizer.step()\n\n                        if self.scheduler is not None:\n                            self.scheduler.step(metrics=losses.avg)\n\n                        if self.fp16:\n                            self.scaler.update()\n\n                        if b_idx > 0:\n                            self.optimizer.zero_grad()\n\n            \n            losses.update(loss.item(), data_loader.batch_size)\n            tk0.set_postfix(loss=losses.avg)\n            \n        tk0.close()\n        return losses.avg\n\n    def evaluate(self, data_loader, return_predictions=False):\n        losses = AverageMeter()\n        f1_score_losses =  AverageMeter()\n        print_idx = int(len(data_loader) * self.tpu_print / 100)\n        self.model.eval()\n        final_predictions = []\n        with torch.no_grad():\n            tk0 = tqdm(data_loader, total=len(data_loader))\n            for b_idx, batch_data_dict in enumerate(tk0):\n                for key, value in batch_data_dict.items():\n                    batch_data_dict[key] = value.to(self.device)\n                if self.fp16:\n                    with amp.autocast():\n                        output_dict = self.model(batch_data_dict['waveform'], None)\n                        target_dict = {'targets': batch_data_dict['targets']}\n                        nll_loss = clip_nll(output_dict, target_dict)\n                        pan_loss = panns_loss(output_dict, target_dict)\n                        f1_score_loss = f1_score_func(output_dict, target_dict)\n                        loss = nll_loss \n                else:\n                        output_dict = self.model(batch_data_dict['waveform'], None)\n                        target_dict = {'targets': batch_data_dict['targets']}\n                        nll_loss = clip_nll(output_dict, target_dict)\n                        pan_loss = panns_loss(output_dict, target_dict)\n                        f1_score_loss = f1_score_func(output_dict, target_dict)\n                        loss = nll_loss \n                        \n                if return_predictions:\n                    final_predictions.append(output_dict)\n                    \n                f1_score_losses.update(f1_score_loss.item(), data_loader.batch_size)\n                losses.update(loss.item(), data_loader.batch_size)\n                tk0.set_postfix(loss=losses.avg)\n            tk0.close()\n        if return_predictions:\n            return losses.avg,f1_score_losses, final_predictions\n        else:\n            return losses.avg, f1_score_losses.avg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ntrain_engine = Engine(model=model,\n                      optimizer=optimizer,\n                      device=device,\n                      scheduler=scheduler,\n                      model_fn=model_fn,\n                      accumulation_steps=4)\nEPOCHS = 1\nes = EarlyStopping(patience=5, mode=\"max\", tpu=False)\nfor epoch in range(EPOCHS):\n        train_loss = train_engine.train(loaders['train'])\n        valid_loss, f1_loss = train_engine.evaluate(loaders['valid'])\n        print(f\"Epoch = {epoch}, LOSS = {valid_loss}, F1 Score: {f1_loss}\")\n#         scheduler.step(valid_loss)\n        es(f1_loss, model, model_path=f\"model_fold_{use_fold}.bin\")\n        if es.early_stop:\n            print(\"Early stopping\")\n            break\n        gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Seems it's learning something.\n\nNow I'll show how this model works in the inference phase. I'll use trained model of this which I trained by myself using the data of this competition in my local environment.\n\nSince [several concerns](https://www.kaggle.com/c/birdsong-recognition/discussion/172356) are expressed about over-sharing of top solutions during competition, and since I do respect those people who have worked hard to improve their scores, I would not make trained weight in common and would not share how I trained this model.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Prediction with SED model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model_config = {\n    \"sample_rate\": 32000,\n    \"window_size\": 1024,\n    \"hop_size\": 320,\n    \"mel_bins\": 64,\n    \"fmin\": 50,\n    \"fmax\": 14000,\n    \"classes_num\": 264\n}\n\nweights_path = \"./model_fold_0.bin\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prediction_for_clip(test_df: pd.DataFrame,\n                        clip: np.ndarray, \n                        model: PANNsResNet38Att,\n                        threshold=0.5):\n    PERIOD = 30\n    audios = []\n    y = clip.astype(np.float32)\n    len_y = len(y)\n    start = 0\n    end = PERIOD * SR\n    while True:\n        y_batch = y[start:end].astype(np.float32)\n        if len(y_batch) != PERIOD * SR:\n            y_pad = np.zeros(PERIOD * SR, dtype=np.float32)\n            y_pad[:len(y_batch)] = y_batch\n            audios.append(y_pad)\n            break\n        start = end\n        end += PERIOD * SR\n        audios.append(y_batch)\n        \n    array = np.asarray(audios)\n    tensors = torch.from_numpy(array)\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    \n    model.eval()\n    estimated_event_list = []\n    global_time = 0.0\n    site = test_df[\"site\"].values[0]\n    audio_id = test_df[\"audio_id\"].values[0]\n    for image in progress_bar(tensors):\n        image = image.view(1, image.size(0))\n        image = image.to(device)\n\n        with torch.no_grad():\n            prediction = model(image)\n            framewise_outputs = prediction[\"framewise_output\"].detach(\n                ).cpu().numpy()[0]\n                \n        thresholded = framewise_outputs >= threshold\n\n        for target_idx in range(thresholded.shape[1]):\n            if thresholded[:, target_idx].mean() == 0:\n                pass\n            else:\n                detected = np.argwhere(thresholded[:, target_idx]).reshape(-1)\n                head_idx = 0\n                tail_idx = 0\n                while True:\n                    if (tail_idx + 1 == len(detected)) or (\n                            detected[tail_idx + 1] - \n                            detected[tail_idx] != 1):\n                        onset = 0.01 * detected[\n                            head_idx] + global_time\n                        offset = 0.01 * detected[\n                            tail_idx] + global_time\n                        onset_idx = detected[head_idx]\n                        offset_idx = detected[tail_idx]\n                        max_confidence = framewise_outputs[\n                            onset_idx:offset_idx, target_idx].max()\n                        mean_confidence = framewise_outputs[\n                            onset_idx:offset_idx, target_idx].mean()\n                        estimated_event = {\n                            \"site\": site,\n                            \"audio_id\": audio_id,\n                            \"ebird_code\": INV_BIRD_CODE[target_idx],\n                            \"onset\": onset,\n                            \"offset\": offset,\n                            \"max_confidence\": max_confidence,\n                            \"mean_confidence\": mean_confidence\n                        }\n                        estimated_event_list.append(estimated_event)\n                        head_idx = tail_idx + 1\n                        tail_idx = tail_idx + 1\n                        if head_idx >= len(detected):\n                            break\n                    else:\n                        tail_idx += 1\n        global_time += PERIOD\n        \n    prediction_df = pd.DataFrame(estimated_event_list)\n    return prediction_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prediction(test_df: pd.DataFrame,\n               test_audio: Path,\n               model_config: dict,\n               weights_path: str,\n               threshold=0.5):\n    model = train_engine.model\n    unique_audio_id = test_df.audio_id.unique()\n\n    warnings.filterwarnings(\"ignore\")\n    prediction_dfs = []\n    for audio_id in unique_audio_id:\n        with timer(f\"Loading {audio_id}\"):\n            clip, _ = librosa.load(test_audio / (audio_id + \".mp3\"),\n                                   sr=SR,\n                                   mono=True,\n                                   res_type=\"kaiser_fast\")\n        \n        test_df_for_audio_id = test_df.query(\n            f\"audio_id == '{audio_id}'\").reset_index(drop=True)\n        with timer(f\"Prediction on {audio_id}\"):\n            prediction_df = prediction_for_clip(test_df_for_audio_id,\n                                                clip=clip,\n                                                model=model,\n                                                threshold=threshold)\n\n        prediction_dfs.append(prediction_df)\n    \n    prediction_df = pd.concat(prediction_dfs, axis=0, sort=False).reset_index(drop=True)\n    return prediction_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction_df = prediction(test_df=test,\n                           test_audio=TEST_AUDIO_DIR,\n                           model_config=model_config,\n                           weights_path=weights_path,\n                           threshold=0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Postprocess","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = {}\n\nfor audio_id, sub_df in prediction_df.groupby(\"audio_id\"):\n    events = sub_df[[\"ebird_code\", \"onset\", \"offset\", \"max_confidence\", \"site\"]].values\n    n_events = len(events)\n    removed_event = []\n    for i in range(n_events):\n        for j in range(n_events):\n            if i == j:\n                continue\n            if i in removed_event:\n                continue\n            if j in removed_event:\n                continue\n            \n            event_i = events[i]\n            event_j = events[j]\n            \n            if (event_i[1] - event_j[2] >= 0) or (event_j[1] - event_i[2] >= 0):\n                pass\n            else:\n                later_onset = max(event_i[1], event_j[1])\n                sooner_onset = min(event_i[1], event_j[1])\n                sooner_offset = min(event_i[2], event_j[2])\n                later_offset = max(event_i[2], event_j[2])\n\n                intersection = sooner_offset - later_onset\n                union = later_offset - sooner_onset\n                \n                iou = intersection / union\n                if iou > 0.4:\n                    if event_i[3] > event_j[3]:\n                        removed_event.append(j)\n                    else:\n                        removed_event.append(i)\n\n    site = events[0][4]\n    for i in range(n_events):\n        if i in removed_event:\n            continue\n        event = events[i][0]\n        onset = events[i][1]\n        offset = events[i][2]\n        if site in {\"site_1\", \"site_2\"}:\n            start_section = int((onset // 5) * 5) + 5\n            end_section = int((offset // 5) * 5) + 5\n            cur_section = start_section\n\n            row_id = f\"{site}_{audio_id}_{start_section}\"\n            if labels.get(row_id) is not None:\n                labels[row_id].add(event)\n            else:\n                labels[row_id] = set()\n                labels[row_id].add(event)\n\n            while cur_section != end_section:\n                cur_section += 5\n                row_id = f\"{site}_{audio_id}_{cur_section}\"\n                if labels.get(row_id) is not None:\n                    labels[row_id].add(event)\n                else:\n                    labels[row_id] = set()\n                    labels[row_id].add(event)\n        else:\n            row_id = f\"{site}_{audio_id}\"\n            if labels.get(row_id) is not None:\n                labels[row_id].add(event)\n            else:\n                labels[row_id] = set()\n                labels[row_id].add(event)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for key in labels:\n    labels[key] = \" \".join(sorted(list(labels[key])))\n    \n    \nrow_ids = list(labels.keys())\nbirds = list(labels.values())\npost_processed = pd.DataFrame({\n    \"row_id\": row_ids,\n    \"birds\": birds\n})\npost_processed.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_row_id = test[[\"row_id\"]]\nsubmission = all_row_id.merge(post_processed, on=\"row_id\", how=\"left\")\nsubmission = submission.fillna(\"nocall\")\nsubmission.to_csv(\"submission.csv\", index=False)\nsubmission.head(20)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## EOF","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.birds.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}