{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":19596,"databundleVersionId":1292430,"sourceType":"competition"},{"sourceId":1262046,"sourceType":"datasetVersion","datasetId":726424},{"sourceId":1305829,"sourceType":"datasetVersion","datasetId":755946},{"sourceId":1306285,"sourceType":"datasetVersion","datasetId":756248},{"sourceId":1306896,"sourceType":"datasetVersion","datasetId":756642},{"sourceId":1307127,"sourceType":"datasetVersion","datasetId":756745},{"sourceId":1307546,"sourceType":"datasetVersion","datasetId":756953},{"sourceId":1346120,"sourceType":"datasetVersion","datasetId":782938},{"sourceId":1405119,"sourceType":"datasetVersion","datasetId":821504}],"dockerImageVersionId":29987,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import cv2\nimport audioread\nimport logging\nimport os\nimport random\nimport time\nimport warnings\n\nimport librosa\nimport librosa.display as display\nimport numpy as np\nimport pandas as pd\nimport soundfile as sf\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.utils.data as data\n\nfrom contextlib import contextmanager\nfrom IPython.display import Audio\nfrom pathlib import Path\nfrom typing import Optional, List\n\nfrom catalyst.dl import SupervisedRunner, State, CallbackOrder, Callback, CheckpointCallback\nfrom fastprogress import progress_bar\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score, average_precision_score","metadata":{"_kg_hide-input":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T16:40:52.548209Z","iopub.execute_input":"2025-01-16T16:40:52.548625Z","iopub.status.idle":"2025-01-16T16:40:52.559382Z","shell.execute_reply.started":"2025-01-16T16:40:52.548582Z","shell.execute_reply":"2025-01-16T16:40:52.558229Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def set_seed(seed: int = 42):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)  # type: ignore\n    torch.backends.cudnn.deterministic = True  # type: ignore\n    torch.backends.cudnn.benchmark = True  # type: ignore\n    \n    \ndef get_logger(out_file=None):\n    logger = logging.getLogger()\n    formatter = logging.Formatter(\"%(asctime)s - %(levelname)s - %(message)s\")\n    logger.handlers = []\n    logger.setLevel(logging.INFO)\n\n    handler = logging.StreamHandler()\n    handler.setFormatter(formatter)\n    handler.setLevel(logging.INFO)\n    logger.addHandler(handler)\n\n    if out_file is not None:\n        fh = logging.FileHandler(out_file)\n        fh.setFormatter(formatter)\n        fh.setLevel(logging.INFO)\n        logger.addHandler(fh)\n    logger.info(\"logger set up\")\n    return logger\n    \n    \n@contextmanager\ndef timer(name: str, logger: Optional[logging.Logger] = None):\n    t0 = time.time()\n    msg = f\"[{name}] start\"\n    if logger is None:\n        print(msg)\n    else:\n        logger.info(msg)\n    yield\n\n    msg = f\"[{name}] done in {time.time() - t0:.2f} s\"\n    if logger is None:\n        print(msg)\n    else:\n        logger.info(msg)\n    \n    \nset_seed(1213)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-01-16T16:40:52.561000Z","iopub.execute_input":"2025-01-16T16:40:52.561258Z","iopub.status.idle":"2025-01-16T16:40:52.572262Z","shell.execute_reply.started":"2025-01-16T16:40:52.561234Z","shell.execute_reply":"2025-01-16T16:40:52.571574Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ROOT = Path.cwd().parent\nINPUT_ROOT = ROOT / \"input\"\nRAW_DATA = INPUT_ROOT / \"birdsong-recognition\"\nTRAIN_AUDIO_DIR = RAW_DATA / \"train_audio\"\nTRAIN_RESAMPLED_AUDIO_DIRS = [\n  INPUT_ROOT / \"birdsong-resampled-train-audio-{:0>2}\".format(i)  for i in range(5)\n]\nTEST_AUDIO_DIR = RAW_DATA / \"test_audio\"","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-01-16T16:40:52.573367Z","iopub.execute_input":"2025-01-16T16:40:52.573598Z","iopub.status.idle":"2025-01-16T16:40:52.577923Z","shell.execute_reply.started":"2025-01-16T16:40:52.573574Z","shell.execute_reply":"2025-01-16T16:40:52.577276Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(TRAIN_RESAMPLED_AUDIO_DIRS[0] / \"train_mod.csv\")\n\nif not TEST_AUDIO_DIR.exists():\n    TEST_AUDIO_DIR = INPUT_ROOT / \"birdcall-check\" / \"test_audio\"\n    test = pd.read_csv(INPUT_ROOT / \"birdcall-check\" / \"test.csv\")\nelse:\n    test = pd.read_csv(RAW_DATA / \"test.csv\")","metadata":{"_kg_hide-input":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T16:40:52.579288Z","iopub.execute_input":"2025-01-16T16:40:52.579525Z","iopub.status.idle":"2025-01-16T16:40:53.016087Z","shell.execute_reply.started":"2025-01-16T16:40:52.579502Z","shell.execute_reply":"2025-01-16T16:40:53.015276Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class DFTBase(nn.Module):\n    def __init__(self):\n        \"\"\"Base class for DFT and IDFT matrix\"\"\"\n        super(DFTBase, self).__init__()\n\n    def dft_matrix(self, n):\n        (x, y) = np.meshgrid(np.arange(n), np.arange(n))\n        omega = np.exp(-2 * np.pi * 1j / n)\n        W = np.power(omega, x * y)\n        return W\n\n    def idft_matrix(self, n):\n        (x, y) = np.meshgrid(np.arange(n), np.arange(n))\n        omega = np.exp(2 * np.pi * 1j / n)\n        W = np.power(omega, x * y)\n        return W\n    \n    \nclass STFT(DFTBase):\n    def __init__(self, n_fft=2048, hop_length=None, win_length=None, \n        window='hann', center=True, pad_mode='reflect', freeze_parameters=True):\n        \"\"\"Implementation of STFT with Conv1d. The function has the same output \n        of librosa.core.stft\n        \"\"\"\n        super(STFT, self).__init__()\n\n        assert pad_mode in ['constant', 'reflect']\n\n        self.n_fft = n_fft\n        self.center = center\n        self.pad_mode = pad_mode\n\n        # By default, use the entire frame\n        if win_length is None:\n            win_length = n_fft\n\n        # Set the default hop, if it's not already specified\n        if hop_length is None:\n            hop_length = int(win_length // 4)\n\n        fft_window = librosa.filters.get_window(window, win_length, fftbins=True)\n\n        # Pad the window out to n_fft size\n        fft_window = librosa.util.pad_center(fft_window, n_fft)\n\n        # DFT & IDFT matrix\n        self.W = self.dft_matrix(n_fft)\n\n        out_channels = n_fft // 2 + 1\n\n        self.conv_real = nn.Conv1d(in_channels=1, out_channels=out_channels, \n            kernel_size=n_fft, stride=hop_length, padding=0, dilation=1, \n            groups=1, bias=False)\n\n        self.conv_imag = nn.Conv1d(in_channels=1, out_channels=out_channels, \n            kernel_size=n_fft, stride=hop_length, padding=0, dilation=1, \n            groups=1, bias=False)\n\n        self.conv_real.weight.data = torch.Tensor(\n            np.real(self.W[:, 0 : out_channels] * fft_window[:, None]).T)[:, None, :]\n        # (n_fft // 2 + 1, 1, n_fft)\n\n        self.conv_imag.weight.data = torch.Tensor(\n            np.imag(self.W[:, 0 : out_channels] * fft_window[:, None]).T)[:, None, :]\n        # (n_fft // 2 + 1, 1, n_fft)\n\n        if freeze_parameters:\n            for param in self.parameters():\n                param.requires_grad = False\n\n    def forward(self, input):\n        \"\"\"input: (batch_size, data_length)\n        Returns:\n          real: (batch_size, n_fft // 2 + 1, time_steps)\n          imag: (batch_size, n_fft // 2 + 1, time_steps)\n        \"\"\"\n\n        x = input[:, None, :]   # (batch_size, channels_num, data_length)\n\n        if self.center:\n            x = F.pad(x, pad=(self.n_fft // 2, self.n_fft // 2), mode=self.pad_mode)\n\n        real = self.conv_real(x)\n        imag = self.conv_imag(x)\n        # (batch_size, n_fft // 2 + 1, time_steps)\n\n        real = real[:, None, :, :].transpose(2, 3)\n        imag = imag[:, None, :, :].transpose(2, 3)\n        # (batch_size, 1, time_steps, n_fft // 2 + 1)\n\n        return real, imag\n    \n    \nclass Spectrogram(nn.Module):\n    def __init__(self, n_fft=2048, hop_length=None, win_length=None, \n        window='hann', center=True, pad_mode='reflect', power=2.0, \n        freeze_parameters=True):\n        \"\"\"Calculate spectrogram using pytorch. The STFT is implemented with \n        Conv1d. The function has the same output of librosa.core.stft\n        \"\"\"\n        super(Spectrogram, self).__init__()\n\n        self.power = power\n\n        self.stft = STFT(n_fft=n_fft, hop_length=hop_length, \n            win_length=win_length, window=window, center=center, \n            pad_mode=pad_mode, freeze_parameters=True)\n\n    def forward(self, input):\n        \"\"\"input: (batch_size, 1, time_steps, n_fft // 2 + 1)\n        Returns:\n          spectrogram: (batch_size, 1, time_steps, n_fft // 2 + 1)\n        \"\"\"\n\n        (real, imag) = self.stft.forward(input)\n        # (batch_size, n_fft // 2 + 1, time_steps)\n\n        spectrogram = real ** 2 + imag ** 2\n\n        if self.power == 2.0:\n            pass\n        else:\n            spectrogram = spectrogram ** (power / 2.0)\n\n        return spectrogram\n\n    \nclass LogmelFilterBank(nn.Module):\n    def __init__(self, sr=32000, n_fft=2048, n_mels=64, fmin=50, fmax=14000, is_log=True, \n        ref=1.0, amin=1e-10, top_db=80.0, freeze_parameters=True):\n        \"\"\"Calculate logmel spectrogram using pytorch. The mel filter bank is \n        the pytorch implementation of as librosa.filters.mel \n        \"\"\"\n        super(LogmelFilterBank, self).__init__()\n\n        self.is_log = is_log\n        self.ref = ref\n        self.amin = amin\n        self.top_db = top_db\n\n        self.melW = librosa.filters.mel(sr=sr, n_fft=n_fft, n_mels=n_mels,\n            fmin=fmin, fmax=fmax).T\n        # (n_fft // 2 + 1, mel_bins)\n\n        self.melW = nn.Parameter(torch.Tensor(self.melW))\n\n        if freeze_parameters:\n            for param in self.parameters():\n                param.requires_grad = False\n\n    def forward(self, input):\n        \"\"\"input: (batch_size, channels, time_steps)\n        \n        Output: (batch_size, time_steps, mel_bins)\n        \"\"\"\n\n        # Mel spectrogram\n        mel_spectrogram = torch.matmul(input, self.melW)\n\n        # Logmel spectrogram\n        if self.is_log:\n            output = self.power_to_db(mel_spectrogram)\n        else:\n            output = mel_spectrogram\n\n        return output\n\n\n    def power_to_db(self, input):\n        \"\"\"Power to db, this function is the pytorch implementation of \n        librosa.core.power_to_lb\n        \"\"\"\n        ref_value = self.ref\n        log_spec = 10.0 * torch.log10(torch.clamp(input, min=self.amin, max=np.inf))\n        log_spec -= 10.0 * np.log10(np.maximum(self.amin, ref_value))\n\n        if self.top_db is not None:\n            if self.top_db < 0:\n                raise ParameterError('top_db must be non-negative')\n            log_spec = torch.clamp(log_spec, min=log_spec.max().item() - self.top_db, max=np.inf)\n\n        return log_spec","metadata":{"_kg_hide-input":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T16:40:53.018423Z","iopub.execute_input":"2025-01-16T16:40:53.018678Z","iopub.status.idle":"2025-01-16T16:40:53.049718Z","shell.execute_reply.started":"2025-01-16T16:40:53.018654Z","shell.execute_reply":"2025-01-16T16:40:53.048342Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class DropStripes(nn.Module):\n    def __init__(self, dim, drop_width, stripes_num):\n        \"\"\"Drop stripes. \n        Args:\n          dim: int, dimension along which to drop\n          drop_width: int, maximum width of stripes to drop\n          stripes_num: int, how many stripes to drop\n        \"\"\"\n        super(DropStripes, self).__init__()\n\n        assert dim in [2, 3]    # dim 2: time; dim 3: frequency\n\n        self.dim = dim\n        self.drop_width = drop_width\n        self.stripes_num = stripes_num\n\n    def forward(self, input):\n        \"\"\"input: (batch_size, channels, time_steps, freq_bins)\"\"\"\n\n        assert input.ndimension() == 4\n\n        if self.training is False:\n            return input\n\n        else:\n            batch_size = input.shape[0]\n            total_width = input.shape[self.dim]\n\n            for n in range(batch_size):\n                self.transform_slice(input[n], total_width)\n\n            return input\n\n\n    def transform_slice(self, e, total_width):\n        \"\"\"e: (channels, time_steps, freq_bins)\"\"\"\n\n        for _ in range(self.stripes_num):\n            distance = torch.randint(low=0, high=self.drop_width, size=(1,))[0]\n            bgn = torch.randint(low=0, high=total_width - distance, size=(1,))[0]\n\n            if self.dim == 2:\n                e[:, bgn : bgn + distance, :] = 0\n            elif self.dim == 3:\n                e[:, :, bgn : bgn + distance] = 0\n\n\nclass SpecAugmentation(nn.Module):\n    def __init__(self, time_drop_width, time_stripes_num, freq_drop_width, \n        freq_stripes_num):\n        \"\"\"Spec augmetation. \n        [ref] Park, D.S., Chan, W., Zhang, Y., Chiu, C.C., Zoph, B., Cubuk, E.D. \n        and Le, Q.V., 2019. Specaugment: A simple data augmentation method \n        for automatic speech recognition. arXiv preprint arXiv:1904.08779.\n        Args:\n          time_drop_width: int\n          time_stripes_num: int\n          freq_drop_width: int\n          freq_stripes_num: int\n        \"\"\"\n\n        super(SpecAugmentation, self).__init__()\n\n        self.time_dropper = DropStripes(dim=2, drop_width=time_drop_width, \n            stripes_num=time_stripes_num)\n\n        self.freq_dropper = DropStripes(dim=3, drop_width=freq_drop_width, \n            stripes_num=freq_stripes_num)\n\n    def forward(self, input):\n        x = self.time_dropper(input)\n        x = self.freq_dropper(x)\n        return x","metadata":{"_kg_hide-input":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T16:40:53.054318Z","iopub.execute_input":"2025-01-16T16:40:53.054746Z","iopub.status.idle":"2025-01-16T16:40:53.073652Z","shell.execute_reply.started":"2025-01-16T16:40:53.054703Z","shell.execute_reply":"2025-01-16T16:40:53.072674Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Building blocks","metadata":{}},{"cell_type":"code","source":"def init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n\n\ndef interpolate(x: torch.Tensor, ratio: int):\n    \"\"\"Interpolate data in time domain. This is used to compensate the\n    resolution reduction in downsampling of a CNN.\n\n    Args:\n      x: (batch_size, time_steps, classes_num)\n      ratio: int, ratio to interpolate\n    Returns:\n      upsampled: (batch_size, time_steps * ratio, classes_num)\n    \"\"\"\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):\n    \"\"\"Pad framewise_output to the same length as input frames. The pad value\n    is the same as the value of the last frame.\n    Args:\n      framewise_output: (batch_size, frames_num, classes_num)\n      frames_num: int, number of frames to pad\n    Outputs:\n      output: (batch_size, frames_num, classes_num)\n    \"\"\"\n    pad = framewise_output[:, -1:, :].repeat(\n        1, frames_num - framewise_output.shape[1], 1)\n    \"\"\"tensor for padding\"\"\"\n\n    output = torch.cat((framewise_output, pad), dim=1)\n    \"\"\"(batch_size, frames_num, classes_num)\"\"\"\n\n    return output\n\n\nclass ConvBlock(nn.Module):\n    def __init__(self, in_channels: int, out_channels: int):\n        super().__init__()\n\n        self.conv1 = nn.Conv2d(\n            in_channels=in_channels,\n            out_channels=out_channels,\n            kernel_size=(3, 3),\n            stride=(1, 1),\n            padding=(1, 1),\n            bias=False)\n\n        self.conv2 = nn.Conv2d(\n            in_channels=out_channels,\n            out_channels=out_channels,\n            kernel_size=(3, 3),\n            stride=(1, 1),\n            padding=(1, 1),\n            bias=False)\n\n        self.bn1 = nn.BatchNorm2d(out_channels)\n        self.bn2 = nn.BatchNorm2d(out_channels)\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_layer(self.conv1)\n        init_layer(self.conv2)\n        init_bn(self.bn1)\n        init_bn(self.bn2)\n\n    def forward(self, input, pool_size=(2, 2), pool_type='avg'):\n\n        x = input\n        x = F.relu_(self.bn1(self.conv1(x)))\n        x = F.relu_(self.bn2(self.conv2(x)))\n        if pool_type == 'max':\n            x = F.max_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg':\n            x = F.avg_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg+max':\n            x1 = F.avg_pool2d(x, kernel_size=pool_size)\n            x2 = F.max_pool2d(x, kernel_size=pool_size)\n            x = x1 + x2\n        else:\n            raise Exception('Incorrect argument!')\n\n        return x\n\n\nclass AttBlock(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"linear\",\n                 temperature=1.0):\n        super().__init__()\n\n        self.activation = activation\n        self.temperature = temperature\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.bn_att = nn.BatchNorm1d(out_features)\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n        init_bn(self.bn_att)\n\n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.clamp(self.att(x), -10, 10), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T16:40:53.075493Z","iopub.execute_input":"2025-01-16T16:40:53.075897Z","iopub.status.idle":"2025-01-16T16:40:53.113278Z","shell.execute_reply.started":"2025-01-16T16:40:53.075855Z","shell.execute_reply":"2025-01-16T16:40:53.112319Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class PANNsCNN14Att(nn.Module):\n    def __init__(self, sample_rate: int, window_size: int, hop_size: int,\n                 mel_bins: int, fmin: int, fmax: int, classes_num: int):\n        super().__init__()\n\n        window = 'hann'\n        center = True\n        pad_mode = 'reflect'\n        ref = 1.0\n        amin = 1e-10\n        top_db = None\n        self.interpolate_ratio = 32  # Downsampled ratio\n\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(\n            n_fft=window_size,\n            hop_length=hop_size,\n            win_length=window_size,\n            window=window,\n            center=center,\n            pad_mode=pad_mode,\n            freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(\n            sr=sample_rate,\n            n_fft=window_size,\n            n_mels=mel_bins,\n            fmin=fmin,\n            fmax=fmax,\n            ref=ref,\n            amin=amin,\n            top_db=top_db,\n            freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(\n            time_drop_width=64,\n            time_stripes_num=2,\n            freq_drop_width=8,\n            freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(mel_bins)\n\n        self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)\n        self.conv_block2 = ConvBlock(in_channels=64, out_channels=128)\n        self.conv_block3 = ConvBlock(in_channels=128, out_channels=256)\n        self.conv_block4 = ConvBlock(in_channels=256, out_channels=512)\n        self.conv_block5 = ConvBlock(in_channels=512, out_channels=1024)\n        self.conv_block6 = ConvBlock(in_channels=1024, out_channels=2048)\n\n        self.fc1 = nn.Linear(2048, 2048, bias=True)\n        self.att_block = AttBlock(2048, classes_num, activation='sigmoid')\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n        \n    def cnn_feature_extractor(self, x):\n        x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block2(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block3(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block4(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block5(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block6(x, pool_size=(1, 1), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        return x\n    \n    def preprocess(self, input, mixup_lambda=None):\n        # t1 = time.time()\n        x = self.spectrogram_extractor(input)  # (batch_size, 1, time_steps, freq_bins)\n        x = self.logmel_extractor(x)  # (batch_size, 1, time_steps, mel_bins)\n\n        frames_num = x.shape[2]\n\n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n\n        if self.training:\n            x = self.spec_augmenter(x)\n\n        # Mixup on spectrogram\n        if self.training and mixup_lambda is not None:\n            x = do_mixup(x, mixup_lambda)\n        return x, frames_num\n        \n\n    def forward(self, input, mixup_lambda=None):\n        \"\"\"\n        Input: (batch_size, data_length)\"\"\"\n        x, frames_num = self.preprocess(input, mixup_lambda=mixup_lambda)\n\n        # Output shape (batch size, channels, time, frequency)\n        x = self.cnn_feature_extractor(x)\n        \n        # Aggregate in frequency axis\n        x = torch.mean(x, dim=3)\n\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.5, training=self.training)\n\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        # Get framewise output\n        framewise_output = interpolate(segmentwise_output,\n                                       self.interpolate_ratio)\n        framewise_output = pad_framewise_output(framewise_output, frames_num)\n\n        output_dict = {\n            'framewise_output': framewise_output,\n            'clipwise_output': clipwise_output\n        }\n\n        return output_dict","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T16:40:53.114697Z","iopub.execute_input":"2025-01-16T16:40:53.115055Z","iopub.status.idle":"2025-01-16T16:40:53.148288Z","shell.execute_reply.started":"2025-01-16T16:40:53.115012Z","shell.execute_reply":"2025-01-16T16:40:53.147625Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SR = 32000\n\ny, _ = librosa.load(TRAIN_RESAMPLED_AUDIO_DIRS[0] / \"aldfly\" / \"XC134874.wav\",\n                    sr=SR,\n                    res_type=\"kaiser_fast\",\n                    mono=True)\n\nAudio(y, rate=SR)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T16:40:53.149244Z","iopub.execute_input":"2025-01-16T16:40:53.149462Z","iopub.status.idle":"2025-01-16T16:40:53.221225Z","shell.execute_reply.started":"2025-01-16T16:40:53.149442Z","shell.execute_reply":"2025-01-16T16:40:53.219936Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display.waveplot(y, sr=SR);","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T16:40:53.222699Z","iopub.execute_input":"2025-01-16T16:40:53.223166Z","iopub.status.idle":"2025-01-16T16:40:53.391864Z","shell.execute_reply.started":"2025-01-16T16:40:53.223116Z","shell.execute_reply":"2025-01-16T16:40:53.391037Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_config = {\n    \"sample_rate\": 32000,\n    \"window_size\": 1024,\n    \"hop_size\": 320,\n    \"mel_bins\": 64,\n    \"fmin\": 50,\n    \"fmax\": 14000,\n    \"classes_num\": 264\n}\n\nmodel = PANNsCNN14Att(**model_config)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T16:40:53.392948Z","iopub.execute_input":"2025-01-16T16:40:53.393193Z","iopub.status.idle":"2025-01-16T16:40:54.846609Z","shell.execute_reply.started":"2025-01-16T16:40:53.393169Z","shell.execute_reply":"2025-01-16T16:40:54.845883Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"chunk = torch.from_numpy(y[:SR * 5]).unsqueeze(0)\nmelspec, _ = model.preprocess(chunk)\nmelspec.size()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T16:40:54.847672Z","iopub.execute_input":"2025-01-16T16:40:54.847922Z","iopub.status.idle":"2025-01-16T16:40:54.934899Z","shell.execute_reply.started":"2025-01-16T16:40:54.847897Z","shell.execute_reply":"2025-01-16T16:40:54.934081Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"melspec_numpy = melspec.detach().numpy()[0, 0].transpose(1, 0)\ndisplay.specshow(melspec_numpy, sr=SR, y_axis=\"mel\");","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T16:40:54.936766Z","iopub.execute_input":"2025-01-16T16:40:54.937146Z","iopub.status.idle":"2025-01-16T16:40:55.045476Z","shell.execute_reply.started":"2025-01-16T16:40:54.937109Z","shell.execute_reply":"2025-01-16T16:40:55.044672Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"`PANNsCNN14Att.cnn_feature_extractor()` method will take this as input and output feature map. Let's check the output of the feature extractor.","metadata":{}},{"cell_type":"code","source":"feature_map = model.cnn_feature_extractor(melspec)\nfeature_map.size()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T16:40:55.046815Z","iopub.execute_input":"2025-01-16T16:40:55.047240Z","iopub.status.idle":"2025-01-16T16:40:55.406259Z","shell.execute_reply.started":"2025-01-16T16:40:55.047198Z","shell.execute_reply":"2025-01-16T16:40:55.405391Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Dataset","metadata":{}},{"cell_type":"code","source":"BIRD_CODE = {\n    'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,\n    'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,\n    'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,\n    'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,\n    'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,\n    'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,\n    'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,\n    'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,\n    'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,\n    'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,\n    'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,\n    'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,\n    'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,\n    'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,\n    'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,\n    'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,\n    'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,\n    'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,\n    'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,\n    'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,\n    'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,\n    'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,\n    'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,\n    'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,\n    'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,\n    'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,\n    'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,\n    'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,\n    'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,\n    'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,\n    'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,\n    'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,\n    'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,\n    'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,\n    'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,\n    'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,\n    'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,\n    'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,\n    'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,\n    'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,\n    'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,\n    'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,\n    'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,\n    'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,\n    'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,\n    'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,\n    'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,\n    'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,\n    'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,\n    'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,\n    'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,\n    'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,\n    'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263\n}\n\nINV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items()}","metadata":{"_kg_hide-input":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T16:40:55.407920Z","iopub.execute_input":"2025-01-16T16:40:55.408290Z","iopub.status.idle":"2025-01-16T16:40:55.433805Z","shell.execute_reply.started":"2025-01-16T16:40:55.408252Z","shell.execute_reply":"2025-01-16T16:40:55.432889Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"PERIOD = 5\n\nclass PANNsDataset(data.Dataset):\n    def __init__(\n            self,\n            file_list: List[List[str]],\n            waveform_transforms=None):\n        self.file_list = file_list  # list of list: [file_path, ebird_code]\n        self.waveform_transforms = waveform_transforms\n\n    def __len__(self):\n        return len(self.file_list)\n\n    def __getitem__(self, idx: int):\n        wav_path, ebird_code = self.file_list[idx]\n\n        y, sr = sf.read(wav_path)\n\n        if self.waveform_transforms:\n            y = self.waveform_transforms(y)\n        else:\n            len_y = len(y)\n            effective_length = sr * PERIOD\n            if len_y < effective_length:\n                new_y = np.zeros(effective_length, dtype=y.dtype)\n                start = np.random.randint(effective_length - len_y)\n                new_y[start:start + len_y] = y\n                y = new_y.astype(np.float32)\n            elif len_y > effective_length:\n                start = np.random.randint(len_y - effective_length)\n                y = y[start:start + effective_length].astype(np.float32)\n            else:\n                y = y.astype(np.float32)\n\n        labels = np.zeros(len(BIRD_CODE), dtype=\"f\")\n        labels[BIRD_CODE[ebird_code]] = 1\n\n        return {\"waveform\": y, \"targets\": labels}","metadata":{"_kg_hide-input":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T16:40:55.435146Z","iopub.execute_input":"2025-01-16T16:40:55.435488Z","iopub.status.idle":"2025-01-16T16:40:55.448260Z","shell.execute_reply.started":"2025-01-16T16:40:55.435452Z","shell.execute_reply":"2025-01-16T16:40:55.447513Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Criterion","metadata":{}},{"cell_type":"code","source":"class PANNsLoss(nn.Module):\n    def __init__(self):\n        super().__init__()\n\n        self.bce = nn.BCELoss()\n\n    def forward(self, input, target):\n        input_ = input[\"clipwise_output\"]\n        input_ = torch.where(torch.isnan(input_),\n                             torch.zeros_like(input_),\n                             input_)\n        input_ = torch.where(torch.isinf(input_),\n                             torch.zeros_like(input_),\n                             input_)\n\n        target = target.float()\n\n        return self.bce(input_, target)","metadata":{"_kg_hide-input":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T16:40:55.449549Z","iopub.execute_input":"2025-01-16T16:40:55.449893Z","iopub.status.idle":"2025-01-16T16:40:55.457075Z","shell.execute_reply.started":"2025-01-16T16:40:55.449858Z","shell.execute_reply":"2025-01-16T16:40:55.456318Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Callbacks","metadata":{}},{"cell_type":"code","source":"class F1Callback(Callback):\n    def __init__(self,\n                 input_key: str = \"targets\",\n                 output_key: str = \"logits\",\n                 model_output_key: str = \"clipwise_output\",\n                 prefix: str = \"f1\"):\n        super().__init__(CallbackOrder.Metric)\n\n        self.input_key = input_key\n        self.output_key = output_key\n        self.model_output_key = model_output_key\n        self.prefix = prefix\n\n    def on_loader_start(self, state: State):\n        self.prediction: List[np.ndarray] = []\n        self.target: List[np.ndarray] = []\n\n    def on_batch_end(self, state: State):\n        targ = state.input[self.input_key].detach().cpu().numpy()\n        out = state.output[self.output_key]\n\n        clipwise_output = out[self.model_output_key].detach().cpu().numpy()\n\n        self.prediction.append(clipwise_output)\n        self.target.append(targ)\n\n        y_pred = clipwise_output.argmax(axis=1)\n        y_true = targ.argmax(axis=1)\n\n        score = f1_score(y_true, y_pred, average=\"macro\")\n        state.batch_metrics[self.prefix] = score\n\n    def on_loader_end(self, state: State):\n        y_pred = np.concatenate(self.prediction, axis=0).argmax(axis=1)\n        y_true = np.concatenate(self.target, axis=0).argmax(axis=1)\n        score = f1_score(y_true, y_pred, average=\"macro\")\n        state.loader_metrics[self.prefix] = score\n        if state.is_valid_loader:\n            state.epoch_metrics[state.valid_loader + \"_epoch_\" +\n                                self.prefix] = score\n        else:\n            state.epoch_metrics[\"train_epoch_\" + self.prefix] = score\n\n\nclass mAPCallback(Callback):\n    def __init__(self,\n                 input_key: str = \"targets\",\n                 output_key: str = \"logits\",\n                 model_output_key: str = \"clipwise_output\",\n                 prefix: str = \"mAP\"):\n        super().__init__(CallbackOrder.Metric)\n        self.input_key = input_key\n        self.output_key = output_key\n        self.model_output_key = model_output_key\n        self.prefix = prefix\n\n    def on_loader_start(self, state: State):\n        self.prediction: List[np.ndarray] = []\n        self.target: List[np.ndarray] = []\n\n    def on_batch_end(self, state: State):\n        targ = state.input[self.input_key].detach().cpu().numpy()\n        out = state.output[self.output_key]\n\n        clipwise_output = out[self.model_output_key].detach().cpu().numpy()\n\n        self.prediction.append(clipwise_output)\n        self.target.append(targ)\n\n        score = average_precision_score(targ, clipwise_output, average=None)\n        score = np.nan_to_num(score).mean()\n        state.batch_metrics[self.prefix] = score\n\n    def on_loader_end(self, state: State):\n        y_pred = np.concatenate(self.prediction, axis=0)\n        y_true = np.concatenate(self.target, axis=0)\n        score = average_precision_score(y_true, y_pred, average=None)\n        score = np.nan_to_num(score).mean()\n        state.loader_metrics[self.prefix] = score\n        if state.is_valid_loader:\n            state.epoch_metrics[state.valid_loader + \"_epoch_\" +\n                                self.prefix] = score\n        else:\n            state.epoch_metrics[\"train_epoch_\" + self.prefix] = score","metadata":{"_kg_hide-input":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T16:40:55.458662Z","iopub.execute_input":"2025-01-16T16:40:55.459009Z","iopub.status.idle":"2025-01-16T16:40:55.485934Z","shell.execute_reply.started":"2025-01-16T16:40:55.458975Z","shell.execute_reply":"2025-01-16T16:40:55.485210Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Train\n\n","metadata":{}},{"cell_type":"code","source":"tmp_list = []\nfor audio_d in TRAIN_RESAMPLED_AUDIO_DIRS:\n    if not audio_d.exists():\n        continue\n    for ebird_d in audio_d.iterdir():\n        if ebird_d.is_file():\n            continue\n        for wav_f in ebird_d.iterdir():\n            tmp_list.append([ebird_d.name, wav_f.name, wav_f.as_posix()])\n            \ntrain_wav_path_exist = pd.DataFrame(\n    tmp_list, columns=[\"ebird_code\", \"resampled_filename\", \"file_path\"])\n\ndel tmp_list\n\ntrain_all = pd.merge(\n    train, train_wav_path_exist, on=[\"ebird_code\", \"resampled_filename\"], how=\"inner\")\n\nprint(train.shape)\nprint(train_wav_path_exist.shape)\nprint(train_all.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T16:40:55.487369Z","iopub.execute_input":"2025-01-16T16:40:55.487703Z","iopub.status.idle":"2025-01-16T16:40:57.093985Z","shell.execute_reply.started":"2025-01-16T16:40:55.487670Z","shell.execute_reply":"2025-01-16T16:40:57.093041Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n\ntrain_all[\"fold\"] = -1\nfor fold_id, (train_index, val_index) in enumerate(skf.split(train_all, train_all[\"ebird_code\"])):\n    train_all.iloc[val_index, -1] = fold_id\n    \n# # check the propotion\nfold_proportion = pd.pivot_table(train_all, index=\"ebird_code\", columns=\"fold\", values=\"xc_id\", aggfunc=len)\nprint(fold_proportion.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T16:40:57.095323Z","iopub.execute_input":"2025-01-16T16:40:57.095831Z","iopub.status.idle":"2025-01-16T16:40:57.166172Z","shell.execute_reply.started":"2025-01-16T16:40:57.095789Z","shell.execute_reply":"2025-01-16T16:40:57.165424Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"use_fold = 0\ntrain_file_list = train_all.query(\"fold != @use_fold\")[[\"file_path\", \"ebird_code\"]].values.tolist()\nval_file_list = train_all.query(\"fold == @use_fold\")[[\"file_path\", \"ebird_code\"]].values.tolist()\n\nprint(\"[fold {}] train: {}, val: {}\".format(use_fold, len(train_file_list), len(val_file_list)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T16:40:57.167275Z","iopub.execute_input":"2025-01-16T16:40:57.167501Z","iopub.status.idle":"2025-01-16T16:40:57.198068Z","shell.execute_reply.started":"2025-01-16T16:40:57.167479Z","shell.execute_reply":"2025-01-16T16:40:57.197168Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = torch.device(\"cuda:0\")\n\n# loaders\nloaders = {\n    \"train\": data.DataLoader(PANNsDataset(train_file_list, None), \n                             batch_size=64, \n                             shuffle=True, \n                             num_workers=2, \n                             pin_memory=True, \n                             drop_last=True),\n    \"valid\": data.DataLoader(PANNsDataset(val_file_list, None), \n                             batch_size=64, \n                             shuffle=False,\n                             num_workers=2,\n                             pin_memory=True,\n                             drop_last=False)\n}\n\n# model\nmodel_config[\"classes_num\"] = 527\nmodel = PANNsCNN14Att(**model_config)\nweights = torch.load(\"../input/pannscnn14-decisionlevelatt-weight/Cnn14_DecisionLevelAtt_mAP0.425.pth\")\n# Fixed in V3\nmodel.load_state_dict(weights[\"model\"])\nmodel.att_block = AttBlock(2048, 264, activation='sigmoid')\nmodel.att_block.init_weights()\nmodel.to(device)\n\n# Optimizer\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# Scheduler\nscheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10)\n\n# Loss\ncriterion = PANNsLoss().to(device)\n\n# callbacks\ncallbacks = [\n    F1Callback(input_key=\"targets\", output_key=\"logits\", prefix=\"f1\"),\n    mAPCallback(input_key=\"targets\", output_key=\"logits\", prefix=\"mAP\"),\n    CheckpointCallback(save_n_best=0)\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T16:40:57.199267Z","iopub.execute_input":"2025-01-16T16:40:57.199625Z","iopub.status.idle":"2025-01-16T16:41:01.752330Z","shell.execute_reply.started":"2025-01-16T16:40:57.199578Z","shell.execute_reply":"2025-01-16T16:41:01.751513Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"warnings.simplefilter(\"ignore\")\n\nrunner = SupervisedRunner(\n    device=device,\n    input_key=\"waveform\",\n    input_target_key=\"targets\")\n\nrunner.train(\n    model=model,\n    criterion=criterion,\n    loaders=loaders,\n    optimizer=optimizer,\n    scheduler=scheduler,\n    num_epochs=10,\n    verbose=True,\n    logdir=f\"fold0\",\n    callbacks=callbacks,\n    main_metric=\"epoch_f1\",\n    minimize_metric=False)","metadata":{"_kg_hide-output":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T16:41:01.755919Z","iopub.execute_input":"2025-01-16T16:41:01.756205Z","iopub.status.idle":"2025-01-16T18:06:10.241458Z","shell.execute_reply.started":"2025-01-16T16:41:01.756179Z","shell.execute_reply":"2025-01-16T18:06:10.240184Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Prediction with SED model","metadata":{}},{"cell_type":"code","source":"model_config = {\n    \"sample_rate\": 32000,\n    \"window_size\": 1024,\n    \"hop_size\": 320,\n    \"mel_bins\": 64,\n    \"fmin\": 50,\n    \"fmax\": 14000,\n    \"classes_num\": 264\n}\n\nweights_path = \"../input/birdcall-pannsatt-aux-weak/best.pth\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T18:06:10.245604Z","iopub.execute_input":"2025-01-16T18:06:10.246130Z","iopub.status.idle":"2025-01-16T18:06:10.253623Z","shell.execute_reply.started":"2025-01-16T18:06:10.246072Z","shell.execute_reply":"2025-01-16T18:06:10.252649Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_model(config: dict, weights_path: str):\n    model = PANNsCNN14Att(**config)\n    checkpoint = torch.load(weights_path)\n    model.load_state_dict(checkpoint[\"model_state_dict\"])\n    device = torch.device(\"cuda\")\n    model.to(device)\n    model.eval()\n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T18:06:10.255115Z","iopub.execute_input":"2025-01-16T18:06:10.255480Z","iopub.status.idle":"2025-01-16T18:06:10.259855Z","shell.execute_reply.started":"2025-01-16T18:06:10.255447Z","shell.execute_reply":"2025-01-16T18:06:10.259215Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def prediction_for_clip(test_df: pd.DataFrame,\n                        clip: np.ndarray, \n                        model: PANNsCNN14Att,\n                        threshold=0.5):\n    PERIOD = 30\n    audios = []\n    y = clip.astype(np.float32)\n    len_y = len(y)\n    start = 0\n    end = PERIOD * SR\n    while True:\n        y_batch = y[start:end].astype(np.float32)\n        if len(y_batch) != PERIOD * SR:\n            y_pad = np.zeros(PERIOD * SR, dtype=np.float32)\n            y_pad[:len(y_batch)] = y_batch\n            audios.append(y_pad)\n            break\n        start = end\n        end += PERIOD * SR\n        audios.append(y_batch)\n        \n    array = np.asarray(audios)\n    tensors = torch.from_numpy(array)\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    \n    model.eval()\n    estimated_event_list = []\n    global_time = 0.0\n    site = test_df[\"site\"].values[0]\n    audio_id = test_df[\"audio_id\"].values[0]\n    for image in progress_bar(tensors):\n        image = image.view(1, image.size(0))\n        image = image.to(device)\n\n        with torch.no_grad():\n            prediction = model(image)\n            framewise_outputs = prediction[\"framewise_output\"].detach(\n                ).cpu().numpy()[0]\n                \n        thresholded = framewise_outputs >= threshold\n\n        for target_idx in range(thresholded.shape[1]):\n            if thresholded[:, target_idx].mean() == 0:\n                pass\n            else:\n                detected = np.argwhere(thresholded[:, target_idx]).reshape(-1)\n                head_idx = 0\n                tail_idx = 0\n                while True:\n                    if (tail_idx + 1 == len(detected)) or (\n                            detected[tail_idx + 1] - \n                            detected[tail_idx] != 1):\n                        onset = 0.01 * detected[\n                            head_idx] + global_time\n                        offset = 0.01 * detected[\n                            tail_idx] + global_time\n                        onset_idx = detected[head_idx]\n                        offset_idx = detected[tail_idx]\n                        max_confidence = framewise_outputs[\n                            onset_idx:offset_idx, target_idx].max()\n                        mean_confidence = framewise_outputs[\n                            onset_idx:offset_idx, target_idx].mean()\n                        estimated_event = {\n                            \"site\": site,\n                            \"audio_id\": audio_id,\n                            \"ebird_code\": INV_BIRD_CODE[target_idx],\n                            \"onset\": onset,\n                            \"offset\": offset,\n                            \"max_confidence\": max_confidence,\n                            \"mean_confidence\": mean_confidence\n                        }\n                        estimated_event_list.append(estimated_event)\n                        head_idx = tail_idx + 1\n                        tail_idx = tail_idx + 1\n                        if head_idx >= len(detected):\n                            break\n                    else:\n                        tail_idx += 1\n        global_time += PERIOD\n        \n    prediction_df = pd.DataFrame(estimated_event_list)\n    return prediction_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T18:06:10.261495Z","iopub.execute_input":"2025-01-16T18:06:10.261727Z","iopub.status.idle":"2025-01-16T18:06:10.277204Z","shell.execute_reply.started":"2025-01-16T18:06:10.261705Z","shell.execute_reply":"2025-01-16T18:06:10.276157Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def prediction(test_df: pd.DataFrame,\n               test_audio: Path,\n               model_config: dict,\n               weights_path: str,\n               threshold=0.5):\n    model = get_model(model_config, weights_path)\n    unique_audio_id = test_df.audio_id.unique()\n\n    warnings.filterwarnings(\"ignore\")\n    prediction_dfs = []\n    for audio_id in unique_audio_id:\n        with timer(f\"Loading {audio_id}\"):\n            clip, _ = librosa.load(test_audio / (audio_id + \".mp3\"),\n                                   sr=SR,\n                                   mono=True,\n                                   res_type=\"kaiser_fast\")\n        \n        test_df_for_audio_id = test_df.query(\n            f\"audio_id == '{audio_id}'\").reset_index(drop=True)\n        with timer(f\"Prediction on {audio_id}\"):\n            prediction_df = prediction_for_clip(test_df_for_audio_id,\n                                                clip=clip,\n                                                model=model,\n                                                threshold=threshold)\n\n        prediction_dfs.append(prediction_df)\n    \n    prediction_df = pd.concat(prediction_dfs, axis=0, sort=False).reset_index(drop=True)\n    return prediction_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T18:06:10.278550Z","iopub.execute_input":"2025-01-16T18:06:10.278895Z","iopub.status.idle":"2025-01-16T18:06:10.289281Z","shell.execute_reply.started":"2025-01-16T18:06:10.278859Z","shell.execute_reply":"2025-01-16T18:06:10.288571Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"prediction_df = prediction(test_df=test,\n                           test_audio=TEST_AUDIO_DIR,\n                           model_config=model_config,\n                           weights_path=weights_path,\n                           threshold=0.5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T18:06:10.290667Z","iopub.execute_input":"2025-01-16T18:06:10.291028Z","iopub.status.idle":"2025-01-16T18:06:33.458997Z","shell.execute_reply.started":"2025-01-16T18:06:10.290990Z","shell.execute_reply":"2025-01-16T18:06:33.458040Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"prediction_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T18:06:33.460845Z","iopub.execute_input":"2025-01-16T18:06:33.461165Z","iopub.status.idle":"2025-01-16T18:06:33.479389Z","shell.execute_reply.started":"2025-01-16T18:06:33.461135Z","shell.execute_reply":"2025-01-16T18:06:33.478244Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Postprocess\n","metadata":{}},{"cell_type":"code","source":"labels = {}\n\nfor audio_id, sub_df in prediction_df.groupby(\"audio_id\"):\n    events = sub_df[[\"ebird_code\", \"onset\", \"offset\", \"max_confidence\", \"site\"]].values\n    n_events = len(events)\n    removed_event = []\n    # Overlap deletion: this part may not be necessary\n    # I deleted this part in other model and found there's no difference on the public LB score.\n    for i in range(n_events):\n        for j in range(n_events):\n            if i == j:\n                continue\n            if i in removed_event:\n                continue\n            if j in removed_event:\n                continue\n            \n            event_i = events[i]\n            event_j = events[j]\n            \n            if (event_i[1] - event_j[2] >= 0) or (event_j[1] - event_i[2] >= 0):\n                pass\n            else:\n                later_onset = max(event_i[1], event_j[1])\n                sooner_onset = min(event_i[1], event_j[1])\n                sooner_offset = min(event_i[2], event_j[2])\n                later_offset = max(event_i[2], event_j[2])\n\n                intersection = sooner_offset - later_onset\n                union = later_offset - sooner_onset\n                \n                iou = intersection / union\n                if iou > 0.4:\n                    if event_i[3] > event_j[3]:\n                        removed_event.append(j)\n                    else:\n                        removed_event.append(i)\n\n    site = events[0][4]\n    for i in range(n_events):\n        if i in removed_event:\n            continue\n        event = events[i][0]\n        onset = events[i][1]\n        offset = events[i][2]\n        if site in {\"site_1\", \"site_2\"}:\n            start_section = int((onset // 5) * 5) + 5\n            end_section = int((offset // 5) * 5) + 5\n            cur_section = start_section\n\n            row_id = f\"{site}_{audio_id}_{start_section}\"\n            if labels.get(row_id) is not None:\n                labels[row_id].add(event)\n            else:\n                labels[row_id] = set()\n                labels[row_id].add(event)\n\n            while cur_section != end_section:\n                cur_section += 5\n                row_id = f\"{site}_{audio_id}_{cur_section}\"\n                if labels.get(row_id) is not None:\n                    labels[row_id].add(event)\n                else:\n                    labels[row_id] = set()\n                    labels[row_id].add(event)\n        else:\n            row_id = f\"{site}_{audio_id}\"\n            if labels.get(row_id) is not None:\n                labels[row_id].add(event)\n            else:\n                labels[row_id] = set()\n                labels[row_id].add(event)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T18:06:33.481193Z","iopub.execute_input":"2025-01-16T18:06:33.481639Z","iopub.status.idle":"2025-01-16T18:06:33.537530Z","shell.execute_reply.started":"2025-01-16T18:06:33.481593Z","shell.execute_reply":"2025-01-16T18:06:33.536726Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for key in labels:\n    labels[key] = \" \".join(sorted(list(labels[key])))\n    \n    \nrow_ids = list(labels.keys())\nbirds = list(labels.values())\npost_processed = pd.DataFrame({\n    \"row_id\": row_ids,\n    \"birds\": birds\n})\npost_processed.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T18:06:33.538635Z","iopub.execute_input":"2025-01-16T18:06:33.538907Z","iopub.status.idle":"2025-01-16T18:06:33.550343Z","shell.execute_reply.started":"2025-01-16T18:06:33.538881Z","shell.execute_reply":"2025-01-16T18:06:33.549514Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_row_id = test[[\"row_id\"]]\nsubmission = all_row_id.merge(post_processed, on=\"row_id\", how=\"left\")\nsubmission = submission.fillna(\"nocall\")\nsubmission.to_csv(\"submission.csv\", index=False)\nsubmission.head(20)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T18:06:33.551508Z","iopub.execute_input":"2025-01-16T18:06:33.551766Z","iopub.status.idle":"2025-01-16T18:06:33.574105Z","shell.execute_reply.started":"2025-01-16T18:06:33.551739Z","shell.execute_reply":"2025-01-16T18:06:33.573161Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## EOF","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}