{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":19596,"databundleVersionId":1292430,"sourceType":"competition"},{"sourceId":1262046,"sourceType":"datasetVersion","datasetId":726424},{"sourceId":1305829,"sourceType":"datasetVersion","datasetId":755946},{"sourceId":1306285,"sourceType":"datasetVersion","datasetId":756248},{"sourceId":1306896,"sourceType":"datasetVersion","datasetId":756642},{"sourceId":1307127,"sourceType":"datasetVersion","datasetId":756745},{"sourceId":1307546,"sourceType":"datasetVersion","datasetId":756953},{"sourceId":1346120,"sourceType":"datasetVersion","datasetId":782938},{"sourceId":1405119,"sourceType":"datasetVersion","datasetId":821504},{"sourceId":7971661,"sourceType":"datasetVersion","datasetId":4690510},{"sourceId":7973700,"sourceType":"datasetVersion","datasetId":4692188},{"sourceId":21369,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":17697}],"dockerImageVersionId":29987,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import cv2\nimport audioread\nimport logging\nimport os\nimport random\nimport time\nimport warnings\n\nimport librosa\nimport librosa.display as display\nimport numpy as np\nimport pandas as pd\nimport soundfile as sf\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.utils.data as data\n\nfrom contextlib import contextmanager\nfrom IPython.display import Audio\nfrom pathlib import Path\nfrom typing import Optional, List\n\nfrom catalyst.dl import SupervisedRunner, State, CallbackOrder, Callback, CheckpointCallback\nfrom fastprogress import progress_bar\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score, average_precision_score","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2024-03-29T09:03:35.493851Z","iopub.execute_input":"2024-03-29T09:03:35.494175Z","iopub.status.idle":"2024-03-29T09:03:45.266403Z","shell.execute_reply.started":"2024-03-29T09:03:35.494147Z","shell.execute_reply":"2024-03-29T09:03:45.265400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 设定随机种子\ndef set_seed(seed: int = 42):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)  # type: ignore\n    torch.backends.cudnn.deterministic = True  # type: ignore\n    torch.backends.cudnn.benchmark = True  # type: ignore\n    \n\n# 这段代码的目的是创建一个配置好的日志记录器，它可以将日志信息输出到控制台\ndef get_logger(out_file=None):\n    logger = logging.getLogger()\n    formatter = logging.Formatter(\"%(asctime)s - %(levelname)s - %(message)s\")\n    logger.handlers = []\n    logger.setLevel(logging.INFO)\n\n    handler = logging.StreamHandler()\n    handler.setFormatter(formatter)\n    handler.setLevel(logging.INFO)\n    logger.addHandler(handler)\n\n    if out_file is not None:\n        fh = logging.FileHandler(out_file)\n        fh.setFormatter(formatter)\n        fh.setLevel(logging.INFO)\n        logger.addHandler(fh)\n        \n    logger.info(\"logger set up\")\n    return logger\n    \n    \n    \n# 创建一个上下文管理器，它可以自动地在一段代码执行前后记录时间，从而方便地测量代码的执行时间。\n@contextmanager\ndef timer(name: str, logger: Optional[logging.Logger] = None):\n    t0 = time.time()\n    msg = f\"[{name}] start\"\n    if logger is None:\n        print(msg)\n    else:\n        logger.info(msg)\n    yield\n\n    msg = f\"[{name}] done in {time.time() - t0:.2f} s\"\n    if logger is None:\n        print(msg)\n    else:\n        logger.info(msg)\n    \n# 设置种子\nset_seed(1213)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2024-03-29T09:03:45.268502Z","iopub.execute_input":"2024-03-29T09:03:45.268801Z","iopub.status.idle":"2024-03-29T09:03:45.288299Z","shell.execute_reply.started":"2024-03-29T09:03:45.268754Z","shell.execute_reply":"2024-03-29T09:03:45.287429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 设置各个输入的路径\nROOT = Path.cwd().parent\nINPUT_ROOT = ROOT / \"input\"\n# RAW_DATA = INPUT_ROOT / \"birdsong-recognition\"\nRAW_DATA = INPUT_ROOT / \"birdcall-check\"\nTRAIN_AUDIO_DIR = RAW_DATA / \"train_audio\"\nTEST_AUDIO_DIR = RAW_DATA / \"test_audio\"\nTELL_BY_TYPE = INPUT_ROOT / \"tellbytype\" / \"mytrain_mod.csv\"\nTRAIN_RESAMPLED_AUDIO_DIRS = [\n  INPUT_ROOT / \"birdsong-resampled-train-audio-{:0>2}\".format(i)  for i in range(5)\n]\n","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-03-29T11:04:15.036341Z","iopub.execute_input":"2024-03-29T11:04:15.036690Z","iopub.status.idle":"2024-03-29T11:04:15.042999Z","shell.execute_reply.started":"2024-03-29T11:04:15.036659Z","shell.execute_reply":"2024-03-29T11:04:15.042162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"设置输入路径","metadata":{}},{"cell_type":"code","source":"# 读取birdcall-check的 test_audio 读取test.cvs\n\ntrain = pd.read_csv(TELL_BY_TYPE)\n\nif not TEST_AUDIO_DIR.exists():\n    TEST_AUDIO_DIR = INPUT_ROOT / \"birdcall-check\" / \"test_audio\"\n    test = pd.read_csv(INPUT_ROOT / \"birdcall-check\" / \"test.csv\")\nelse:\n    test = pd.read_csv(RAW_DATA / \"test.csv\")","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2024-03-29T09:03:45.304017Z","iopub.execute_input":"2024-03-29T09:03:45.304317Z","iopub.status.idle":"2024-03-29T09:03:45.745278Z","shell.execute_reply.started":"2024-03-29T09:03:45.304289Z","shell.execute_reply":"2024-03-29T09:03:45.744364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 这是用于创建离散傅里叶变换（DFT）和逆离散傅里叶变换（IDFT）矩阵的基类\nclass DFTBase(nn.Module):\n    def __init__(self):\n        \"\"\"Base class for DFT and IDFT matrix\"\"\"\n        super(DFTBase, self).__init__()\n\n    def dft_matrix(self, n):\n        (x, y) = np.meshgrid(np.arange(n), np.arange(n))\n        omega = np.exp(-2 * np.pi * 1j / n)\n        W = np.power(omega, x * y)\n        return W\n\n    def idft_matrix(self, n):\n        (x, y) = np.meshgrid(np.arange(n), np.arange(n))\n        omega = np.exp(2 * np.pi * 1j / n)\n        W = np.power(omega, x * y)\n        return W\n    \n\n#用于实现短时傅里叶变换，实现STFT，可以将信号从时域转换到频域，同时保留了信号的时间信息。\nclass STFT(DFTBase):\n    def __init__(self, n_fft=2048, hop_length=None, win_length=None, \n        window='hann', center=True, pad_mode='reflect', freeze_parameters=True):\n        \"\"\"Implementation of STFT with Conv1d. The function has the same output \n        of librosa.core.stft\n        \"\"\"\n        super(STFT, self).__init__()\n\n        assert pad_mode in ['constant', 'reflect']\n\n        self.n_fft = n_fft\n        self.center = center\n        self.pad_mode = pad_mode\n\n        # By default, use the entire frame\n        if win_length is None:\n            win_length = n_fft\n\n        # Set the default hop, if it's not already specified\n        if hop_length is None:\n            hop_length = int(win_length // 4)\n\n        fft_window = librosa.filters.get_window(window, win_length, fftbins=True)\n\n        # Pad the window out to n_fft size\n        fft_window = librosa.util.pad_center(fft_window, n_fft)\n\n        # DFT & IDFT matrix\n        self.W = self.dft_matrix(n_fft)\n\n        out_channels = n_fft // 2 + 1\n\n        self.conv_real = nn.Conv1d(in_channels=1, out_channels=out_channels, \n            kernel_size=n_fft, stride=hop_length, padding=0, dilation=1, \n            groups=1, bias=False)\n\n        self.conv_imag = nn.Conv1d(in_channels=1, out_channels=out_channels, \n            kernel_size=n_fft, stride=hop_length, padding=0, dilation=1, \n            groups=1, bias=False)\n\n        self.conv_real.weight.data = torch.Tensor(\n            np.real(self.W[:, 0 : out_channels] * fft_window[:, None]).T)[:, None, :]\n        # (n_fft // 2 + 1, 1, n_fft)\n\n        self.conv_imag.weight.data = torch.Tensor(\n            np.imag(self.W[:, 0 : out_channels] * fft_window[:, None]).T)[:, None, :]\n        # (n_fft // 2 + 1, 1, n_fft)\n\n        if freeze_parameters:\n            for param in self.parameters():\n                param.requires_grad = False\n\n    def forward(self, input):\n        \"\"\"input: (batch_size, data_length)\n        Returns:\n          real: (batch_size, n_fft // 2 + 1, time_steps)\n          imag: (batch_size, n_fft // 2 + 1, time_steps)\n        \"\"\"\n\n        x = input[:, None, :]   # (batch_size, channels_num, data_length)\n\n        if self.center:\n            x = F.pad(x, pad=(self.n_fft // 2, self.n_fft // 2), mode=self.pad_mode)\n\n        real = self.conv_real(x)\n        imag = self.conv_imag(x)\n        # (batch_size, n_fft // 2 + 1, time_steps)\n\n        real = real[:, None, :, :].transpose(2, 3)\n        imag = imag[:, None, :, :].transpose(2, 3)\n        # (batch_size, 1, time_steps, n_fft // 2 + 1)\n\n        return real, imag\n    \n# 用于计算声谱图（spectrogram）\nclass Spectrogram(nn.Module):\n    def __init__(self, n_fft=2048, hop_length=None, win_length=None, \n        window='hann', center=True, pad_mode='reflect', power=2.0, \n        freeze_parameters=True):\n        \"\"\"Calculate spectrogram using pytorch. The STFT is implemented with \n        Conv1d. The function has the same output of librosa.core.stft\n        \"\"\"\n        super(Spectrogram, self).__init__()\n\n        self.power = power\n\n        self.stft = STFT(n_fft=n_fft, hop_length=hop_length, \n            win_length=win_length, window=window, center=center, \n            pad_mode=pad_mode, freeze_parameters=True)\n\n    def forward(self, input):\n        \"\"\"input: (batch_size, 1, time_steps, n_fft // 2 + 1)\n        Returns:\n          spectrogram: (batch_size, 1, time_steps, n_fft // 2 + 1)\n        \"\"\"\n\n        (real, imag) = self.stft.forward(input)\n        # (batch_size, n_fft // 2 + 1, time_steps)\n\n        spectrogram = real ** 2 + imag ** 2\n\n        if self.power == 2.0:\n            pass\n        else:\n            spectrogram = spectrogram ** (power / 2.0)\n\n        return spectrogram\n\n# 这段代码实现了一个用于声音信号处理的频谱图计算模块    \nclass LogmelFilterBank(nn.Module):\n    def __init__(self, sr=32000, n_fft=2048, n_mels=64, fmin=50, fmax=14000, is_log=True, \n        ref=1.0, amin=1e-10, top_db=80.0, freeze_parameters=True):\n        \"\"\"Calculate logmel spectrogram using pytorch. The mel filter bank is \n        the pytorch implementation of as librosa.filters.mel \n        \"\"\"\n        super(LogmelFilterBank, self).__init__()\n\n        self.is_log = is_log\n        self.ref = ref\n        self.amin = amin\n        self.top_db = top_db\n\n        self.melW = librosa.filters.mel(sr=sr, n_fft=n_fft, n_mels=n_mels,\n            fmin=fmin, fmax=fmax).T\n        # (n_fft // 2 + 1, mel_bins)\n\n        self.melW = nn.Parameter(torch.Tensor(self.melW))\n\n        if freeze_parameters:\n            for param in self.parameters():\n                param.requires_grad = False\n\n    def forward(self, input):\n        \"\"\"input: (batch_size, channels, time_steps)\n        \n        Output: (batch_size, time_steps, mel_bins)\n        \"\"\"\n\n        # Mel spectrogram\n        mel_spectrogram = torch.matmul(input, self.melW)\n\n        # Logmel spectrogram\n        if self.is_log:\n            output = self.power_to_db(mel_spectrogram)\n        else:\n            output = mel_spectrogram\n\n        return output\n\n\n    def power_to_db(self, input):\n        \"\"\"Power to db, this function is the pytorch implementation of \n        librosa.core.power_to_lb\n        \"\"\"\n        ref_value = self.ref\n        log_spec = 10.0 * torch.log10(torch.clamp(input, min=self.amin, max=np.inf))\n        log_spec -= 10.0 * np.log10(np.maximum(self.amin, ref_value))\n\n        if self.top_db is not None:\n            if self.top_db < 0:\n                raise ParameterError('top_db must be non-negative')\n            log_spec = torch.clamp(log_spec, min=log_spec.max().item() - self.top_db, max=np.inf)\n\n        return log_spec","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2024-03-29T09:03:45.748382Z","iopub.execute_input":"2024-03-29T09:03:45.748669Z","iopub.status.idle":"2024-03-29T09:03:45.794603Z","shell.execute_reply.started":"2024-03-29T09:03:45.748640Z","shell.execute_reply":"2024-03-29T09:03:45.793804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"上方代码主要是在进行频谱图的计算","metadata":{}},{"cell_type":"code","source":"# 这段代码的目的是在输入的频谱图中删除一定数量和宽度的条纹，以便在训练过程中进行数据增强=\nclass DropStripes(nn.Module):\n    def __init__(self, dim, drop_width, stripes_num):\n        \"\"\"Drop stripes. \n        Args:\n          dim: int, dimension along which to drop\n          drop_width: int, maximum width of stripes to drop\n          stripes_num: int, how many stripes to drop\n        \"\"\"\n        super(DropStripes, self).__init__()\n\n        assert dim in [2, 3]    # dim 2: time; dim 3: frequency\n\n        self.dim = dim\n        self.drop_width = drop_width\n        self.stripes_num = stripes_num\n\n    def forward(self, input):\n        \"\"\"input: (batch_size, channels, time_steps, freq_bins)\"\"\"\n\n        assert input.ndimension() == 4\n\n        if self.training is False:\n            return input\n\n        else:\n            batch_size = input.shape[0]\n            total_width = input.shape[self.dim]\n\n            for n in range(batch_size):\n                self.transform_slice(input[n], total_width)\n\n            return input\n\n\n    def transform_slice(self, e, total_width):\n        \"\"\"e: (channels, time_steps, freq_bins)\"\"\"\n\n        for _ in range(self.stripes_num):\n            distance = torch.randint(low=0, high=self.drop_width, size=(1,))[0]\n            bgn = torch.randint(low=0, high=total_width - distance, size=(1,))[0]\n\n            if self.dim == 2:\n                e[:, bgn : bgn + distance, :] = 0\n            elif self.dim == 3:\n                e[:, :, bgn : bgn + distance] = 0\n\n\n# 用于进行频谱图数据增强\nclass SpecAugmentation(nn.Module):\n    def __init__(self, time_drop_width, time_stripes_num, freq_drop_width, \n        freq_stripes_num):\n        \"\"\"Spec augmetation. \n        [ref] Park, D.S., Chan, W., Zhang, Y., Chiu, C.C., Zoph, B., Cubuk, E.D. \n        and Le, Q.V., 2019. Specaugment: A simple data augmentation method \n        for automatic speech recognition. arXiv preprint arXiv:1904.08779.\n        Args:\n          time_drop_width: int\n          time_stripes_num: int\n          freq_drop_width: int\n          freq_stripes_num: int\n        \"\"\"\n\n        super(SpecAugmentation, self).__init__()\n\n        self.time_dropper = DropStripes(dim=2, drop_width=time_drop_width, \n            stripes_num=time_stripes_num)\n\n        self.freq_dropper = DropStripes(dim=3, drop_width=freq_drop_width, \n            stripes_num=freq_stripes_num)\n\n    def forward(self, input):\n        x = self.time_dropper(input)\n        x = self.freq_dropper(x)\n        return x","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2024-03-29T09:03:45.798037Z","iopub.execute_input":"2024-03-29T09:03:45.798342Z","iopub.status.idle":"2024-03-29T09:03:45.816023Z","shell.execute_reply.started":"2024-03-29T09:03:45.798315Z","shell.execute_reply":"2024-03-29T09:03:45.815263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"进行数据增强","metadata":{}},{"cell_type":"code","source":"#使用 Xavier 初始化方法来初始化权重。如果该层具有偏置项，则将偏置项初始化为零。\ndef init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\n#该函数用于初始化Batch Normalization层的参数。将 BN 层的偏置项初始化为零。将 BN 层的权重初始化为 1.0。\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n\n#该函数用于在时间域内插值数据。通常用于补偿卷积神经网络（CNN）下采样导致的分辨率降低。\ndef interpolate(x: torch.Tensor, ratio: int):\n    \"\"\"Interpolate data in time domain. This is used to compensate the\n    resolution reduction in downsampling of a CNN.\n\n    Args:\n      x: (batch_size, time_steps, classes_num)\n      ratio: int, ratio to interpolate\n    Returns:\n      upsampled: (batch_size, time_steps * ratio, classes_num)\n    \"\"\"\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n#确保输出的频谱图与输入的帧数一致，以便在后续处理中保持一致性。\ndef pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):\n    \"\"\"Pad framewise_output to the same length as input frames. The pad value\n    is the same as the value of the last frame.\n    Args:\n      framewise_output: (batch_size, frames_num, classes_num)\n      frames_num: int, number of frames to pad\n    Outputs:\n      output: (batch_size, frames_num, classes_num)\n    \"\"\"\n    pad = framewise_output[:, -1:, :].repeat(\n        1, frames_num - framewise_output.shape[1], 1)\n    \"\"\"tensor for padding\"\"\"\n\n    output = torch.cat((framewise_output, pad), dim=1)\n    \"\"\"(batch_size, frames_num, classes_num)\"\"\"\n\n    return output\n\n#这是一个用于构建卷积层的模块\nclass ConvBlock(nn.Module):\n    def __init__(self, in_channels: int, out_channels: int):\n        super().__init__()\n\n        self.conv1 = nn.Conv2d(\n            in_channels=in_channels,\n            out_channels=out_channels,\n            kernel_size=(3, 3),\n            stride=(1, 1),\n            padding=(1, 1),\n            bias=False)\n\n        self.conv2 = nn.Conv2d(\n            in_channels=out_channels,\n            out_channels=out_channels,\n            kernel_size=(3, 3),\n            stride=(1, 1),\n            padding=(1, 1),\n            bias=False)\n\n        self.bn1 = nn.BatchNorm2d(out_channels)\n        self.bn2 = nn.BatchNorm2d(out_channels)\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_layer(self.conv1)\n        init_layer(self.conv2)\n        init_bn(self.bn1)\n        init_bn(self.bn2)\n\n    def forward(self, input, pool_size=(2, 2), pool_type='avg'):\n\n        x = input\n        x = F.relu_(self.bn1(self.conv1(x)))\n        x = F.relu_(self.bn2(self.conv2(x)))\n        if pool_type == 'max':\n            x = F.max_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg':\n            x = F.avg_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg+max':\n            x1 = F.avg_pool2d(x, kernel_size=pool_size)\n            x2 = F.max_pool2d(x, kernel_size=pool_size)\n            x = x1 + x2\n        else:\n            raise Exception('Incorrect argument!')\n\n        return x\n\n# AttBlock 是一个用于实现注意力机制的模块\nclass AttBlock(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"linear\",\n                 temperature=1.0):\n        super().__init__()\n\n        self.activation = activation\n        self.temperature = temperature\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.bn_att = nn.BatchNorm1d(out_features)\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n        init_bn(self.bn_att)\n\n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.clamp(self.att(x), -10, 10), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T09:03:45.817509Z","iopub.execute_input":"2024-03-29T09:03:45.817877Z","iopub.status.idle":"2024-03-29T09:03:45.852201Z","shell.execute_reply.started":"2024-03-29T09:03:45.817848Z","shell.execute_reply":"2024-03-29T09:03:45.851390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 定义模型\nclass PANNsCNN14Att(nn.Module):\n    def __init__(self, sample_rate: int, window_size: int, hop_size: int,\n                 mel_bins: int, fmin: int, fmax: int, classes_num: int):\n        super().__init__()\n\n        window = 'hann'\n        center = True\n        pad_mode = 'reflect'\n        ref = 1.0\n        amin = 1e-10\n        top_db = None\n        self.interpolate_ratio = 32  # Downsampled ratio\n\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(\n            n_fft=window_size,\n            hop_length=hop_size,\n            win_length=window_size,\n            window=window,\n            center=center,\n            pad_mode=pad_mode,\n            freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(\n            sr=sample_rate,\n            n_fft=window_size,\n            n_mels=mel_bins,\n            fmin=fmin,\n            fmax=fmax,\n            ref=ref,\n            amin=amin,\n            top_db=top_db,\n            freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(\n            time_drop_width=64,\n            time_stripes_num=2,\n            freq_drop_width=8,\n            freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(mel_bins)\n\n        self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)\n        self.conv_block2 = ConvBlock(in_channels=64, out_channels=128)\n        self.conv_block3 = ConvBlock(in_channels=128, out_channels=256)\n        self.conv_block4 = ConvBlock(in_channels=256, out_channels=512)\n        self.conv_block5 = ConvBlock(in_channels=512, out_channels=1024)\n        self.conv_block6 = ConvBlock(in_channels=1024, out_channels=2048)\n\n        self.fc1 = nn.Linear(2048, 2048, bias=True)\n        self.att_block = AttBlock(2048, classes_num, activation='sigmoid')\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n        \n    def cnn_feature_extractor(self, x):\n        x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block2(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block3(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block4(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block5(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block6(x, pool_size=(1, 1), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        return x\n    \n    def preprocess(self, input, mixup_lambda=None):\n        # t1 = time.time()\n        x = self.spectrogram_extractor(input)  # (batch_size, 1, time_steps, freq_bins)\n        x = self.logmel_extractor(x)  # (batch_size, 1, time_steps, mel_bins)\n\n        frames_num = x.shape[2]\n\n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n\n        if self.training:\n            x = self.spec_augmenter(x)\n\n        # Mixup on spectrogram\n        if self.training and mixup_lambda is not None:\n            x = do_mixup(x, mixup_lambda)\n        return x, frames_num\n        \n\n    def forward(self, input, mixup_lambda=None):\n        \"\"\"\n        Input: (batch_size, data_length)\"\"\"\n        x, frames_num = self.preprocess(input, mixup_lambda=mixup_lambda)\n\n        # Output shape (batch size, channels, time, frequency)\n        x = self.cnn_feature_extractor(x)\n        \n        # Aggregate in frequency axis\n        x = torch.mean(x, dim=3)\n\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.5, training=self.training)\n\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        # Get framewise output\n        framewise_output = interpolate(segmentwise_output,\n                                       self.interpolate_ratio)\n        framewise_output = pad_framewise_output(framewise_output, frames_num)\n\n        output_dict = {\n            'framewise_output': framewise_output,\n            'clipwise_output': clipwise_output\n        }\n\n        return output_dict","metadata":{"execution":{"iopub.status.busy":"2024-03-29T09:03:45.853652Z","iopub.execute_input":"2024-03-29T09:03:45.853977Z","iopub.status.idle":"2024-03-29T09:03:45.891290Z","shell.execute_reply.started":"2024-03-29T09:03:45.853948Z","shell.execute_reply":"2024-03-29T09:03:45.890318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"What is good in PANNs models is that they accept raw audio clip as input. Let's put a chunk into the CNN feature extractor of the model above.","metadata":{}},{"cell_type":"code","source":"# 尝试加载数据 \nSR = 32000\n\ny, _ = librosa.load(TRAIN_RESAMPLED_AUDIO_DIRS[0] / \"aldfly\" / \"XC134874.wav\",\n                    sr=SR,\n                    res_type=\"kaiser_fast\",\n                    mono=True)\n\nAudio(y, rate=SR)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T09:03:45.892420Z","iopub.execute_input":"2024-03-29T09:03:45.892750Z","iopub.status.idle":"2024-03-29T09:03:46.061888Z","shell.execute_reply.started":"2024-03-29T09:03:45.892721Z","shell.execute_reply":"2024-03-29T09:03:46.060683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display.waveplot(y, sr=SR);","metadata":{"execution":{"iopub.status.busy":"2024-03-29T09:03:46.063556Z","iopub.execute_input":"2024-03-29T09:03:46.064052Z","iopub.status.idle":"2024-03-29T09:03:46.245433Z","shell.execute_reply.started":"2024-03-29T09:03:46.064019Z","shell.execute_reply":"2024-03-29T09:03:46.244409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 模型参数\nmodel_config = {\n    \"sample_rate\": 32000,\n    \"window_size\": 1024,\n    \"hop_size\": 320,\n    \"mel_bins\": 64,\n    \"fmin\": 50,\n    \"fmax\": 14000,\n    \"classes_num\": 4 ##种类数量\n}\n\nmodel = PANNsCNN14Att(**model_config)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T09:03:46.246923Z","iopub.execute_input":"2024-03-29T09:03:46.247299Z","iopub.status.idle":"2024-03-29T09:03:47.637243Z","shell.execute_reply.started":"2024-03-29T09:03:46.247268Z","shell.execute_reply":"2024-03-29T09:03:47.636469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"上面需要修改种类数量\nIn `PANNsCNN14Att`, input raw waveform will be converted into log-melspectrogram using `torchlibrosa`'s utilities. I put this functionality in `PANNsCNN14Att.preprocess()` method. Let's check the output.","metadata":{}},{"cell_type":"code","source":"# 这段代码的目的是将音频数据转换为梅尔频谱图\nchunk = torch.from_numpy(y[:SR * 5]).unsqueeze(0)\nmelspec, _ = model.preprocess(chunk)\nmelspec.size()","metadata":{"execution":{"iopub.status.busy":"2024-03-29T09:03:47.638680Z","iopub.execute_input":"2024-03-29T09:03:47.639140Z","iopub.status.idle":"2024-03-29T09:03:47.722124Z","shell.execute_reply.started":"2024-03-29T09:03:47.639096Z","shell.execute_reply":"2024-03-29T09:03:47.721221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 这段代码的目的是将梅尔频谱图以图像形式展示出来，以便更好地理解音频数据的特征。\nmelspec_numpy = melspec.detach().numpy()[0, 0].transpose(1, 0)\ndisplay.specshow(melspec_numpy, sr=SR, y_axis=\"mel\");","metadata":{"execution":{"iopub.status.busy":"2024-03-29T09:03:47.723737Z","iopub.execute_input":"2024-03-29T09:03:47.724129Z","iopub.status.idle":"2024-03-29T09:03:47.855542Z","shell.execute_reply.started":"2024-03-29T09:03:47.724090Z","shell.execute_reply":"2024-03-29T09:03:47.854788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"`PANNsCNN14Att.cnn_feature_extractor()` method will take this as input and output feature map. Let's check the output of the feature extractor.","metadata":{}},{"cell_type":"code","source":"feature_map = model.cnn_feature_extractor(melspec)\nfeature_map.size()","metadata":{"execution":{"iopub.status.busy":"2024-03-29T09:03:47.856899Z","iopub.execute_input":"2024-03-29T09:03:47.857240Z","iopub.status.idle":"2024-03-29T09:03:48.206307Z","shell.execute_reply.started":"2024-03-29T09:03:47.857207Z","shell.execute_reply":"2024-03-29T09:03:48.205484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 数字对应的鸟类\nBIRD_CODE = {\n    'callandsong': 0, 'call': 1, 'song': 2, 'other': 3\n}\n\nINV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items()}\nprint(INV_BIRD_CODE)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-03-29T09:03:48.207630Z","iopub.execute_input":"2024-03-29T09:03:48.207960Z","iopub.status.idle":"2024-03-29T09:03:48.213712Z","shell.execute_reply.started":"2024-03-29T09:03:48.207927Z","shell.execute_reply":"2024-03-29T09:03:48.212941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 定义加载数据集，设置五秒为分界线\nPERIOD = 5\n\nclass PANNsDataset(data.Dataset):\n    def __init__(\n            self,\n            file_list: List[List[str]],\n            waveform_transforms=None):\n        self.file_list = file_list  # list of list: [file_path, ebird_code]\n        self.waveform_transforms = waveform_transforms\n\n    def __len__(self):\n        return len(self.file_list)\n\n    def __getitem__(self, idx: int):\n        wav_path, ebird_code = self.file_list[idx]\n\n        y, sr = sf.read(wav_path)\n\n        if self.waveform_transforms:\n            y = self.waveform_transforms(y)\n        else:\n            len_y = len(y)\n            effective_length = sr * PERIOD\n            if len_y < effective_length:\n                new_y = np.zeros(effective_length, dtype=y.dtype)\n                start = np.random.randint(effective_length - len_y)\n                new_y[start:start + len_y] = y\n                y = new_y.astype(np.float32)\n            elif len_y > effective_length:\n                start = np.random.randint(len_y - effective_length)\n                y = y[start:start + effective_length].astype(np.float32)\n            else:\n                y = y.astype(np.float32)\n\n        labels = np.zeros(len(BIRD_CODE), dtype=\"f\")\n        labels[BIRD_CODE[ebird_code]] = 1\n\n        return {\"waveform\": y, \"targets\": labels}","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-03-29T09:03:48.215194Z","iopub.execute_input":"2024-03-29T09:03:48.215478Z","iopub.status.idle":"2024-03-29T09:03:48.230758Z","shell.execute_reply.started":"2024-03-29T09:03:48.215436Z","shell.execute_reply":"2024-03-29T09:03:48.229939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Criterion","metadata":{}},{"cell_type":"code","source":"# 定义损失函数\nclass PANNsLoss(nn.Module):\n    def __init__(self):\n        super().__init__()\n\n        self.bce = nn.BCELoss()\n\n    def forward(self, input, target):\n        input_ = input[\"clipwise_output\"]\n        input_ = torch.where(torch.isnan(input_),\n                             torch.zeros_like(input_),\n                             input_)\n        input_ = torch.where(torch.isinf(input_),\n                             torch.zeros_like(input_),\n                             input_)\n\n        target = target.float()\n\n        return self.bce(input_, target)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-03-29T09:03:48.232385Z","iopub.execute_input":"2024-03-29T09:03:48.232749Z","iopub.status.idle":"2024-03-29T09:03:48.244494Z","shell.execute_reply.started":"2024-03-29T09:03:48.232713Z","shell.execute_reply":"2024-03-29T09:03:48.243707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Callbacks","metadata":{}},{"cell_type":"code","source":"# 评估F1分数\nclass F1Callback(Callback):\n    def __init__(self,\n                 input_key: str = \"targets\",\n                 output_key: str = \"logits\",\n                 model_output_key: str = \"clipwise_output\",\n                 prefix: str = \"f1\"):\n        super().__init__(CallbackOrder.Metric)\n\n        self.input_key = input_key\n        self.output_key = output_key\n        self.model_output_key = model_output_key\n        self.prefix = prefix\n\n    def on_loader_start(self, state: State):\n        self.prediction: List[np.ndarray] = []\n        self.target: List[np.ndarray] = []\n\n    def on_batch_end(self, state: State):\n        targ = state.input[self.input_key].detach().cpu().numpy()\n        out = state.output[self.output_key]\n\n        clipwise_output = out[self.model_output_key].detach().cpu().numpy()\n\n        self.prediction.append(clipwise_output)\n        self.target.append(targ)\n\n        y_pred = clipwise_output.argmax(axis=1)\n        y_true = targ.argmax(axis=1)\n\n        score = f1_score(y_true, y_pred, average=\"macro\")\n        state.batch_metrics[self.prefix] = score\n\n    def on_loader_end(self, state: State):\n        y_pred = np.concatenate(self.prediction, axis=0).argmax(axis=1)\n        y_true = np.concatenate(self.target, axis=0).argmax(axis=1)\n        score = f1_score(y_true, y_pred, average=\"macro\")\n        state.loader_metrics[self.prefix] = score\n        if state.is_valid_loader:\n            state.epoch_metrics[state.valid_loader + \"_epoch_\" +\n                                self.prefix] = score\n        else:\n            state.epoch_metrics[\"train_epoch_\" + self.prefix] = score\n\n#评估mAP分数\nclass mAPCallback(Callback):\n    def __init__(self,\n                 input_key: str = \"targets\",\n                 output_key: str = \"logits\",\n                 model_output_key: str = \"clipwise_output\",\n                 prefix: str = \"mAP\"):\n        super().__init__(CallbackOrder.Metric)\n        self.input_key = input_key\n        self.output_key = output_key\n        self.model_output_key = model_output_key\n        self.prefix = prefix\n\n    def on_loader_start(self, state: State):\n        self.prediction: List[np.ndarray] = []\n        self.target: List[np.ndarray] = []\n\n    def on_batch_end(self, state: State):\n        targ = state.input[self.input_key].detach().cpu().numpy()\n        out = state.output[self.output_key]\n\n        clipwise_output = out[self.model_output_key].detach().cpu().numpy()\n\n        self.prediction.append(clipwise_output)\n        self.target.append(targ)\n\n        score = average_precision_score(targ, clipwise_output, average=None)\n        score = np.nan_to_num(score).mean()\n        state.batch_metrics[self.prefix] = score\n\n    def on_loader_end(self, state: State):\n        y_pred = np.concatenate(self.prediction, axis=0)\n        y_true = np.concatenate(self.target, axis=0)\n        score = average_precision_score(y_true, y_pred, average=None)\n        score = np.nan_to_num(score).mean()\n        state.loader_metrics[self.prefix] = score\n        if state.is_valid_loader:\n            state.epoch_metrics[state.valid_loader + \"_epoch_\" +\n                                self.prefix] = score\n        else:\n            state.epoch_metrics[\"train_epoch_\" + self.prefix] = score","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-03-29T09:03:48.246068Z","iopub.execute_input":"2024-03-29T09:03:48.246424Z","iopub.status.idle":"2024-03-29T09:03:48.282702Z","shell.execute_reply.started":"2024-03-29T09:03:48.246388Z","shell.execute_reply":"2024-03-29T09:03:48.281817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train\n\nSome code are taken from https://www.kaggle.com/ttahara/training-birdsong-baseline-resnest50-fast .\nThanks @ttahara!","metadata":{}},{"cell_type":"code","source":"# 真正加载数据\ntmp_list = []\nfor audio_d in TRAIN_RESAMPLED_AUDIO_DIRS:\n    if not audio_d.exists():\n        continue\n    for ebird_d in audio_d.iterdir():\n        if ebird_d.is_file():\n            continue\n        for wav_f in ebird_d.iterdir():\n            tmp_list.append([ebird_d.name, wav_f.name, wav_f.as_posix()])\n            \ntrain_wav_path_exist = pd.DataFrame(\n    tmp_list, columns=[\"typebird\", \"resampled_filename\", \"file_path\"]\n)\n\ndel tmp_list\n\ntrain_all = pd.merge(\n    train, train_wav_path_exist, on=[\"resampled_filename\"], how=\"inner\")\n\nprint(train.shape)\nprint(train_wav_path_exist.shape)\n# type_x才是对的\nprint(train_all['type'])","metadata":{"execution":{"iopub.status.busy":"2024-03-29T09:03:48.284046Z","iopub.execute_input":"2024-03-29T09:03:48.284417Z","iopub.status.idle":"2024-03-29T09:03:59.675411Z","shell.execute_reply.started":"2024-03-29T09:03:48.284380Z","shell.execute_reply":"2024-03-29T09:03:59.674633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"读入数据集信息","metadata":{}},{"cell_type":"code","source":"#分层 K 折交叉验证将数据集划分为训练集和验证集，并计算不同鸟类的样本比例\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n\ntrain_all[\"fold\"] = -1\nfor fold_id, (train_index, val_index) in enumerate(skf.split(train_all, train_all[\"type\"])):\n    train_all.iloc[val_index, -1] = fold_id\n    \n# # check the propotion\nfold_proportion = pd.pivot_table(train_all, index=\"type\", columns=\"fold\", values=\"xc_id\", aggfunc=len)\nprint(fold_proportion.shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T09:03:59.676439Z","iopub.execute_input":"2024-03-29T09:03:59.676753Z","iopub.status.idle":"2024-03-29T09:03:59.765288Z","shell.execute_reply.started":"2024-03-29T09:03:59.676723Z","shell.execute_reply":"2024-03-29T09:03:59.764426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(fold_proportion)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T09:03:59.766672Z","iopub.execute_input":"2024-03-29T09:03:59.767079Z","iopub.status.idle":"2024-03-29T09:03:59.774092Z","shell.execute_reply.started":"2024-03-29T09:03:59.767037Z","shell.execute_reply":"2024-03-29T09:03:59.773260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"use_fold = 0\ntrain_file_list = train_all.query(\"fold != @use_fold\")[[\"file_path\", \"type\"]].values.tolist()\nval_file_list = train_all.query(\"fold == @use_fold\")[[\"file_path\", \"type\"]].values.tolist()\n\nprint(\"[fold {}] train: {}, val: {}\".format(use_fold, len(train_file_list), len(val_file_list)))","metadata":{"execution":{"iopub.status.busy":"2024-03-29T09:03:59.775432Z","iopub.execute_input":"2024-03-29T09:03:59.775728Z","iopub.status.idle":"2024-03-29T09:03:59.829641Z","shell.execute_reply.started":"2024-03-29T09:03:59.775694Z","shell.execute_reply":"2024-03-29T09:03:59.828830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda:0\")\n\n# loaders\nloaders = {\n    \"train\": data.DataLoader(PANNsDataset(train_file_list, None), \n                             batch_size=64, \n                             shuffle=True, \n                             num_workers=2, \n                             pin_memory=True, \n                             drop_last=True),\n    \"valid\": data.DataLoader(PANNsDataset(val_file_list, None), \n                             batch_size=64, \n                             shuffle=False,\n                             num_workers=2,\n                             pin_memory=True,\n                             drop_last=False)\n}\n\n# model\nmodel_config[\"classes_num\"] = 527\nmodel = PANNsCNN14Att(**model_config)\nweights = torch.load(\"../input/pannscnn14-decisionlevelatt-weight/Cnn14_DecisionLevelAtt_mAP0.425.pth\")\n# Fixed in V3\nmodel.load_state_dict(weights[\"model\"])\nmodel.att_block = AttBlock(2048, 4, activation='sigmoid') ##这里改过\nmodel.att_block.init_weights()\nmodel.to(device)\n\n# Optimizer\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# Scheduler\nscheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10)\n\n# Loss\ncriterion = PANNsLoss().to(device)\n\n# callbacks\ncallbacks = [\n    F1Callback(input_key=\"targets\", output_key=\"logits\", prefix=\"f1\"),\n    mAPCallback(input_key=\"targets\", output_key=\"logits\", prefix=\"mAP\"),\n    CheckpointCallback(save_n_best=0)\n]","metadata":{"execution":{"iopub.status.busy":"2024-03-29T09:03:59.830914Z","iopub.execute_input":"2024-03-29T09:03:59.831185Z","iopub.status.idle":"2024-03-29T09:04:11.719853Z","shell.execute_reply.started":"2024-03-29T09:03:59.831158Z","shell.execute_reply":"2024-03-29T09:04:11.719041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.simplefilter(\"ignore\")\n# 运行\nrunner = SupervisedRunner(\n    device=device,\n    input_key=\"waveform\",\n    input_target_key=\"targets\")\n\nrunner.train(\n    model=model,\n    criterion=criterion,\n    loaders=loaders,\n    optimizer=optimizer,\n    scheduler=scheduler,\n    num_epochs=10,\n    verbose=True,\n    logdir=f\"fold0\",\n    callbacks=callbacks,\n    main_metric=\"epoch_f1\",\n    minimize_metric=False\n)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-29T09:04:11.721192Z","iopub.execute_input":"2024-03-29T09:04:11.721564Z","iopub.status.idle":"2024-03-29T10:54:19.457559Z","shell.execute_reply.started":"2024-03-29T09:04:11.721527Z","shell.execute_reply":"2024-03-29T10:54:19.455548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Seems it's learning something.\n\nNow I'll show how this model works in the inference phase. I'll use trained model of this which I trained by myself using the data of this competition in my local environment.\n\nSince [several concerns](https://www.kaggle.com/c/birdsong-recognition/discussion/172356) are expressed about over-sharing of top solutions during competition, and since I do respect those people who have worked hard to improve their scores, I would not make trained weight in common and would not share how I trained this model.","metadata":{}},{"cell_type":"code","source":"import os\nos.chdir('/kaggle/working')\nprint(os.getcwd())\nprint(os.listdir(\"/kaggle/working\"))\nfrom IPython.display import FileLink,FileLinks\nFileLink('nao.csv')","metadata":{"execution":{"iopub.status.busy":"2024-03-29T11:26:30.182827Z","iopub.execute_input":"2024-03-29T11:26:30.183192Z","iopub.status.idle":"2024-03-29T11:26:30.192602Z","shell.execute_reply.started":"2024-03-29T11:26:30.183158Z","shell.execute_reply":"2024-03-29T11:26:30.191851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prediction with SED model","metadata":{}},{"cell_type":"code","source":"model_config = {\n    \"sample_rate\": 32000,\n    \"window_size\": 1024,\n    \"hop_size\": 320,\n    \"mel_bins\": 64,\n    \"fmin\": 50,\n    \"fmax\": 14000,\n    \"classes_num\": 4\n}\n\nweights_path = \"/kaggle/input/testpth/best.pth\"","metadata":{"execution":{"iopub.status.busy":"2024-03-29T11:23:45.430520Z","iopub.execute_input":"2024-03-29T11:23:45.430908Z","iopub.status.idle":"2024-03-29T11:23:45.436229Z","shell.execute_reply.started":"2024-03-29T11:23:45.430878Z","shell.execute_reply":"2024-03-29T11:23:45.435282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(config: dict, weights_path: str):\n    model = PANNsCNN14Att(**config)\n    checkpoint = torch.load(weights_path)\n    model.load_state_dict(checkpoint[\"model_state_dict\"])\n    device = torch.device(\"cuda\")\n    model.to(device)\n    model.eval()\n    return model","metadata":{"execution":{"iopub.status.busy":"2024-03-29T11:02:50.760309Z","iopub.execute_input":"2024-03-29T11:02:50.760664Z","iopub.status.idle":"2024-03-29T11:02:50.767106Z","shell.execute_reply.started":"2024-03-29T11:02:50.760635Z","shell.execute_reply":"2024-03-29T11:02:50.765858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prediction_for_clip(test_df: pd.DataFrame,\n                        clip: np.ndarray, \n                        model: PANNsCNN14Att,\n                        threshold=0.5):\n    PERIOD = 30\n    audios = []\n    y = clip.astype(np.float32)\n    len_y = len(y)\n    start = 0\n    end = PERIOD * SR\n    while True:\n        y_batch = y[start:end].astype(np.float32)\n        if len(y_batch) != PERIOD * SR:\n            y_pad = np.zeros(PERIOD * SR, dtype=np.float32)\n            y_pad[:len(y_batch)] = y_batch\n            audios.append(y_pad)\n            break\n        start = end\n        end += PERIOD * SR\n        audios.append(y_batch)\n        \n    array = np.asarray(audios)\n    tensors = torch.from_numpy(array)\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    \n    model.eval()\n    estimated_event_list = []\n    global_time = 0.0\n    site = test_df[\"site\"].values[0]\n    audio_id = test_df[\"audio_id\"].values[0]\n    for image in progress_bar(tensors):\n        image = image.view(1, image.size(0))\n        image = image.to(device)\n\n        with torch.no_grad():\n            prediction = model(image)\n            framewise_outputs = prediction[\"framewise_output\"].detach(\n                ).cpu().numpy()[0]\n                \n        thresholded = framewise_outputs >= threshold\n\n        for target_idx in range(thresholded.shape[1]):\n            if thresholded[:, target_idx].mean() == 0:\n                pass\n            else:\n                detected = np.argwhere(thresholded[:, target_idx]).reshape(-1)\n                head_idx = 0\n                tail_idx = 0\n                while True:\n                    if (tail_idx + 1 == len(detected)) or (\n                            detected[tail_idx + 1] - \n                            detected[tail_idx] != 1):\n                        onset = 0.01 * detected[\n                            head_idx] + global_time\n                        offset = 0.01 * detected[\n                            tail_idx] + global_time\n                        onset_idx = detected[head_idx]\n                        offset_idx = detected[tail_idx]\n                        max_confidence = framewise_outputs[\n                            onset_idx:offset_idx, target_idx].max()\n                        mean_confidence = framewise_outputs[\n                            onset_idx:offset_idx, target_idx].mean()\n                        estimated_event = {\n                            \"site\": site,\n                            \"audio_id\": audio_id,\n                            \"ebird_code\": INV_BIRD_CODE[target_idx],\n                            \"onset\": onset,\n                            \"offset\": offset,\n                            \"max_confidence\": max_confidence,\n                            \"mean_confidence\": mean_confidence\n                        }\n                        estimated_event_list.append(estimated_event)\n                        head_idx = tail_idx + 1\n                        tail_idx = tail_idx + 1\n                        if head_idx >= len(detected):\n                            break\n                    else:\n                        tail_idx += 1\n        global_time += PERIOD\n        \n    prediction_df = pd.DataFrame(estimated_event_list)\n    return prediction_df","metadata":{"execution":{"iopub.status.busy":"2024-03-29T11:02:59.542059Z","iopub.execute_input":"2024-03-29T11:02:59.542400Z","iopub.status.idle":"2024-03-29T11:02:59.565911Z","shell.execute_reply.started":"2024-03-29T11:02:59.542372Z","shell.execute_reply":"2024-03-29T11:02:59.565118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prediction(test_df: pd.DataFrame,\n               test_audio: Path,\n               model_config: dict,\n               weights_path: str,\n               threshold=0.5):\n    model = get_model(model_config, weights_path)\n    unique_audio_id = test_df.audio_id.unique()\n\n    warnings.filterwarnings(\"ignore\")\n    prediction_dfs = []\n    for audio_id in unique_audio_id:\n        with timer(f\"Loading {audio_id}\"):\n            clip, _ = librosa.load(test_audio / (audio_id + \".mp3\"),\n                                   sr=SR,\n                                   mono=True,\n                                   res_type=\"kaiser_fast\")\n        \n        test_df_for_audio_id = test_df.query(\n            f\"audio_id == '{audio_id}'\").reset_index(drop=True)\n        with timer(f\"Prediction on {audio_id}\"):\n            prediction_df = prediction_for_clip(test_df_for_audio_id,\n                                                clip=clip,\n                                                model=model,\n                                                threshold=threshold)\n\n        prediction_dfs.append(prediction_df)\n    \n    prediction_df = pd.concat(prediction_dfs, axis=0, sort=False).reset_index(drop=True)\n    return prediction_df","metadata":{"execution":{"iopub.status.busy":"2024-03-29T11:05:23.759766Z","iopub.execute_input":"2024-03-29T11:05:23.760271Z","iopub.status.idle":"2024-03-29T11:05:23.771383Z","shell.execute_reply.started":"2024-03-29T11:05:23.760231Z","shell.execute_reply":"2024-03-29T11:05:23.770441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction_df = prediction(test_df=test,\n                           test_audio=TEST_AUDIO_DIR,\n                           model_config=model_config,\n                           weights_path=weights_path,\n                           threshold=0.4)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T11:25:19.823317Z","iopub.execute_input":"2024-03-29T11:25:19.823686Z","iopub.status.idle":"2024-03-29T11:25:40.162119Z","shell.execute_reply.started":"2024-03-29T11:25:19.823658Z","shell.execute_reply":"2024-03-29T11:25:40.161026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction_df.tail(40)\nprediction_df.to_csv('nao.csv')\n","metadata":{"execution":{"iopub.status.busy":"2024-03-29T11:26:10.187548Z","iopub.execute_input":"2024-03-29T11:26:10.187928Z","iopub.status.idle":"2024-03-29T11:26:10.197091Z","shell.execute_reply.started":"2024-03-29T11:26:10.187897Z","shell.execute_reply":"2024-03-29T11:26:10.196212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Postprocess\n","metadata":{}},{"cell_type":"code","source":"labels = {}\n\nfor audio_id, sub_df in prediction_df.groupby(\"audio_id\"):\n    events = sub_df[[\"ebird_code\", \"onset\", \"offset\", \"max_confidence\", \"site\"]].values\n    n_events = len(events)\n    removed_event = []\n    # Overlap deletion: this part may not be necessary\n    # I deleted this part in other model and found there's no difference on the public LB score.\n    for i in range(n_events):\n        for j in range(n_events):\n            if i == j:\n                continue\n            if i in removed_event:\n                continue\n            if j in removed_event:\n                continue\n            \n            event_i = events[i]\n            event_j = events[j]\n            \n            if (event_i[1] - event_j[2] >= 0) or (event_j[1] - event_i[2] >= 0):\n                pass\n            else:\n                later_onset = max(event_i[1], event_j[1])\n                sooner_onset = min(event_i[1], event_j[1])\n                sooner_offset = min(event_i[2], event_j[2])\n                later_offset = max(event_i[2], event_j[2])\n\n                intersection = sooner_offset - later_onset\n                union = later_offset - sooner_onset\n                \n                iou = intersection / union\n                if iou > 0.4:\n                    if event_i[3] > event_j[3]:\n                        removed_event.append(j)\n                    else:\n                        removed_event.append(i)\n\n    site = events[0][4]\n    for i in range(n_events):\n        if i in removed_event:\n            continue\n        event = events[i][0]\n        onset = events[i][1]\n        offset = events[i][2]\n        if site in {\"site_1\", \"site_2\"}:\n            start_section = int((onset // 5) * 5) + 5\n            end_section = int((offset // 5) * 5) + 5\n            cur_section = start_section\n\n            row_id = f\"{site}_{audio_id}_{start_section}\"\n            if labels.get(row_id) is not None:\n                labels[row_id].add(event)\n            else:\n                labels[row_id] = set()\n                labels[row_id].add(event)\n\n            while cur_section != end_section:\n                cur_section += 5\n                row_id = f\"{site}_{audio_id}_{cur_section}\"\n                if labels.get(row_id) is not None:\n                    labels[row_id].add(event)\n                else:\n                    labels[row_id] = set()\n                    labels[row_id].add(event)\n        else:\n            row_id = f\"{site}_{audio_id}\"\n            if labels.get(row_id) is not None:\n                labels[row_id].add(event)\n            else:\n                labels[row_id] = set()\n                labels[row_id].add(event)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T11:17:48.482262Z","iopub.execute_input":"2024-03-29T11:17:48.482598Z","iopub.status.idle":"2024-03-29T11:17:48.532153Z","shell.execute_reply.started":"2024-03-29T11:17:48.482570Z","shell.execute_reply":"2024-03-29T11:17:48.531156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for key in labels:\n    labels[key] = \" \".join(sorted(list(labels[key])))\n\n    \nrow_ids = list(labels.keys())\nbirds = list(labels.values())\npost_processed = pd.DataFrame({\n    \"row_id\": row_ids,\n    \"birds\": birds\n})\npost_processed.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-29T11:17:52.446028Z","iopub.execute_input":"2024-03-29T11:17:52.446394Z","iopub.status.idle":"2024-03-29T11:17:52.460237Z","shell.execute_reply.started":"2024-03-29T11:17:52.446359Z","shell.execute_reply":"2024-03-29T11:17:52.459382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_row_id = test[[\"row_id\"]]\nsubmission = all_row_id.merge(post_processed, on=\"row_id\", how=\"left\")\nsubmission = submission.fillna(\"nocall\")\nsubmission.to_csv(\"submission.csv\", index=False)\nsubmission.head(40)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T11:17:54.787035Z","iopub.execute_input":"2024-03-29T11:17:54.787415Z","iopub.status.idle":"2024-03-29T11:17:54.812720Z","shell.execute_reply.started":"2024-03-29T11:17:54.787380Z","shell.execute_reply":"2024-03-29T11:17:54.811881Z"},"trusted":true},"execution_count":null,"outputs":[]}]}