{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":12072618,"sourceType":"datasetVersion","datasetId":7599446},{"sourceId":426034,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":347293,"modelId":368546},{"sourceId":426046,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":347302,"modelId":368556}],"dockerImageVersionId":31040,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Inference of our own Model","metadata":{}},{"cell_type":"markdown","source":"# Data Loading","metadata":{}},{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"code","source":"import os\n\nimport torch\nimport torchaudio\n\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchaudio.transforms import MelSpectrogram\n\nfrom pathlib import Path","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T18:06:58.356918Z","iopub.execute_input":"2025-06-05T18:06:58.357244Z","iopub.status.idle":"2025-06-05T18:07:05.570402Z","shell.execute_reply.started":"2025-06-05T18:06:58.357222Z","shell.execute_reply":"2025-06-05T18:07:05.569388Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"class BirdCLEFInferenceDataset(Dataset):\n    \"\"\"\n    Dataset for 1-minute test soundscape files.\n    Each file is split into 12 × 5-second segments.\n    Returns Mel spectrogram and a row_id like 'soundscape_1234_5'.\n    \"\"\"\n\n    def __init__(\n        self,\n        audio_dir: str | Path,\n        wave_sec: int = 5,\n        sample_rate: int = 32000,\n        n_fft: int = 1024,\n        win_length: int = 1024,\n        hop_length: int = 512,\n        f_min: int = 50,\n        f_max: int = 16000,\n        n_mels: int = 128\n    ):\n        self.audio_dir = Path(audio_dir)\n        self.audio_files = sorted(list(self.audio_dir.glob(\"*.ogg\")))  # or *.wav if needed\n        self.wave_sec = wave_sec\n        self.sample_rate = sample_rate\n        self.chunk_samples = sample_rate * wave_sec\n\n        self.spectrogram_transform = MelSpectrogram(\n            sample_rate=sample_rate,\n            n_fft=n_fft,\n            win_length=win_length,\n            hop_length=hop_length,\n            center=True,\n            f_min=f_min,\n            f_max=f_max,\n            pad_mode=\"reflect\",\n            power=2.0,\n            norm='slaney',\n            n_mels=n_mels,\n            mel_scale=\"htk\",\n        )\n\n    @staticmethod\n    def normalize_std(spec: torch.Tensor, eps: float = 1e-6) -> torch.Tensor:\n        mean = torch.mean(spec)\n        std = torch.std(spec)\n        return torch.where(std == 0, spec - mean, (spec - mean) / (std + eps))\n\n    def __len__(self) -> int:\n        return len(self.audio_files) * 12  # 12 chunks per file\n\n    def __getitem__(self, idx: int):\n        file_idx = idx // 12  # which file\n        chunk_idx = idx % 12  # which 5s segment in the file\n\n        filepath = self.audio_files[file_idx]\n        filename_stem = filepath.stem  # 'soundscape_8358733'\n        row_id = f\"{filename_stem}_{(chunk_idx + 1) * 5}\"\n\n        waveform, _ = torchaudio.load(filepath, backend=\"soundfile\")\n        waveform = waveform[0, :].unsqueeze(0)  # mono (1, samples)\n\n        start = chunk_idx * self.chunk_samples\n        end = start + self.chunk_samples\n        chunk = waveform[:, start:end]\n\n        if chunk.shape[1] < self.chunk_samples:\n            pad = self.chunk_samples - chunk.shape[1]\n            chunk = torch.nn.functional.pad(chunk, (0, pad))\n\n        melspec = self.spectrogram_transform(chunk)\n        melspec = torch.log(melspec + 1e-6)\n        melspec = self.normalize_std(melspec)\n\n        return melspec, row_id","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T18:07:05.572220Z","iopub.execute_input":"2025-06-05T18:07:05.572730Z","iopub.status.idle":"2025-06-05T18:07:05.585889Z","shell.execute_reply.started":"2025-06-05T18:07:05.572705Z","shell.execute_reply":"2025-06-05T18:07:05.584806Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Loading Data","metadata":{}},{"cell_type":"code","source":"dataset = BirdCLEFInferenceDataset(\n    audio_dir=\"/kaggle/input/birdclef-2025/test_soundscapes\"\n)\n\nloader = DataLoader(\n    dataset, \n    batch_size=1, \n    shuffle=False, \n    num_workers=2\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T18:07:05.587165Z","iopub.execute_input":"2025-06-05T18:07:05.587527Z","iopub.status.idle":"2025-06-05T18:07:05.881416Z","shell.execute_reply.started":"2025-06-05T18:07:05.587488Z","shell.execute_reply":"2025-06-05T18:07:05.880334Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# having a look\nfor mel, row_id in loader:\n    print(mel.shape)\n    print(row_id)\n    break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T18:07:05.883839Z","iopub.execute_input":"2025-06-05T18:07:05.884350Z","iopub.status.idle":"2025-06-05T18:07:06.269417Z","shell.execute_reply.started":"2025-06-05T18:07:05.884257Z","shell.execute_reply":"2025-06-05T18:07:06.268382Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"import datetime\nimport tqdm\n\nimport timm\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\n\nfrom typing import Dict, Tuple, Any, Optional","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T18:07:06.270607Z","iopub.execute_input":"2025-06-05T18:07:06.271003Z","iopub.status.idle":"2025-06-05T18:07:15.812418Z","shell.execute_reply.started":"2025-06-05T18:07:06.270964Z","shell.execute_reply":"2025-06-05T18:07:15.811224Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def init_layer(layer: nn.Module): \n    \"\"\"\n    Applies Xavier uniform initialization to the weights.\n    To have better variance for the initial weights which leads to avoiding\n    e.g. sigmoids weaknesses of getting linear for weights close to 0.\n    \"\"\"\n    nn.init.xavier_uniform_(layer.weight)\n\n    # for beeing unbiased in the beginning if we have a bias\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.0)\n            \n\nclass AttBlockV2(nn.Module):\n    \"\"\"\n    Implements a temporal attention mechanism for audio classification.\n\n    This block takes a sequence of features (frame-level embeddings from a backbone CNN) and\n    computes a weighted average of these features to produce a single, clip-level representation.\n    The attention weights are learned, indicating which parts of the audio sequence are most\n    relevant for each class. This allows the model to selectively focus on informative segments\n    (e.g., where a bird call is present) rather than treating all time segments equally.\n    This is crucial for sound event detection and classification where target sounds might be\n    sparse within a long recording.\n    \"\"\"\n    def __init__(\n        self,\n        in_features: int,\n        out_features: int,\n        activation: str = \"sigmoid\"\n    ) -> None:\n        \"\"\"\n        Initializes the AttBlockV2.\n\n        Args:\n            in_features: Number of input features per time step from the backbone.\n            out_features: Number of output classes.\n            activation: Activation function to apply to the classification branch ('linear' or 'sigmoid').\n        \"\"\"\n        super().__init__()\n\n        self.activation = activation\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True,\n        )\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True,\n        )\n\n        self.init_weights()\n\n    def init_weights(self) -> None:\n        \"\"\"\n        Initializes weights of the attention and classification convolutional layers.\n        \"\"\"\n        init_layer(self.att)\n        init_layer(self.cla)\n\n    def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:\n        \"\"\"\n        Performs the forward pass through the attention block.\n\n        Args:\n            x: Input tensor of shape (batch_size, in_features, n_time).\n\n        Returns:\n            A tuple containing:\n            - clipwise_output: (batch_size, out_features) - The clip-level aggregated features for each class.\n            - norm_att: (batch_size, out_features, n_time) - Normalized attention weights across time.\n            - cla_output: (batch_size, out_features, n_time) - Frame-level classification features (before weighted sum).\n        \"\"\"\n        # to learn attention scores \n        norm_att = torch.softmax(torch.tanh(self.att(x)), dim=-1)\n\n        # frame-level class scores\n        cla_output = self.nonlinear_transform(self.cla(x))\n\n        # combine those\n        clipwise_output = torch.sum(norm_att * cla_output, dim=2)\n\n        return clipwise_output, norm_att, cla_output\n\n    def nonlinear_transform(self, x: torch.Tensor) -> torch.Tensor:\n        \"\"\"\n        Applies the specified activation function to the input.\n\n        Args:\n            x: Input tensor.\n\n        Returns:\n            Output tensor after applying activation.\n        \"\"\"\n        if self.activation == \"linear\":\n            return x\n        elif self.activation == \"sigmoid\":\n            return torch.sigmoid(x)\n        else:\n            raise NotImplementedError(f\"Activation '{self.activation}' not supported.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T18:07:15.813961Z","iopub.execute_input":"2025-06-05T18:07:15.814504Z","iopub.status.idle":"2025-06-05T18:07:15.828432Z","shell.execute_reply.started":"2025-06-05T18:07:15.814473Z","shell.execute_reply":"2025-06-05T18:07:15.826001Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def image_delta(x: torch.Tensor) -> torch.Tensor:\n    \"\"\"\n    Computes first and second order differences along the time axis of a spectrogram.\n    This expands a single-channel spectrogram into a 3-channel (original, delta, delta-delta) input.\n    delta --> difference in mel energy betwen time frames \n\n    Args:\n        x: Input spectrogram tensor of shape (batch, 1, freq, time).\n\n    Returns:\n        A tensor of shape (batch, 3, freq, time) with original, delta, and delta-delta channels.\n    \"\"\"\n    if x.shape[1] != 1:\n        raise ValueError(\"image_delta expects input with 1 channel for this placeholder.\")\n\n    diff1_raw = x[:, :, :, 1:] - x[:, :, :, :-1]\n    delta1 = F.pad(diff1_raw, (0, 1, 0, 0), 'constant', 0)\n    \n    diff2_raw = delta1[:, :, :, 1:] - delta1[:, :, :, :-1]\n    delta2 = F.pad(diff2_raw, (0, 1, 0, 0), 'constant', 0)\n\n    return torch.cat([x, delta1, delta2], dim=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T18:07:15.829510Z","iopub.execute_input":"2025-06-05T18:07:15.829899Z","iopub.status.idle":"2025-06-05T18:07:15.861471Z","shell.execute_reply.started":"2025-06-05T18:07:15.829876Z","shell.execute_reply":"2025-06-05T18:07:15.860409Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BirdCLEFModel(nn.Module):\n    \"\"\"\n    A trainable PyTorch model for BirdCLEF audio classification.\n\n    This model takes Mel spectrograms as input, processes them through a\n    backbone CNN for feature extraction, and then uses an attention mechanism\n    to aggregate temporal features for clip-level classification.\n    \"\"\"\n    def __init__(self, cfg: Dict[str, Any]) -> None:\n        \"\"\"\n        Initializes the BirdCLEFModel.\n\n        Args:\n            cfg: A dictionary containing model configuration parameters, e.g.:\n                - 'n_mels': Number of Mel bins (frequency dimension).\n                - 'model_name': Name of the timm backbone model (e.g., 'efficientnet_b0').\n                - 'SR': Sample rate (used for infer_duration calculation).\n                - 'duration_train': Duration of training audio segments in seconds.\n                - 'infer_duration': Duration of inference audio segments in seconds.\n                - 'device': 'cuda' or 'cpu'.\n        \"\"\"\n        super().__init__()\n        self.cfg = cfg\n\n        taxonomy_df = pd.read_csv('/kaggle/input/birdclef-2025/taxonomy.csv')\n        self.num_classes = len(taxonomy_df) \n\n        self.bn0 = nn.BatchNorm2d(cfg['n_mels']) \n\n        self.backbone = timm.create_model(\n            cfg['model_name'],\n            pretrained=False, # was false but True for imagenet base\n            in_chans=3,\n            drop_rate=0.2,\n            drop_path_rate=0.2,\n        )\n\n        layers = list(self.backbone.children())[:-2]\n        self.encoder = nn.Sequential(*layers)\n\n        # Determine the number of output features from the backbone's encoder\n        if \"efficientnet\" in self.cfg['model_name']:\n            backbone_out = self.backbone.classifier.in_features\n        elif \"eca\" in self.cfg['model_name']:\n            backbone_out = self.backbone.head.fc.in_features\n        elif \"res\" in self.cfg['model_name']:\n            backbone_out = self.backbone.fc.in_features\n        else:\n            raise NotImplementedError(f\"Model: '{cfg['model_name']}' not supported.\")\n\n        self.fc1 = nn.Linear(backbone_out, backbone_out, bias=True)\n\n        self.att_block = AttBlockV2(backbone_out, self.num_classes, activation=\"sigmoid\")\n\n    def extract_feature(self, x: torch.Tensor) -> Tuple[torch.Tensor, int]:\n        \"\"\"\n        Extracts frame-level features from the spectrogram using the backbone encoder.\n\n        Args:\n            x: Input spectrogram tensor of shape (batch_size, channels, n_mels, n_frames).\n\n        Returns:\n            A tuple containing:\n            - Features tensor: (batch_size, backbone_out_features, n_frames_reduced)\n            - Original number of time frames.\n        \"\"\"\n        original_frames_num = x.shape[3]\n\n        x = x.transpose(1, 2)\n        x = self.bn0(x)\n        x = x.transpose(1, 2)\n\n        # The pretrained model from timm without head\n        x = self.encoder(x) \n\n        x = torch.mean(x, dim=2) \n\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n\n        x = F.dropout(x, p=0.5, training=self.training)\n\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2) \n\n        x = F.dropout(x, p=0.5, training=self.training)\n\n        return x, original_frames_num\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        \"\"\"\n        Forward pass for training.\n\n        Args:\n            x: Input Mel spectrogram tensor of shape (batch_size, channels, n_mels, n_frames).\n               Channels will be 1 for mono, to apply the image detla after it.\n               The model assumes normalization (e.g., to [0,1]) has already been applied.\n\n        Returns:\n            Logits for clip-wise classification output of shape (batch_size, num_classes).\n        \"\"\"\n        if x.shape[1] != 1: \n            raise ValueError(f\"Warning: Model expects 1 channel, but input has {x.shape[1]}.\")\n            \n        x = image_delta(x)\n        features, _ = self.extract_feature(x)\n\n        (clipwise_output, _, _) = self.att_block(features)\n\n        return torch.logit(clipwise_output)\n\n    def attention_infer(self, start: int, end: int, x: torch.Tensor) -> torch.Tensor:\n        \"\"\"\n        Helper function for inference, computing framewise predictions from a slice of features.\n\n        Args:\n            start: Start frame index for the feature slice.\n            end: End frame index for the feature slice.\n            x: Feature tensor of shape (batch_size, features, n_time).\n\n        Returns:\n            Max framewise probabilities for each class (batch_size, num_classes).\n        \"\"\"\n        feat_slice = x[:, :, start:end]\n        # Get framewise probabilities from the classification branch\n        framewise_pred = torch.sigmoid(self.att_block.cla(feat_slice))\n        # Take the maximum probability across frames for each class\n        framewise_pred_max = framewise_pred.max(dim=2)[0]\n        return framewise_pred_max\n\n    def infer(self, x: torch.Tensor, tta_delta: int = 2) -> torch.Tensor:\n        \"\"\"\n        Performs inference with Test Time Augmentation (TTA).\n\n        Args:\n            x: Input Mel spectrogram tensor of shape (batch_size, channels, n_mels, n_frames).\n            tta_delta: Number of frames to shift for TTA.\n\n        Returns:\n            Averaged clip-wise predictions (probabilities) of shape (batch_size, num_classes).\n        \"\"\"\n        with torch.no_grad():\n            if x.shape[1] != 1:\n               raise ValueError(f\"Warning: Model expects 1 channel, but input has {x.shape[1]}.\")\n                \n            x = image_delta(x)\n            features, _ = self.extract_feature(x)\n\n            feat_time = features.size(-1)\n\n            # Calculate central crop start and end points\n            start = int(feat_time / 2 - feat_time * (self.cfg['infer_duration'] / self.cfg['duration_train']) / 2)\n            end = int(start + feat_time * (self.cfg['infer_duration'] / self.cfg['duration_train']))\n\n            # Base --> Get prediction for central crop\n            pred = self.attention_infer(start, end, features)\n\n            # TTA --> shifted\n            start_minus = max(0, start - tta_delta)\n            end_minus = end - tta_delta\n            pred_minus = self.attention_infer(start_minus, end_minus, features)\n\n            start_plus = start + tta_delta\n            end_plus = min(feat_time, end + tta_delta)\n            pred_plus = self.attention_infer(start_plus, end_plus, features)\n\n            # combinbe it\n            final_pred = 0.5 * pred + 0.25 * pred_minus + 0.25 * pred_plus\n            return final_pred","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T18:07:15.862700Z","iopub.execute_input":"2025-06-05T18:07:15.863047Z","iopub.status.idle":"2025-06-05T18:07:15.888980Z","shell.execute_reply.started":"2025-06-05T18:07:15.863015Z","shell.execute_reply":"2025-06-05T18:07:15.887936Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Inference Loop","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport time","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T18:07:15.890364Z","iopub.execute_input":"2025-06-05T18:07:15.890748Z","iopub.status.idle":"2025-06-05T18:07:16.500970Z","shell.execute_reply.started":"2025-06-05T18:07:15.890721Z","shell.execute_reply":"2025-06-05T18:07:16.500150Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cfg = {\n    'csv_path': '/kaggle/input/birdclef-2025/train.csv',\n    'audio_path': '/kaggle/input/birdclef-2025/train_audio',\n    'SR': 32000,\n    'target_duration': 10.0,\n    'hop_length': 512, #320,\n    'win_length': 1024,\n    'n_mels': 128,\n    'f_min': 20,\n    'f_max': 16000,\n    'n_fft': 1024, #2048,\n    'device': 'cuda' if torch.cuda.is_available() else 'cpu',\n    'in_channels': 1,\n    'model_name': 'efficientnet_b0',\n    'duration_train': 5,\n    'infer_duration': 5,\n    'num_workers': 4,\n\n    'model_weights_path': '/kaggle/input/model/pytorch/default/1/birdclef_model.pth'\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T18:07:16.503225Z","iopub.execute_input":"2025-06-05T18:07:16.503665Z","iopub.status.idle":"2025-06-05T18:07:16.512642Z","shell.execute_reply.started":"2025-06-05T18:07:16.503644Z","shell.execute_reply":"2025-06-05T18:07:16.511630Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Loading model: {cfg['model_name']}\")\nmodel = BirdCLEFModel(cfg).to(cfg['device'])\n\nprint(\"Class Labels ...\")\nclass_labels = sorted(os.listdir('../input/birdclef-2025/train_audio/'))\n\n#class_labels = loader.dataset.idx2label\n\nprint(f\"Loading weights from: {cfg['model_weights_path']}\")\nmodel.load_state_dict(torch.load(cfg['model_weights_path'], map_location=cfg['device']))\nprint(\"Model weights loaded successfully.\")\n\nprint(\"Create pred\")\npred_dict = {'row_id': []}\nfor species_code in class_labels:\n    pred_dict[species_code] = []\n\nmodel.eval() \n\nstart_time = time.time()\n\nwith torch.no_grad():\n    for mel_specs, row_id in loader:\n        mel_specs = mel_specs.to(cfg['device'])\n        row_id = row_id[0]\n        \n        probs = model.infer(mel_specs)\n        \n        probs_np = probs.cpu().numpy()[0]\n        \n        pred_dict['row_id'].append(row_id)\n        for class_idx, class_name in enumerate(class_labels):\n            pred_dict[class_name].append(probs_np[class_idx])\n\nend_time = time.time() \nduration = end_time - start_time \n\nprint(\"Chunk inference complete.\")\nprint(f\"Inference duration: {duration:.2f} seconds\") ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T18:08:48.183753Z","iopub.execute_input":"2025-06-05T18:08:48.184516Z","iopub.status.idle":"2025-06-05T18:08:48.727354Z","shell.execute_reply.started":"2025-06-05T18:08:48.184474Z","shell.execute_reply":"2025-06-05T18:08:48.726113Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission Prep","metadata":{}},{"cell_type":"code","source":"results_df = pd.DataFrame(pred_dict)\nfinal_columns = ['row_id'] + class_labels\nresults_df = results_df[final_columns]\n\nresults_df.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T18:08:52.790617Z","iopub.execute_input":"2025-06-05T18:08:52.790961Z","iopub.status.idle":"2025-06-05T18:08:52.826133Z","shell.execute_reply.started":"2025-06-05T18:08:52.790932Z","shell.execute_reply":"2025-06-05T18:08:52.825135Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}