{"metadata":{"kernelspec":{"display_name":"venv","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.3"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nimport pandas as pd\nimport os\nimport numpy as np\nfrom torch.utils.data.sampler import WeightedRandomSampler\nfrom timm.scheduler import CosineLRScheduler\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torch.cuda.amp import GradScaler, autocast\nfrom tqdm import tqdm\nimport copy\nimport time\nimport os\nimport numpy as np\nimport gc\nimport torch\nfrom sklearn.model_selection import StratifiedKFold, StratifiedGroupKFold\nfrom warmup_scheduler import GradualWarmupScheduler\nfrom torch.optim import AdamW\nimport torch\nimport numpy as np\nimport sklearn\nimport torchaudio\nimport torchaudio.transforms as T\nimport warnings\nimport random\nwarnings.filterwarnings(\"ignore\")\nimport audiomentations as AA\nimport albumentations as A\nimport librosa\nimport librosa.display\nimport torch\nimport torch.nn.functional as F\nimport numpy as np\nfrom torchvision import transforms\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport soundfile as sf\nimport cv2\nimport numpy as np\nimport random\nimport timm\nfrom utils.metrics import MetricMeter\nfrom utils.data_utils import downsample_data, upsample_data\nfrom torch.amp import autocast, GradScaler\nimport torchvision\nfrom torchvision.transforms import v2\nimport json\nimport time","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub = pd.read_csv(f\"../../../data/birdclef-2025/sample_submission.csv\")\ntarget_columns = sub.columns.tolist()[1:]","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fix Warmup Bug\nclass GradualWarmupSchedulerV2(GradualWarmupScheduler):\n    def __init__(self, optimizer, multiplier, total_epoch, after_scheduler=None):\n        super(GradualWarmupSchedulerV2, self).__init__(optimizer, multiplier, total_epoch, after_scheduler)\n    def get_lr(self):\n        if self.last_epoch > self.total_epoch:\n            if self.after_scheduler:\n                if not self.finished:\n                    self.after_scheduler.base_lrs = [base_lr * self.multiplier for base_lr in self.base_lrs]\n                    self.finished = True\n                return self.after_scheduler.get_lr()\n            return [base_lr * self.multiplier for base_lr in self.base_lrs]\n        if self.multiplier == 1.0:\n            return [base_lr * (float(self.last_epoch) / self.total_epoch) for base_lr in self.base_lrs]\n        else:\n            return [base_lr * ((self.multiplier - 1.) * self.last_epoch / self.total_epoch + 1.) for base_lr in self.base_lrs]\n        \n\ndef padded_cmap(solution, submission, padding_factor=5):\n    solution = solution#.drop(['row_id'], axis=1, errors='ignore')\n    submission = submission#.drop(['row_id'], axis=1, errors='ignore')\n    new_rows = []\n    for i in range(padding_factor):\n        new_rows.append([1 for i in range(len(solution.columns))])\n    new_rows = pd.DataFrame(new_rows)\n    new_rows.columns = solution.columns\n    padded_solution = pd.concat([solution, new_rows]).reset_index(drop=True).copy()\n    padded_submission = pd.concat([submission, new_rows]).reset_index(drop=True).copy()\n    score = sklearn.metrics.average_precision_score(\n        padded_solution.values,\n        padded_submission.values,\n        average='macro',\n    )\n    return score\n\ndef map_score(solution, submission):\n    solution = solution#.drop(['row_id'], axis=1, errors='ignore')\n    submission = submission#.drop(['row_id'], axis=1, errors='ignore')\n    score = sklearn.metrics.average_precision_score(\n        solution.values,\n        submission.values,\n        average='micro',\n    )\n    return score\n\ndef calculate_competition_metrics(gt, preds, target_columns):\n    val_df = pd.DataFrame(gt, columns=target_columns)\n    pred_df = pd.DataFrame(preds, columns=target_columns)\n    cmAP_1 = padded_cmap(val_df, pred_df, padding_factor=1)\n    cmAP_5 = padded_cmap(val_df, pred_df, padding_factor=5)\n    mAP = map_score(val_df, pred_df)\n    val_df['id'] = [f'id_{i}' for i in range(len(val_df))]\n    pred_df['id'] = [f'id_{i}' for i in range(len(pred_df))]\n    return {\n        \"cmAP_1\": cmAP_1,\n        \"cmAP_5\": cmAP_5,\n        \"mAP\": mAP,\n    }\ndef metrics_to_string(scores, key_word):\n  log_info = \"\"\n  for key in scores.keys():\n      log_info = log_info + f\"{key_word} {key} : {scores[key]:.4f}, \"\n  return log_info\n\n\ndef mixup(data, targets, alpha):\n    indices = torch.randperm(data.size(0))\n    data2 = data[indices]\n    targets2 = targets[indices]\n\n    lam = torch.FloatTensor([np.random.beta(alpha, alpha)])\n    data = data * lam + data2 * (1 - lam)\n    targets = targets * lam + targets2 * (1 - lam)\n\n    return data, targets\n\n\ndef set_seed(seed=42):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.mps.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport copy\nimport torch\nimport numpy as np\nfrom collections import OrderedDict\n\nclass ModelSoup:\n    \"\"\"\n    Implementation of Model Soup technique that averages weights of models\n    that improve validation performance.\n    \n    Reference: https://arxiv.org/abs/2203.05482\n    \"\"\"\n    def __init__(self, base_model, save_dir=\"model_soup\", exclude_nocall=False, max_trick=False):\n        \"\"\"\n        Initialize Model Soup with a base model\n        \n        Args:\n            base_model: Initial model to start with\n            save_dir: Directory to save individual models\n        \"\"\"\n        self.base_model = copy.deepcopy(base_model)\n        self.best_models = OrderedDict()  # Maps epoch -> model state dict\n        self.best_val_score = float('-inf')\n        self.initial_val_score = None\n        self.save_dir = save_dir\n        self.max_trick = max_trick\n        self.exclude_nocall = exclude_nocall\n        \n        # Create save directory if it doesn't exist\n        os.makedirs(save_dir, exist_ok=True)\n        \n    def evaluate_model(self, unseen_index, model, val_loader, criterion, device):\n        \"\"\"\n        Evaluate model on validation set\n        \n        Args:\n            model: Model to evaluate\n            val_loader: Validation data loader\n            criterion: Loss function\n            device: Device to run evaluation on\n            \n        Returns:\n            float: Validation accuracy\n        \"\"\"\n        if unseen_index is not None:\n            scores = MetricMeter(indices_ignore=unseen_index)\n        model.eval()\n        val_loss = []\n        PREDS = []\n        TRUES = []\n        bar = tqdm(val_loader)\n        with torch.no_grad():\n            for batch in bar:\n\n                target = batch['target'].to(device)\n                spec = batch['spec'].to(device)\n\n                if self.max_trick:\n                    BS, K, C, H, W = spec.shape\n                    spec = spec.view(BS * K, C, H, W)\n\n                \n                predictions = model(spec)\n                if self.max_trick:\n                    predictions = predictions.view(BS, K, -1)\n                    predictions = torch.max(predictions, dim=1).values\n\n                \n                if self.exclude_nocall:\n                    target = target[:, :-1]\n                    predictions = predictions[:, :-1]\n\n                loss = criterion(predictions, target)\n                \n                val_loss.append(loss.detach().cpu().numpy())\n                PREDS.append(predictions.detach().cpu())\n                \n                target = torch.round(target).long()\n\n                TRUES.append(target.detach().cpu())\n                \n\n                if unseen_index is not None:\n                    scores.update(target, predictions)\n                \n                loss_np = loss.detach().cpu().item()\n                bar.set_description('loss: %.5f' % (loss_np))\n\n                \n        val_loss = np.mean(val_loss)\n        P = np.concatenate(PREDS, axis=0)\n        T = np.concatenate(TRUES, axis=0)\n        metrics = calculate_competition_metrics(T, P, target_columns)\n        metrics['score'] = scores.avg\n        return metrics\n    \n    def add_model_if_improved(self, unseen_index, model, epoch, val_loader, device, criterion):\n        \"\"\"\n        Add model to soup if it improves validation performance\n        \n        Args:\n            model: Current model\n            epoch: Current epoch\n            val_loader: Validation data loader\n            criterion: Loss function\n            device: Device to run evaluation on\n            \n        Returns:\n            bool: Whether model was added to soup\n        \"\"\"\n        scores = self.evaluate_model(unseen_index, model, val_loader, criterion, device)\n        current_score = scores\n        \n        \n        # Set initial validation score if not set\n        if self.initial_val_score is None:\n            self.initial_val_score = current_score\n            self.best_val_score = current_score\n            \n            # Save initial model\n            self.best_models[epoch] = copy.deepcopy(model.state_dict())\n            torch.save(model.state_dict(), os.path.join(self.save_dir, f\"model_epoch_{epoch}.pt\"))\n            print(f\"Initial model (Epoch {epoch}) - Validation Score: {json.dumps(current_score)}\")\n            return True\n        print(f\"Epoch {epoch} - Validation Scores: {json.dumps(current_score)} (Best: {self.best_val_score})\")\n        \n        # Only add to soup if validation score improves\n        is_better = False\n        for each_key in current_score.keys():\n            if current_score[each_key] > self.best_val_score[each_key]:\n                is_better = True\n                break\n        \n\n\n        if is_better:\n            self.best_val_score = current_score\n            self.best_models[epoch] = copy.deepcopy(model.state_dict())\n            \n            # Save individual model\n            torch.save(model.state_dict(), os.path.join(self.save_dir, f\"model_epoch_{epoch}.pt\"))\n            \n            print(f\"Model from epoch {epoch} added to soup! New best scores: {json.dumps(current_score)}\")\n            return True\n        \n        return False\n    \n    def get_soup_model(self, soup_model_name = \"model_soup_final.pt\"):\n        \"\"\"\n        Create model soup by averaging weights of best models\n        \n        Returns:\n            torch.nn.Module: Model with averaged weights\n        \"\"\"\n        if not self.best_models:\n            print(\"No models in soup yet!\")\n            return self.base_model\n        \n        # Create a new model with the same architecture\n        # Create a new model with the same architecture\n        soup_model = copy.deepcopy(self.base_model)\n\n        # Get the first model to initialize parameters\n        first_model_state = next(iter(self.best_models.values()))\n\n        # Initialize averaged state dict\n        avg_state_dict = OrderedDict()\n        for key in first_model_state.keys():\n            avg_state_dict[key] = torch.zeros_like(first_model_state[key])\n\n        # Sum all parameters\n        for epoch, state_dict in self.best_models.items():\n            for key in state_dict.keys():\n                avg_state_dict[key] += state_dict[key]\n\n        # Divide by number of models to get average\n        num_models = len(self.best_models)\n        for key in avg_state_dict.keys():\n            avg_state_dict[key] = avg_state_dict[key] / int(num_models)\n\n        # Load averaged parameters into the soup model\n        soup_model.load_state_dict(avg_state_dict)\n        \n        # Save the final soup model\n        torch.save(avg_state_dict, os.path.join(self.save_dir, soup_model_name))\n        \n        print(f\"Created model soup from {num_models} models (epochs: {list(self.best_models.keys())})\")\n        return soup_model\n\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Audio to Spec","metadata":{}},{"cell_type":"code","source":"import torch\nimport torchaudio\nimport torch.nn as nn\nimport torchvision.transforms.v2 as v2\nimport torchvision.transforms.functional as F\nimport random\nimport numpy as np\nfrom torch.nn import functional as F_nn\n\n\ndef compute_deltas(\n    specgram: torch.Tensor, win_length: int = 5, mode: str = \"replicate\"\n) -> torch.Tensor:\n    r\"\"\"Compute delta coefficients of a tensor, usually a spectrogram:\n\n    .. math::\n       d_t = \\frac{\\sum_{n=1}^{\\text{N}} n (c_{t+n} - c_{t-n})}{2 \\sum_{n=1}^{\\text{N}} n^2}\n\n    where :math:`d_t` is the deltas at time :math:`t`,\n    :math:`c_t` is the spectrogram coeffcients at time :math:`t`,\n    :math:`N` is ``(win_length-1)//2``.\n\n    Args:\n        specgram (Tensor): Tensor of audio of dimension (..., freq, time)\n        win_length (int, optional): The window length used for computing delta (Default: ``5``)\n        mode (str, optional): Mode parameter passed to padding (Default: ``\"replicate\"``)\n\n    Returns:\n        Tensor: Tensor of deltas of dimension (..., freq, time)\n\n    Example\n        >>> specgram = torch.randn(1, 40, 1000)\n        >>> delta = compute_deltas(specgram)\n        >>> delta2 = compute_deltas(delta)\n    \"\"\"\n    device = specgram.device\n    dtype = specgram.dtype\n\n    # pack batch\n    shape = specgram.size()\n    specgram = specgram.reshape(1, -1, shape[-1])\n\n    assert win_length >= 3\n\n    n = (win_length - 1) // 2\n\n    # twice sum of integer squared\n    denom = n * (n + 1) * (2 * n + 1) / 3\n\n    specgram = torch.nn.functional.pad(specgram, (n, n), mode=mode)\n\n    kernel = torch.arange(-n, n + 1, 1, device=device, dtype=dtype).repeat(\n        specgram.shape[1], 1, 1\n    )\n\n    output = (\n        torch.nn.functional.conv1d(specgram, kernel, groups=specgram.shape[1]) / denom\n    )\n\n    # unpack batch\n    output = output.reshape(shape)\n\n    return output\n\n\nclass SpecAugment(torch.nn.Module):\n    def __init__(self,\n                 freq_mask_param=15,\n                 time_mask_param=35,\n                 num_freq_masks=2,\n                 num_time_masks=2):\n        super().__init__()\n        self.freq_mask = torchaudio.transforms.FrequencyMasking(freq_mask_param=freq_mask_param)\n        self.time_mask = torchaudio.transforms.TimeMasking(time_mask_param=time_mask_param)\n        self.num_freq_masks = num_freq_masks\n        self.num_time_masks = num_time_masks\n\n    def forward(self, spec):\n        for _ in range(self.num_freq_masks):\n            spec = self.freq_mask(spec)\n        for _ in range(self.num_time_masks):\n            spec = self.time_mask(spec)\n        return spec\n\nclass LocalGlobalStretch(torch.nn.Module):\n    \"\"\"\n    Applies local and global stretching to melspectrograms in time and frequency dimensions.\n    \"\"\"\n    def __init__(self, \n                 global_stretch_prob=0.5,\n                 local_stretch_prob=0.5,\n                 max_global_stretch=0.2,\n                 max_local_stretch=0.3,\n                 max_local_regions=3):\n        super().__init__()\n        self.global_stretch_prob = global_stretch_prob\n        self.local_stretch_prob = local_stretch_prob\n        self.max_global_stretch = max_global_stretch\n        self.max_local_stretch = max_local_stretch\n        self.max_local_regions = max_local_regions\n        \n    def _global_stretch(self, spec):\n        \"\"\"Apply global stretching in time or frequency dimension\"\"\"\n        # Get current dimensions\n        _, h, w = spec.shape\n        \n        # Randomly choose to stretch height (frequency) or width (time) or both\n        stretch_dim = random.randint(0, 2)  # 0: freq, 1: time, 2: both\n        \n        # Calculate new dimensions with random stretching factor\n        new_h = h\n        new_w = w\n        \n        if stretch_dim in [0, 2]:  # Stretch frequency\n            stretch_factor = 1.0 + random.uniform(-self.max_global_stretch, self.max_global_stretch)\n            new_h = max(int(h * stretch_factor), 1)\n            \n        if stretch_dim in [1, 2]:  # Stretch time\n            stretch_factor = 1.0 + random.uniform(-self.max_global_stretch, self.max_global_stretch)\n            new_w = max(int(w * stretch_factor), 1)\n        \n        # Apply resize and then resize back to original\n        stretched = F.resize(spec, [new_h, new_w], antialias=True)\n        return F.resize(stretched, [h, w], antialias=True)\n    \n    def _local_stretch(self, spec):\n        \"\"\"Apply local stretching to random regions\"\"\"\n        _, h, w = spec.shape\n        spec_modified = spec.clone()\n        \n        # Apply random number of local stretches\n        num_regions = random.randint(1, self.max_local_regions)\n        \n        for _ in range(num_regions):\n            # Decide whether to stretch in frequency or time direction\n            is_freq_stretch = random.random() < 0.5\n            \n            # Define local region\n            if is_freq_stretch:\n                # Frequency stretch (horizontal region)\n                region_h = random.randint(max(1, int(h * 0.1)), max(2, int(h * 0.5)))\n                start_h = random.randint(0, h - region_h)\n                \n                # Extract region\n                region = spec[:, start_h:start_h+region_h, :]\n                \n                # Stretch factor\n                stretch_factor = 1.0 + random.uniform(-self.max_local_stretch, self.max_local_stretch)\n                new_h = max(int(region_h * stretch_factor), 1)\n                \n                # Stretch and resize back\n                stretched = F.resize(region, [new_h, w], antialias=True)\n                stretched = F.resize(stretched, [region_h, w], antialias=True)\n                \n                # Put back\n                spec_modified[:, start_h:start_h+region_h, :] = stretched\n            else:\n                # Time stretch (vertical region)\n                region_w = random.randint(max(1, int(w * 0.1)), max(2, int(w * 0.5)))\n                start_w = random.randint(0, w - region_w)\n                \n                # Extract region\n                region = spec[:, :, start_w:start_w+region_w]\n                \n                # Stretch factor\n                stretch_factor = 1.0 + random.uniform(-self.max_local_stretch, self.max_local_stretch)\n                new_w = max(int(region_w * stretch_factor), 1)\n                \n                # Stretch and resize back\n                stretched = F.resize(region, [h, new_w], antialias=True)\n                stretched = F.resize(stretched, [h, region_w], antialias=True)\n                \n                # Put back\n                spec_modified[:, :, start_w:start_w+region_w] = stretched\n                \n        return spec_modified\n    \n    def forward(self, spec):\n        # Apply global stretching with probability\n        if random.random() < self.global_stretch_prob:\n            spec = self._global_stretch(spec)\n            \n        # Apply local stretching with probability\n        if random.random() < self.local_stretch_prob:\n            spec = self._local_stretch(spec)\n            \n        return spec\n\nclass AddGaussianNoise(torch.nn.Module):\n    \"\"\"Add Gaussian noise to spectrogram\"\"\"\n    def __init__(self, mean=0., std=0.01):\n        super().__init__()\n        self.mean = mean\n        self.std = std\n        \n    def forward(self, spec):\n        noise = torch.randn_like(spec) * self.std + self.mean\n        return spec + noise\n\nclass TimeShift(torch.nn.Module):\n    \"\"\"Shift spectrogram in time dimension\"\"\"\n    def __init__(self, max_shift_pct=0.1):\n        super().__init__()\n        self.max_shift_pct = max_shift_pct\n        \n    def forward(self, spec):\n        _, _, width = spec.shape\n        max_shift = int(width * self.max_shift_pct)\n        if max_shift < 1:\n            return spec\n            \n        shift = random.randint(-max_shift, max_shift)\n        if shift == 0:\n            return spec\n            \n        # Shift along time dimension (dim=2)\n        return torch.roll(spec, shifts=shift, dims=2)\n\nclass FrequencyShift(torch.nn.Module):\n    \"\"\"Shift spectrogram in frequency dimension (within limit)\"\"\"\n    def __init__(self, max_shift_pct=0.05):\n        super().__init__()\n        self.max_shift_pct = max_shift_pct\n        \n    def forward(self, spec):\n        _, height, _ = spec.shape\n        max_shift = int(height * self.max_shift_pct)\n        if max_shift < 1:\n            return spec\n            \n        shift = random.randint(-max_shift, max_shift)\n        if shift == 0:\n            return spec\n            \n        # Shift along frequency dimension (dim=1)\n        return torch.roll(spec, shifts=shift, dims=1)\n\n\nclass AudioToSpec:\n    def __init__(self, \n                 melSpecParams,\n                 top_db=80,\n                 image_size=None,\n                 delta_stack = False\n                 ):\n        self.melSpec_transform = torchaudio.transforms.MelSpectrogram(**melSpecParams)\n        self.db_transform = torchaudio.transforms.AmplitudeToDB(stype='power', top_db=top_db)\n\n\n        if image_size is None:\n            # freq_mask_param=15\n            # time_mask_param=35\n            freq_mask_param=18\n            time_mask_param=50\n        else: \n            freq_mask_param = int(image_size[0] * 0.04)\n            time_mask_param = int(image_size[1] * 0.1)\n\n        self.spec_aug = SpecAugment(freq_mask_param=freq_mask_param, time_mask_param=time_mask_param)\n        self.stretch_aug = LocalGlobalStretch()\n        self.top_db = top_db\n        self.delta_stack = delta_stack\n\n        print (\"Top DB\", top_db)\n\n        if self.delta_stack:\n            print (\"Apply Delta Stack\")\n\n\n        if image_size is not None:\n            print (\"Using Image Size\", image_size)\n            self.resize_transforms = v2.Compose([\n                v2.Resize(size=image_size),\n            ])\n        else:\n            print (\"No Resizing\")\n            self.resize_transforms = None\n\n        self.train_transforms = v2.Compose([\n            TimeShift(max_shift_pct=0.15),\n            FrequencyShift(max_shift_pct=0.05),\n            AddGaussianNoise(std=0.01),\n            v2.RandomErasing(p=0.7, scale=(0.02, 0.08), ratio=(0.5, 2.0)),\n        ])\n\n\n    def __call__(self, audio, train): #Take Audio Tensor as input (1 x Audio)\n        spec = self.db_transform(self.melSpec_transform(audio)) #1 x Mel_H x Mel_W\n        spec = self.normalize_melspec(spec) #1 x Mel_H x Mel_W\n\n        if self.delta_stack:\n            delta_1 = compute_deltas(spec)\n            delta_2 = compute_deltas(delta_1)\n            spec = torch.cat([spec, delta_1, delta_2], dim=0)  # shape: (3, Mel_H, Mel_W)\n\n        if train:\n            spec = self.spec_aug(spec)\n            spec = self.stretch_aug(spec)\n\n        if not self.delta_stack:\n            spec = spec.expand(3, -1, -1)\n\n        if self.resize_transforms is not None:\n            spec = self.resize_transforms(spec)\n        if train:\n            spec = self.train_transforms(spec)\n\n        return spec\n    \n    def normalize_melspec(self, X):\n        return torch.clamp((X + self.top_db) / self.top_db, 0.0, 1.0)\n\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class TimmCNN(torch.nn.Module):\n    def __init__(self, backbone, pretrained, num_classes):\n        super().__init__()\n\n        self.backbone = timm.create_model(\n            backbone,\n            pretrained=pretrained,\n            in_chans=3,\n            num_classes=num_classes,\n        )\n\n    def forward(self, x):\n        x = self.backbone(x)\n        return x\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport timm\n\ndef init_layer(layer):\n    \"\"\"Initialize a Linear or Convolutional layer.\"\"\"\n    nn.init.xavier_uniform_(layer.weight)\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\ndef init_bn(bn):\n    \"\"\"Initialize a Batch Normalization layer.\"\"\"\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.)\n\n\nclass AttBlockV2(nn.Module):\n    \"\"\"Attention block for SED tasks.\"\"\"\n    def __init__(self, in_features, out_features, activation=\"linear\"):\n        super().__init__()\n\n        self.activation = activation\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True\n        )\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True\n        )\n        self.activation = activation\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n\n    def forward(self, x):\n        # x: (batch_size, channels, time)\n        norm_att = torch.softmax(torch.tanh(self.att(x)), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n\n        if self.activation == \"sigmoid\":\n            eps = 1e-6\n            x = torch.clamp(x, eps, 1 - eps)\n        return x\n\n    def nonlinear_transform(self, x):\n        if self.activation == \"linear\":\n            return x\n        elif self.activation == \"sigmoid\":\n            return torch.sigmoid(x)\n\ndef interpolate(x, ratio):\n    \"\"\"Interpolate data in time domain.\"\"\"\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\ndef pad_framewise_output(framewise_output, frames_num):\n    \"\"\"Pad framewise_output to the same length as input frames.\"\"\"\n    output = torch.nn.functional.interpolate(\n        framewise_output.unsqueeze(1),\n        size=(frames_num, framewise_output.size(2)),\n        align_corners=True,\n        mode=\"bilinear\",\n    ).squeeze(1)\n    return output\n\nclass SEDModel(nn.Module):\n    def __init__(self, \n                 model_name, \n                 num_classes, \n                 n_mels,\n                 in_chans=1, \n                 pretrained=True, \n                 drop_path_rate=0.2, \n                 drop_rate=0.5,\n                 freq_wise = False\n                 ):\n        super().__init__()\n        \n        self.freq_wise = freq_wise\n        self.num_classes = num_classes\n        self.in_chans = in_chans\n        \n        # Batch normalization layer for mel spectrogram input\n        self.bn0 = nn.BatchNorm2d(n_mels)\n        \n        # Create backbone model using timm\n        base_model = timm.create_model(\n            model_name,\n            pretrained=pretrained,\n            in_chans=in_chans,\n            drop_path_rate=drop_path_rate,\n            drop_rate=drop_rate,\n        )\n        \n        # Extract all layers except the classification head\n        layers = list(base_model.children())[:-2]\n        self.encoder = nn.Sequential(*layers)\n        \n        # Determine the feature dimension based on model architecture\n        if \"efficientnet\" in model_name:\n            in_features = base_model.classifier.in_features\n        elif \"eca\" in model_name:\n            in_features = base_model.head.fc.in_features\n        elif \"res\" in model_name:\n            in_features = base_model.fc.in_features\n        else:\n            # Default fallback for other architectures\n            in_features = base_model.num_features\n            \n        # Add fully connected layer\n        self.fc1 = nn.Linear(in_features, in_features, bias=True)\n        \n        # Add attention block for SED\n        self.att_block = AttBlockV2(in_features, num_classes, activation=\"sigmoid\")\n\n        \n        \n        # Initialize weights\n        self.init_weight()\n    \n    def init_weight(self):\n        init_layer(self.fc1)\n        init_bn(self.bn0)\n    \n    def forward(self, x):\n        \"\"\"\n        Args:\n            x: Input tensor of shape (batch_size, channels, time, freq)\n                or (batch_size, channels, freq, time) depending on your data\n        \n        Returns:\n            clipwise_output: Predicted probabilities for each class (batch_size, num_classes)\n        \"\"\"\n        # Expected input shape: (batch_size, channels, time, freq)\n        # Transpose to (batch_size, channels, freq, time) if needed\n\n\n        if not self.freq_wise:\n            x = x.permute(0, 1, 3, 2)\n                \n        frames_num = x.shape[2]\n        \n        # Apply batch normalization\n        x = x.transpose(1, 3).contiguous()\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n        \n        # Re-transpose for the CNN\n        x = x.transpose(2, 3).contiguous()\n        # Now shape: (batch_size, channels, freq, time)\n        \n        # Pass through encoder\n        x = self.encoder(x)\n        \n        # Average pooling over frequency dimension\n        x = torch.mean(x, dim=2)\n        \n        # Apply channel smoothing\n        x1 = torch.nn.functional.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = torch.nn.functional.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n        \n        x = torch.nn.functional.dropout(x, p=0.5, training=self.training)\n\n        # Apply FC layer\n        x = x.transpose(1, 2).contiguous()\n        x = torch.nn.functional.relu_(self.fc1(x))\n        x = x.transpose(1, 2).contiguous()\n        \n        x = torch.nn.functional.dropout(x, p=0.5, training=self.training)\n\n        # # Get clipwise output through attention mechanism\n        clipwise_output = self.att_block(x)\n\n        clipwise_output = torch.logit(clipwise_output)\n        \n        return clipwise_output","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def merge_close_segments(segments, max_gap=0.5):\n    if not segments:\n        return []\n\n    merged = [segments[0]]\n\n    for current in segments[1:]:\n        last = merged[-1]\n        if current['start'] - last['end'] < max_gap:\n            # Merge with the previous segment\n            last['end'] = current['end']\n        else:\n            merged.append(current)\n\n    return merged\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_human_non_human(waves, timestamp, max_gap=1.0, sample_rate=32000):\n\n    if len(timestamp) == 0:\n        return None, None\n\n    speaking_segments = merge_close_segments(timestamp, max_gap=max_gap)\n\n    speaking_indices = [(int(start * sample_rate), int(end * sample_rate)) for start, end in [(s['start'], s['end']) for s in speaking_segments]]\n    human_audio_parts = [waves[start:end] for start, end in speaking_indices]\n    non_human_audio_parts = []\n\n    prev_end = 0\n    for start, end in speaking_indices:\n        if start > prev_end:\n            non_human_audio_parts.append(waves[prev_end:start])\n        prev_end = end\n\n    if prev_end < len(waves):\n        non_human_audio_parts.append(waves[prev_end:])\n\n    human_audio = np.concatenate(human_audio_parts) if human_audio_parts else np.array([], dtype=waves.dtype)\n    non_human_audio = np.concatenate(non_human_audio_parts) if non_human_audio_parts else np.array([], dtype=waves.dtype)\n    return human_audio, non_human_audio\n\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def unique_class_filtered_timestamps():\n    human_model, utils = torch.hub.load(repo_or_dir='snakers4/silero-vad', model='silero_vad')\n    (get_speech_timestamps, _, read_audio, _, _) = utils\n\n    df = pd.read_csv(\"../../../data/birdclef-2025/train.csv\")\n    unique, counts = np.unique(df['primary_label'], return_counts=True)\n    unique_low = unique[counts < 30]\n\n    xdf = df[df['primary_label'].isin(unique_low)]\n    timestamps = {}\n    for i, row in tqdm(xdf.iterrows(), total=xdf.shape[0]):\n        wav = read_audio(f\"../../../data/birdclef-2025/train_audio/{row['filename']}\")\n        speech_timestamps = get_speech_timestamps(\n            wav,\n            human_model,\n            return_seconds=True,\n            )\n        if len(speech_timestamps) > 0:\n            timestamps[row['filename']] = speech_timestamps\n    return timestamps\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def csa_filtered_timestamps():\n    human_model, utils = torch.hub.load(repo_or_dir='snakers4/silero-vad', model='silero_vad')\n    (get_speech_timestamps, _, read_audio, _, _) = utils\n\n    df = pd.read_csv(\"../../../data/birdclef-2025/train.csv\")\n    v = df['filename'].str.split(\"/\", expand=True)\n    v = v[v.columns[1]].values\n    v = [each.startswith('CSA') for each in v]\n    xdf = df[v]\n\n    timestamps = {}\n    for i, row in tqdm(xdf.iterrows(), total=xdf.shape[0]):\n        wav = read_audio(f\"../../../data/birdclef-2025/train_audio/{row['filename']}\")\n        speech_timestamps = get_speech_timestamps(\n            wav,\n            human_model,\n            return_seconds=True,\n            )\n        if len(speech_timestamps) > 0:\n            timestamps[row['filename']] = speech_timestamps\n    return timestamps","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nclass RawBirdClefMelSpec(torch.utils.data.Dataset):\n    \"\"\" \n    This class is used to load the raw signal data from the data directory.\n    \"\"\"\n    def __init__(self,\n                 df,\n                 mel_spec_params,\n                 top_db,\n                 num_classes = -1,\n                 bird2id = None,\n                 is_train=False,\n                 wave_transforms=None,\n                 use_secondary_labels=False,\n                 image_size = None,\n                 delta_stack = False,\n                 human_filter_type = 'CSA', # Possible Options 'CSA', 'UNIQUE', None\n                 human_filter_chance_threshold = 0.5,\n                 add_human_noise = False,\n                 max_trick = False, \n                 max_trick_factor = 1,\n                 base_duration = 5,\n                 combined_sampling = True,\n                 random_shuffle = True\n                 ):\n        self.df = df\n        self.human_filter_chance_threshold = human_filter_chance_threshold\n        self.mel_spec_params = mel_spec_params\n        self.bird2id = bird2id\n        self.is_train = is_train\n        self.num_classes = num_classes\n        self.wave_transforms = wave_transforms\n        self.use_secondary_labels = use_secondary_labels\n        self.add_human_noise = add_human_noise\n        self.max_trick = max_trick\n        self.max_trick_factor = max_trick_factor\n        self.base_duration = base_duration\n        self.combined_sampling = combined_sampling\n        self.random_shuffle = random_shuffle\n        self.audio2spec = AudioToSpec(melSpecParams=mel_spec_params, top_db=top_db, image_size=image_size, delta_stack=delta_stack)\n        if self.is_train:\n            if human_filter_type == 'CSA':\n                if os.path.exists('outputs/csa_timestamps.json'):\n                    with open('outputs/csa_timestamps.json', 'r') as json_file:\n                        self.timestamps = json.load(json_file)\n                else:\n                    self.timestamps = csa_filtered_timestamps()\n                    with open('outputs/csa_timestamps.json', 'w') as json_file:\n                        json.dump(self.timestamps, json_file)\n\n                \n            elif human_filter_type == 'UNIQUE':\n                self.timestamps = unique_class_filtered_timestamps()\n            else:\n                print (\"NO HUMAN AUDIO Filter Applied\")\n                self.timestamps = {}\n        else:\n            self.timestamps = {}\n            \n\n\n    def __len__(self):\n        return len(self.df)\n    \n    def max_norm_wave(self, wave):\n        max_v = np.max(np.abs(wave))\n        if max_v > 1:\n            wave = wave/max_v\n        return wave\n\n    def apply_max_stack(self, wave):\n        if not self.max_trick:\n            return wave\n        \n        new_wave = []\n        for i in range(0, self.max_trick_factor):\n            start = i * self.base_duration * 32000\n            end = (i+1) * self.base_duration * 32000\n            _wave = wave[start:end]\n            new_wave.append(_wave)\n        new_wave = np.stack(new_wave)\n        return new_wave\n    \n\n    def combined_pick_rms_sample(self, waves, duration_samples):\n        if len(waves) <= duration_samples:\n            # Pad if too short\n            pad_length = duration_samples - len(waves)\n            waves = np.pad(waves, (0, pad_length), mode='constant')\n            return waves\n\n        stride = 32000\n        max_rms = 0\n        max_rms_start = 0\n\n        for start in range(0, len(waves) - duration_samples + 1, stride):\n            window = waves[start:start + duration_samples]\n            rms = np.sqrt(np.mean(window ** 2))\n            if rms > max_rms:\n                max_rms = rms\n                max_rms_start = start\n\n        wave = waves[max_rms_start:max_rms_start + duration_samples]\n        return wave\n    \n    def ___pick_rms_sample(self, waves, duration_samples):\n        if len(waves) <= duration_samples:\n            # Pad if too short\n            pad_length = duration_samples - len(waves)\n            waves = np.pad(waves, (0, pad_length), mode='constant')\n            return waves, np.array([])\n\n        stride = 32000\n        max_rms = 0\n        max_rms_start = 0\n\n        for start in range(0, len(waves) - duration_samples + 1, stride):\n            window = waves[start:start + duration_samples]\n            rms = np.sqrt(np.mean(window ** 2))\n            if rms > max_rms:\n                max_rms = rms\n                max_rms_start = start\n\n        wave = waves[max_rms_start:max_rms_start + duration_samples]\n\n        start_base_wave = waves[0:max_rms_start]\n        next_base_wave = waves[max_rms_start + duration_samples:]\n        base_wave = np.concatenate([start_base_wave, next_base_wave], axis=0)\n        \n        return wave, base_wave\n    \n    def chunked_pick_rms_sample(self, waves, duration_samples):\n        total_waves = duration_samples // (self.base_duration * 32000)\n        waves_ = []\n        for i in range(total_waves):\n            wave, waves = self.___pick_rms_sample(waves, self.base_duration * 32000)\n            waves_.append(wave)\n        waves_ = np.concatenate(waves_, axis=0).astype(np.float32)\n        return waves_\n\n\n    def pick_rms_sample(self, waves, duration_samples):\n        if self.combined_sampling:\n            return self.combined_pick_rms_sample(waves, duration_samples)\n        return self.chunked_pick_rms_sample(waves, duration_samples)\n\n    \n    def load_filtered_sample(self, human_audio, non_human_audio):\n        \n        duration = self.base_duration\n        if self.max_trick:\n            duration = duration * self.max_trick_factor\n            human_duration = duration // 3\n            non_human_duration = duration - human_duration\n        else:\n            if duration == 5:\n                human_duration = 2\n                non_human_duration = 3\n            elif duration == 15:\n                human_duration = 5\n                non_human_duration = 10\n\n\n        target_length = 32000 * human_duration\n        human_audio_sample = self.pick_rms_sample(human_audio, target_length)\n        target_length = 32000 * non_human_duration\n        non_human_audio_sample = self.pick_rms_sample(non_human_audio, target_length)\n\n        if random.random() > 0.5:\n            wave = np.concatenate([human_audio_sample, non_human_audio_sample], axis=0)\n        else:\n            wave = np.concatenate([non_human_audio_sample, human_audio_sample], axis=0)\n\n        wave = self.apply_max_stack(wave)\n\n        return wave\n    \n    def load_raw_sample(self, waves, pick_random):\n        target_length = 32000 * self.base_duration\n\n        if self.max_trick:\n            target_length = target_length * self.max_trick_factor\n        \n        if pick_random:\n            wave = self.pick_rms_sample(waves, target_length)\n        else:\n            pad_size = target_length - (waves.shape[0] % target_length)\n            if pad_size != target_length:  # Avoid adding unnecessary padding if already aligned\n                waves = np.pad(waves, (0, pad_size), mode='constant', constant_values=0)\n            waves = np.reshape(waves, newshape=[-1, target_length])\n            wave = waves[0]\n        wave = self.apply_max_stack(wave)\n        return wave\n\n    def convert_to_spec(self, wave):\n        if not self.max_trick:\n            return self.audio2spec(wave.unsqueeze(0), train=self.is_train)\n        \n        specs = []\n        for i in range(0, len(wave)):\n            spec = self.audio2spec(wave[i].unsqueeze(0), train=self.is_train)\n            specs.append(spec)\n        specs = torch.stack(specs)\n        return specs\n\n\n    def __getitem__(self, idx):\n        fn = self.df[\"path\"].iloc[idx]\n        primary_label = self.df[\"primary_label\"].iloc[idx]\n        secondary_labels = self.df[\"secondary_labels\"].iloc[idx]\n        waves, _ = sf.read(fn)\n        if waves.ndim!=1:\n            waves = waves.mean(1)\n        waves = waves.astype(np.float32)\n\n        duration = self.base_duration\n        timstamp = self.timestamps.get(self.df[\"filename\"].iloc[idx], {})\n        human_audio, non_human_audio = get_human_non_human(waves, timstamp, max_gap=1.0, sample_rate=32000)\n        if (len(timstamp) > 0) and (self.is_train == True) and (waves.shape[0] > 32000*duration) and (human_audio.shape[0] > (waves.shape[0] * 0.5)):\n            if random.random() > 0.5 and self.human_filter_chance_threshold:\n                wave = self.load_filtered_sample(human_audio=human_audio, non_human_audio=non_human_audio)\n            else:\n                wave = self.load_raw_sample(non_human_audio, pick_random=True)\n        \n        else:\n            \n            add_human_noise = False\n            human_audio = None\n            actual_waves = waves\n            if self.is_train and self.add_human_noise:\n                if random.random() > 0.2:\n                    timstamp_keys = list(self.timestamps.keys())\n                    timstamp_key = random.choice(timstamp_keys)\n                    timstamp = self.timestamps.get(timstamp_key, {})\n                    if len(timstamp) > 0:\n                        fn = f\"../../../data/birdclef-2025/train_audio/\" + timstamp_key\n                        noise_waves, _ = sf.read(fn)\n                        if noise_waves.ndim!=1:\n                            noise_waves = noise_waves.mean(1)\n                        human_audio, _ = get_human_non_human(noise_waves, timstamp, max_gap=1.0, sample_rate=32000)\n                        if human_audio.shape[0] > 0:\n                            add_human_noise = True\n\n            if add_human_noise:\n                wave = self.load_filtered_sample(human_audio=human_audio, non_human_audio=actual_waves)\n            else:\n                wave = self.load_raw_sample(actual_waves, pick_random=self.is_train)\n\n        if self.max_trick:\n            for i in range(len(wave)):\n                wave[i] = self.max_norm_wave(wave[i])\n        else:\n            wave = self.max_norm_wave(wave)\n        \n        if self.wave_transforms:\n            wave = self.wave_transforms(wave, sr=32000)\n\n        if self.use_secondary_labels:\n            target = np.zeros(self.num_classes, dtype=np.float32)\n            primary_label = self.bird2id[primary_label]\n            target[primary_label] = 1.0\n            \n            secondary_labels = eval(secondary_labels)\n            secondary_labels = [each for each in secondary_labels if each != '']\n            coeff = len(secondary_labels)\n            for label in secondary_labels:\n                label = self.bird2id[label]\n                target[label] = 1.0 #1.0 / coeff\n        else:\n            target = np.zeros(self.num_classes, dtype=np.float32)\n            if primary_label != 'nocall':\n                primary_label = self.bird2id[primary_label]\n                target[primary_label] = 1.0\n        if self.is_train and self.max_trick and self.random_shuffle:\n            idx__ = np.array(list(range(0, self.max_trick_factor)))\n            np.random.shuffle(idx__)\n            wave = wave[idx__]\n\n        wave = torch.from_numpy(wave)\n        spec = self.convert_to_spec(wave)\n        return {\n            \"spec\": spec,\n            \"target\": torch.tensor(target),\n        }\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class FocalLossBCE(torch.nn.Module):\n    def __init__(\n            self,\n            alpha: float = 0.25,\n            gamma: float = 2,\n            reduction: str = \"mean\",\n            bce_weight: float = 1.0,\n            focal_weight: float = 1.0,\n    ):\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.reduction = reduction\n        self.bce = torch.nn.BCEWithLogitsLoss(reduction=reduction)\n        self.bce_weight = bce_weight\n        self.focal_weight = focal_weight\n\n    def forward(self, logits, targets):\n        focall_loss = torchvision.ops.focal_loss.sigmoid_focal_loss(\n            inputs=logits,\n            targets=targets,\n            alpha=self.alpha,\n            gamma=self.gamma,\n            reduction=self.reduction,\n        )\n        bce_loss = self.bce(logits, targets)\n        return self.bce_weight * bce_loss + self.focal_weight * focall_loss","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_epoch(epoch, model, train_loader, optimizer, device, criterion, accumulation_steps=1, use_mixed_precision=False, scaler=None, max_trick = False, use_maximum_mix=False, use_mix_prob=0.5):\n    \"\"\"\n    Unified training function supporting regular training, gradient accumulation, and mixed precision.\n    \n    Args:\n        model: The neural network model\n        train_loader: DataLoader for training data\n        optimizer: The optimizer\n        device: Device to run training on (cuda/cpu)\n        criterion: Loss function\n        accumulation_steps: Number of steps to accumulate gradients (default=1 means no accumulation)\n        use_mixed_precision: Whether to use mixed precision training (default=False)\n    \n    Returns:\n        List of training losses for the epoch\n    \"\"\"\n    model.train()\n    train_loss = []\n    optimizer.zero_grad()\n    if not use_mixed_precision:\n        scaler = None\n    bar = tqdm(enumerate(train_loader), total=len(train_loader))\n    \n    for step, batch in bar:\n        \n\n        spec = batch['spec']\n        target = batch['target']\n\n        spec, target = mixup(spec, target, alpha=1.0)\n\n        if max_trick:\n            BS, K, C, H, W = spec.shape\n            spec = spec.view(BS * K, C, H, W)\n\n        spec = spec.to(device)\n        target = target.to(device)\n        \n        # Forward pass with or without mixed precision\n        if use_mixed_precision:\n            with torch.cuda.amp.autocast():\n                predictions = model(spec)\n                if max_trick:\n                    predictions = predictions.view(BS, K, -1)\n                    predictions = torch.max(predictions, dim=1).values\n\n                if use_maximum_mix and random.random() < use_mix_prob:\n                    indices = torch.randperm(predictions.size(0)).to(device)\n                    predictions = torch.maximum(predictions, predictions[indices])\n                    target = torch.maximum(target, target[indices])\n\n                loss = criterion(predictions, target)\n                # Scale loss according to accumulation steps\n                if accumulation_steps > 1:\n                    loss = loss / accumulation_steps\n        else:\n            predictions = model(spec)\n            if max_trick:\n                predictions = predictions.view(BS, K, -1)\n                predictions = torch.max(predictions, dim=1).values\n\n            if use_maximum_mix and random.random() < use_mix_prob:\n                indices = torch.randperm(predictions.size(0)).to(device)\n                predictions = torch.maximum(predictions, predictions[indices])\n                target = torch.maximum(target, target[indices])\n            \n            loss = criterion(predictions, target)\n            # Scale loss according to accumulation steps\n            if accumulation_steps > 1:\n                loss = loss / accumulation_steps\n        \n        # Backward pass with or without mixed precision\n        if use_mixed_precision:\n            scaler.scale(loss).backward()\n        else:\n            loss.backward()\n        \n        # Store unscaled loss for logging\n        loss_value = loss.detach().cpu().item()\n        if accumulation_steps > 1:\n            loss_value *= accumulation_steps  # Un-normalize for logging\n        train_loss.append(loss_value)\n        \n        # Update weights if needed\n        update_weights = (step + 1) % accumulation_steps == 0 or (step + 1) == len(train_loader)\n        \n        if update_weights:\n            if use_mixed_precision:\n                # Unscale before gradient clipping\n                scaler.unscale_(optimizer)\n                torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=10.0)\n                # Step optimizer and update scaler\n                scaler.step(optimizer)\n                scaler.update()\n            else:\n                torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=10.0)\n                optimizer.step()\n            \n            optimizer.zero_grad()\n        \n        # Update progress bar\n        smooth_loss = sum(train_loss[-100:]) / min(len(train_loss), 100)\n        bar.set_description('loss: %.5f, smth: %.5f' % (loss_value, smooth_loss))\n    \n    return train_loss","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def val_epoch(model, valid_loader, device, target_columns, unseen_index=None, criterion=None, exclude_nocall = False, max_trick = False):\n    if unseen_index is not None:\n        scores = MetricMeter(indices_ignore=unseen_index)\n        \n    loss_function = criterion\n    model.eval()\n    val_loss = []\n    PREDS = []\n    TRUES = []\n    bar = tqdm(valid_loader)\n    with torch.no_grad():\n        for batch in bar:\n            target = batch['target'].to(device)\n            spec = batch['spec'].to(device)\n            \n            if max_trick:\n                BS, K, C, H, W = spec.shape\n                spec = spec.view(BS * K, C, H, W)\n\n            \n            predictions = model(spec)\n            if max_trick:\n                predictions = predictions.view(BS, K, -1)\n                predictions = torch.max(predictions, dim=1).values\n\n            if exclude_nocall:\n                target = target[:, :-1]\n                predictions = predictions[:, :-1]\n\n            loss = loss_function(predictions, target)\n            \n            val_loss.append(loss.detach().cpu().numpy())\n            PREDS.append(predictions.detach().cpu())\n            \n            target = torch.round(target).long()\n\n            TRUES.append(target.detach().cpu())\n            \n\n            if unseen_index is not None:\n                scores.update(target, predictions)\n            \n            loss_np = loss.detach().cpu().item()\n            bar.set_description('loss: %.5f' % (loss_np))\n\n            \n    val_loss = np.mean(val_loss)\n    P = np.concatenate(PREDS, axis=0)\n    T = np.concatenate(TRUES, axis=0)\n    metrics = calculate_competition_metrics(T, P, target_columns)\n    if unseen_index is not None:\n        metrics['score'] = scores.avg\n    log = metrics_to_string(metrics, \"val\")\n    return val_loss, metrics, log","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# better_mel_cfg = {\n#     # keep your acquisition settings\n#     \"sample_rate\": 32_000,          # 32 kHz\n#     # frequency coverage & resolution\n#     \"f_min\":      50,               # ignore infrasonic rumble\n#     \"f_max\":      16_000,           # Nyquist\n#     \"n_fft\":      4096,             # ≈128 ms window  →  fine 7.8 Hz bins\n#     \"hop_length\": 320,              # 10 ms stride   →  tight time cues\n#     # mel projection\n#     \"n_mels\":     448,              # dense  →  good insect harmonics\n#     \"mel_scale\":  \"htk\",            # slightly more linear high‑freq spacing\n#     # dynamic‑range mapping\n#     \"power\":      2.0,              # power‑spectrogram before log\n#     \"normalized\": True,\n# }\n\n# tf_efficientnet_b0.aa_in1k\n# tf_efficientnet_b0.ap_in1k\n# tf_efficientnet_b0.in1k\n# tf_efficientnet_b0.ns_jft_in1k","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def prepare_mel_spec_params(image_size, duration = 5, get_default = False):\n    if get_default:\n        return {\n            \"sample_rate\": 32000,\n            \"n_mels\": 384,\n            \"f_min\": 0,\n            \"f_max\": 16000,\n            \"n_fft\": 3072,\n            \"hop_length\": 420,\n            \"normalized\": True,\n        }\n\n    new_nfft = 4096 #int(image_size[0] * 8)\n    new_mel_spec_params = {\n        \"sample_rate\": 32000,\n        \"n_mels\": image_size[0],\n        \"f_min\": 50,\n        \"f_max\": 16000,\n        \"n_fft\": new_nfft,\n        \"normalized\": True,\n    }\n    new_mel_spec_params['hop_length'] = int((duration * 32000 - new_mel_spec_params['n_fft'] + new_mel_spec_params['n_fft']) / (image_size[1])) + 1\n    print (new_mel_spec_params)\n    return new_mel_spec_params\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"RAW_TRAIN_CONFIG = {\n    \"seed\": 42,\n    \"experiment_name\": \"stage0/b0_cnn_384_3_no_shuffle\",\n    \"batch_size\": 32,\n    \"accumulation_steps\" : 1,\n    \"mixed_precision\" : False,\n    \"num_workers\": 12,\n    \"warmp_epoch\": 3,\n    \"n_epochs\": 15,\n    \"learning_rate\": 0.0001,\n    \"weight_decay\": 0.0001,\n    \"minimum_learning_rate\" : 1e-7,\n    \"fold_n\": 5,\n    \"model_name\": \"tf_efficientnet_b0_ns\",\n    \"drop_rate\" : 0.5,\n    \"drop_path_rate\" : 0.2,\n    \"pretrained\": True,\n    \"base_path\": \"../../../data/birdclef-2025\",\n    \"start_soup_epoch\" : 12,\n    \"end_soup_epoch\" : 15,\n    \"train_use_secondary_labels\" : True,\n    \"val_use_secondary_labels\" : True,\n    \"human_filter_type\" : \"CSA\", #CSA, UNIQUE, NONE``   \n    \"human_filter_chance_threshold\" : True,\n    \"add_human_noise\" : False,\n    \"add_nocall\" : False,\n    \"cap_nocall\" : -1,\n    'fold' : -1,\n    \"image_size\" : (384, 384),\n    \"mel_spec_params\" : prepare_mel_spec_params(image_size=(384, 381), duration=5, get_default=True),\n    \"delta_stack\" : False,\n    \"multi_gpu\" : True,\n    \"device\": torch.device(\"cuda:1\"),\n    \"max_trick\" : True,\n    \"max_trick_factor\" : 3,\n    \"use_maximum_mix\" : False,\n    \"use_mix_prob\" : 0.5,\n    \"base_duration\" : 5,\n    \"save_unlabelled_predictions\" : False,\n    \"top_db\" : 80,\n    \"architecture_type\" : \"CNN\", #SED or CNN\n    \"freq_wise_attn\" : False,\n    \"combined_sampling\" : True,\n    \"random_shuffle\" : False,\n}\n\nmel_spec_params = RAW_TRAIN_CONFIG['mel_spec_params']\nos.makedirs(\"outputs\", exist_ok=True)\nos.makedirs(RAW_TRAIN_CONFIG['experiment_name'], exist_ok=True)\nos.makedirs(os.path.join(RAW_TRAIN_CONFIG['experiment_name'], \"models\"), exist_ok=True)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#### Checking Output Size of MelSpec\naudio = torch.randn(1, 32000*5)\nspec = torchaudio.transforms.AmplitudeToDB(stype='power', top_db=RAW_TRAIN_CONFIG['top_db'])(torchaudio.transforms.MelSpectrogram(**mel_spec_params)(audio))\nprint (spec.size())\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"log_file = os.path.join(RAW_TRAIN_CONFIG['experiment_name'], f\"train_log_fold_{RAW_TRAIN_CONFIG['fold']}.txt\")\nprint(mel_spec_params)\nwith open(log_file, 'a') as f:\n    f.write(str(mel_spec_params) + '\\n')\nprint (RAW_TRAIN_CONFIG)\nwith open(log_file, 'a') as f:\n    f.write(str(RAW_TRAIN_CONFIG) + '\\n')\n\n\n\nset_seed(RAW_TRAIN_CONFIG['seed'])\nsub = pd.read_csv(f\"{RAW_TRAIN_CONFIG['base_path']}/sample_submission.csv\")\ntarget_columns = sub.columns.tolist()[1:]\nnum_classes = len(target_columns)\nbird2id = {b: i for i, b in enumerate(target_columns)}\ndf = pd.read_csv(f\"{RAW_TRAIN_CONFIG['base_path']}/train.csv\")\ndf[\"path\"] = f\"{RAW_TRAIN_CONFIG['base_path']}/train_audio/\" + df[\"filename\"]\ndf['filename_base'] = df['filename'].map(lambda x: x.split('/')[-1])\ndf = df.drop_duplicates(subset=['filename_base']).reset_index(drop=True)\n\n\n####################################################################################################################################################################################\n####################################################################################################################################################################################\n####################################################################################################################################################################################","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\" \nInsecta: (0.994375, 896.57509375, 113.75990141129033, 99.41890625, 114.62581564261338)\nAves: (0.548, 1774.392, 35.03129723345455, 21.0024375, 49.89740423615133)\nMammalia: (1.018, 218.784, 33.827185393258425, 21.762921875, 35.44624156288265)\nAmphibia: (0.54459375, 389.7730625, 30.358085173670666, 16.55584375, 43.96448730451026)\nmin(), _tmp.max(), _tmp.mean(), _tmp.median(), _tmp.std()\n\"\"\"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\ngkf = StratifiedKFold(n_splits=RAW_TRAIN_CONFIG['fold_n'], shuffle=True, random_state=RAW_TRAIN_CONFIG['seed'])\ndf['fold'] = -1\nfor ifold, (train_idx, val_idx) in enumerate(gkf.split(df, y=df.primary_label.tolist())):\n    df.loc[val_idx, 'fold'] = ifold\n\n#Unseen Birds in Folds\nbird_names = set(df.primary_label)\nunseen_birds = []\nfor fold in range(RAW_TRAIN_CONFIG['fold_n']):\n    bird_names_fold = set(df[df['fold']==fold].primary_label)\n    unseen_bird = list(bird_names - bird_names_fold)\n    print(f'fold{fold}: Unseen Birds - ', len(unseen_bird))\n    unseen_birds.extend(unseen_bird)\nunseen_birds = np.unique(unseen_birds)\nprint(f'Overall Unseen Birds - ', len(unseen_birds))\ndf.loc[df['primary_label'].isin(unseen_birds), 'fold'] = -1\n\nunseen_index = []\nfor name in unseen_birds:\n    unseen_index.append(target_columns.index(name))\n\n\n####################################################################################################################################################################################\n####################################################################################################################################################################################\n####################################################################################################################################################################################\n\nif RAW_TRAIN_CONFIG['fold'] != -1:\n    trn_df = df[df['fold'] != RAW_TRAIN_CONFIG['fold']].reset_index(drop=True)\n    val_df = df[df['fold'] == RAW_TRAIN_CONFIG['fold']].reset_index(drop=True)\nelse:\n    trn_df = df.reset_index(drop=True)\n    val_df = df[df['fold'] == 0].reset_index(drop=True)\n\n# trn_df = downsample_data(trn_df, thr=500, seed=RAW_TRAIN_CONFIG['seed'])\ntrn_df = upsample_data(trn_df, thr=15, seed=RAW_TRAIN_CONFIG['seed'])\n\n\nif RAW_TRAIN_CONFIG[\"add_nocall\"]:\n    nocall_path = os.path.join(RAW_TRAIN_CONFIG[\"base_path\"], \"ff1010bird_nocall\")\n    nocall_df = pd.read_csv(os.path.join(nocall_path, \"ff1010bird_metadata_v1.csv\"))\n    nocall_df[\"filename\"] = \"nocall/\" + nocall_df[\"filename\"]\n    nocall_df[\"path\"] = nocall_path + \"/\" + nocall_df[\"filename\"]\n    nocall_df.pop(\"filename\")\n    nocall_df.pop(\"length\")\n    if RAW_TRAIN_CONFIG[\"cap_nocall\"] != -1:\n        nocall_df = nocall_df.sample(n=RAW_TRAIN_CONFIG[\"cap_nocall\"])\n    nocall_df_aligned = nocall_df.reindex(columns=trn_df.columns)\n    print (trn_df.shape)\n    trn_df = pd.concat([trn_df, nocall_df_aligned], ignore_index=True)\n    print (trn_df.shape)\n    bird2id[\"nocall\"] = 206\n    num_classes = num_classes + 1\n\n\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\ntrn_dataset = RawBirdClefMelSpec(df=trn_df.reset_index(drop=True), top_db = RAW_TRAIN_CONFIG['top_db'], delta_stack = RAW_TRAIN_CONFIG['delta_stack'],\n                                bird2id=bird2id, is_train=True, num_classes=num_classes,\n                                wave_transforms=None, use_secondary_labels=RAW_TRAIN_CONFIG['train_use_secondary_labels'],\n                                mel_spec_params=mel_spec_params, image_size=RAW_TRAIN_CONFIG['image_size'], human_filter_type=RAW_TRAIN_CONFIG['human_filter_type'],\n                                human_filter_chance_threshold = RAW_TRAIN_CONFIG['human_filter_chance_threshold'], add_human_noise = RAW_TRAIN_CONFIG['add_human_noise'],\n                                max_trick = RAW_TRAIN_CONFIG['max_trick'], max_trick_factor = RAW_TRAIN_CONFIG[\"max_trick_factor\"],\n                                base_duration = RAW_TRAIN_CONFIG[\"base_duration\"], combined_sampling = RAW_TRAIN_CONFIG['combined_sampling'],\n                                random_shuffle = RAW_TRAIN_CONFIG['random_shuffle']\n                                )\n\nval_dataset = RawBirdClefMelSpec(df=val_df.reset_index(drop=True), top_db = RAW_TRAIN_CONFIG['top_db'], delta_stack = RAW_TRAIN_CONFIG['delta_stack'],\n                                bird2id=bird2id, is_train=False, num_classes=num_classes,\n                                wave_transforms=None, use_secondary_labels=RAW_TRAIN_CONFIG['val_use_secondary_labels'],\n                                mel_spec_params=mel_spec_params, image_size=RAW_TRAIN_CONFIG['image_size'], human_filter_type='NONE',\n                                human_filter_chance_threshold = RAW_TRAIN_CONFIG['human_filter_chance_threshold'], add_human_noise = RAW_TRAIN_CONFIG['add_human_noise'],\n                                max_trick = RAW_TRAIN_CONFIG['max_trick'], max_trick_factor = RAW_TRAIN_CONFIG[\"max_trick_factor\"],\n                                base_duration = RAW_TRAIN_CONFIG[\"base_duration\"], combined_sampling = RAW_TRAIN_CONFIG['combined_sampling'],\n                                random_shuffle = False\n                                )\n\ntrain_loader = torch.utils.data.DataLoader(trn_dataset, shuffle=True, batch_size=RAW_TRAIN_CONFIG['batch_size'], \n                                        drop_last=True, num_workers=RAW_TRAIN_CONFIG['num_workers'], pin_memory=True)\nval_loader = torch.utils.data.DataLoader(val_dataset, shuffle=False, batch_size=RAW_TRAIN_CONFIG['batch_size'], \n                                            drop_last=False, num_workers=RAW_TRAIN_CONFIG['num_workers'], pin_memory=True)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if RAW_TRAIN_CONFIG['architecture_type'] == 'SED':\n\n    if RAW_TRAIN_CONFIG['freq_wise_attn']:\n        attn_dim = RAW_TRAIN_CONFIG['image_size'][1]\n    else:\n        attn_dim = RAW_TRAIN_CONFIG['image_size'][0]\n    \n    model = SEDModel(RAW_TRAIN_CONFIG['model_name'], \n            num_classes, \n            attn_dim,\n            in_chans=3, \n            pretrained=True, \n            drop_rate=RAW_TRAIN_CONFIG['drop_rate'],\n            drop_path_rate=RAW_TRAIN_CONFIG['drop_path_rate'],\n            freq_wise=RAW_TRAIN_CONFIG['freq_wise_attn']\n            )\n    \nelse:\n    model = TimmCNN(backbone=RAW_TRAIN_CONFIG['model_name'], pretrained=True, num_classes=num_classes)\n\nif RAW_TRAIN_CONFIG['multi_gpu']:\n    model = torch.nn.DataParallel(model, device_ids=[1, 0])\n\nmodel = model.to(RAW_TRAIN_CONFIG['device'])","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"optimizer = AdamW(model.parameters(), \n                    lr=RAW_TRAIN_CONFIG['learning_rate'], \n                    weight_decay=RAW_TRAIN_CONFIG['weight_decay'])\n\nscheduler_cosine = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, RAW_TRAIN_CONFIG['n_epochs'], \n                                                                eta_min=RAW_TRAIN_CONFIG['minimum_learning_rate'])\nscheduler_warmup = GradualWarmupSchedulerV2(optimizer, multiplier=10, total_epoch=RAW_TRAIN_CONFIG['warmp_epoch'], \n                                            after_scheduler=scheduler_cosine)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"criterion = FocalLossBCE()\nmodel_soup = ModelSoup(model, save_dir=os.path.join(RAW_TRAIN_CONFIG['experiment_name'], \"model_soup\"), exclude_nocall=RAW_TRAIN_CONFIG[\"add_nocall\"], max_trick=RAW_TRAIN_CONFIG['max_trick'])","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scaler = GradScaler(RAW_TRAIN_CONFIG['device'])\nbest_val_loss = float('inf')\nbest_map_score = 0\nfor epoch in range(0, RAW_TRAIN_CONFIG['n_epochs']):\n    scheduler_warmup.step(epoch)\n    train_loss = train_epoch(epoch + 1, model, train_loader, optimizer, RAW_TRAIN_CONFIG['device'], criterion, accumulation_steps=RAW_TRAIN_CONFIG['accumulation_steps'], \n                            use_mixed_precision=RAW_TRAIN_CONFIG['mixed_precision'], scaler=scaler, max_trick=RAW_TRAIN_CONFIG['max_trick'],\n                            use_maximum_mix = RAW_TRAIN_CONFIG['use_maximum_mix'], use_mix_prob=RAW_TRAIN_CONFIG['use_mix_prob'])\n    \n    val_loss, metrics, log = val_epoch(model, val_loader, RAW_TRAIN_CONFIG['device'], target_columns, unseen_index, criterion=criterion, exclude_nocall=RAW_TRAIN_CONFIG[\"add_nocall\"], \n                                    max_trick=RAW_TRAIN_CONFIG['max_trick'])\n    current_lr = optimizer.param_groups[0]['lr']\n    log = time.ctime() + ' ' + f\"Epoch {epoch}, LR: {current_lr:.6f}, Train Loss: {np.mean(train_loss):.4f}, Val Loss: {val_loss:.4f}, {log}\"\n    \n    if RAW_TRAIN_CONFIG['start_soup_epoch'] <= epoch <= RAW_TRAIN_CONFIG['end_soup_epoch']:\n        model_soup.add_model_if_improved(unseen_index, model, epoch, val_loader, RAW_TRAIN_CONFIG['device'], criterion=criterion)\n    \n    print(log)\n    with open(log_file, 'a') as f:\n        f.write(log + '\\n')\n        \n    if metrics['score'] > best_map_score:\n        best_map_score = metrics['score']\n        torch.save(model.state_dict(), f\"{RAW_TRAIN_CONFIG['experiment_name']}/models/best_score_model_{RAW_TRAIN_CONFIG['fold']}.pt\")\n        print(f\"New best score model saved with score: {best_map_score:.4f}\")\n        \n    if val_loss < best_val_loss:\n        best_val_loss = val_loss\n        torch.save(model.state_dict(), f\"{RAW_TRAIN_CONFIG['experiment_name']}/models/best_val_loss_model_{RAW_TRAIN_CONFIG['fold']}.pt\")\n        print(f\"New best Val Loss model saved with Loss: {best_val_loss:.4f}\")\ntorch.save(model.state_dict(), f\"{RAW_TRAIN_CONFIG['experiment_name']}/models/final_model{RAW_TRAIN_CONFIG['n_epochs']}_{RAW_TRAIN_CONFIG['fold']}.pt\")\n\n\nif epoch > RAW_TRAIN_CONFIG['start_soup_epoch']:\n    experiment_name = RAW_TRAIN_CONFIG['experiment_name'].split('/')[-1]\n    soup_model_name = f\"{experiment_name}_model_soup_final.pt\"\n    final_soup_model = model_soup.get_soup_model(soup_model_name=soup_model_name)\n    soup_accuracy = model_soup.evaluate_model(unseen_index, final_soup_model, val_loader, criterion=criterion, device=RAW_TRAIN_CONFIG['device'])\n    soup_accuracy = soup_accuracy['score']\n    log = f\"Final Model Soup Validation Score: {soup_accuracy:.4f}\\nImprovement over initial model: {soup_accuracy - model_soup.initial_val_score['score']:.4f}\"\n    print(log)\n    with open(log_file, 'a') as f:  \n        f.write(log + '\\n')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import timm\nlist_models = timm.list_models(pretrained=True)\nfor each in list_models:\n    print (each)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null}]}