{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"},{"sourceId":12034490,"sourceType":"datasetVersion","datasetId":7405264}],"dockerImageVersionId":30919,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **BirdCLEF 2025 Training Notebook**\n\nThis is a baseline training pipeline for BirdCLEF 2025 using EfficientNetB0 with PyTorch and Timm(for pretrained EffNet). You can check inference and preprocessing notebooks in the following links: \n\n- [EfficientNet B0 Pytorch [Inference] | BirdCLEF'25](https://www.kaggle.com/code/kadircandrisolu/efficientnet-b0-pytorch-inference-birdclef-25)\n\n  \n- [Transforming Audio-to-Mel Spec. | BirdCLEF'25](https://www.kaggle.com/code/kadircandrisolu/transforming-audio-to-mel-spec-birdclef-25)  \n\nNote that by default this notebook is in Debug Mode, so it will only train the model with 2 epochs, but the [weight](https://www.kaggle.com/datasets/kadircandrisolu/birdclef25-effnetb0-starter-weight) I used in the inference notebook was obtained after 10 epochs of training.\n\n**Features**\n* Implement with Pytorch and Timm\n* Flexible audio processing with both pre-computed and on-the-fly mel spectrograms\n* Stratified 5-fold cross-validation with ensemble capability\n* Mixup training for improved generalization\n* Spectrogram augmentations (time/frequency masking, brightness adjustment)\n* AdamW optimizer with Cosine Annealing LR scheduling\n* Debug mode for quick experimentation with smaller datasets\n\n**Pre-computed Spectrograms**\nFor faster training, you can use pre-computed mel spectrograms from [this dataset](https://www.kaggle.com/datasets/kadircandrisolu/birdclef25-mel-spectrograms) by setting `LOAD_DATA = True`","metadata":{"_uuid":"a2dfbe21-7d1a-4d43-a98b-9640abbc8efc","_cell_guid":"aca5ef5c-cb64-4a49-98a2-2db9f0b6ac23","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"## Libraries","metadata":{"_uuid":"56971144-149e-410d-8250-b7898a6f15f7","_cell_guid":"6804d4db-22a1-425d-9408-f03aa03dfb6d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"import os\nimport logging\nimport random\nimport gc\nimport time\nimport cv2\nimport math\nimport warnings\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nimport librosa\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\n\nimport torchvision\nimport torchvision.transforms as T\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.auto import tqdm\n\nimport timm\n\nwarnings.filterwarnings(\"ignore\")\nlogging.basicConfig(level=logging.ERROR)","metadata":{"_uuid":"9aaf8c93-7ebe-4431-a2a7-c6b79a7bddd0","_cell_guid":"25613782-1342-4973-8c35-f46b7c4cb876","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Configuration","metadata":{"_uuid":"a05187cd-7d0a-419d-8748-3c44199744ab","_cell_guid":"38124767-5185-46d1-9796-495c2aad8cca","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"class CFG:\n    \n    seed = 42\n    debug = False\n    apex = False\n    print_freq = 100\n    num_workers = 2\n    \n    OUTPUT_DIR = '/kaggle/working/'\n\n    train_datadir = '/kaggle/input/birdclef-2025/train_audio'\n    train_csv = '/kaggle/input/birdclef-2025/train.csv'\n    test_soundscapes = '/kaggle/input/birdclef-2025/test_soundscapes'\n    submission_csv = '/kaggle/input/birdclef-2025/sample_submission.csv'\n    taxonomy_csv = '/kaggle/input/birdclef-2025/taxonomy.csv'\n\n    #spectrogram_npy = '/kaggle/input/birdclef2025-melspectrograms/birdclef2025_melspec_5sec_256_256_nohuman_random.npy'\n    #discarded_files_random = '/kaggle/input/birdclef2025-melspectrograms/discarded_files_random.txt'\n    #spectrogram_val_npy = '/kaggle/input/birdclef2025-melspectrograms/birdclef2025_melspec_5sec_256_256_nohuman_first.npy'\n    #discarded_files_first = '/kaggle/input/birdclef2025-melspectrograms/discarded_files_first.txt'\n    spectrogram_npy = '/kaggle/input/birdclef2025-spectrograms/birdclef2025_melspec_5sec_256_256_nohuman_random.npy'\n    discarded_files_random = '/kaggle/input/birdclef2025-spectrograms/discarded_files_random.txt'\n    spectrogram_val_npy = '/kaggle/input/birdclef2025-spectrograms/birdclef2025_melspec_5sec_256_256_nohuman_first.npy'\n    discarded_files_first = '/kaggle/input/birdclef2025-spectrograms/discarded_files_first.txt'\n    \n    spectrogram_pseudo_npy = '/kaggle/input/birdclef2025-spectrograms/birdclef2025_melspec_5sec_256_256_pseudo.npy'\n    pseudo_files = '/kaggle/input/birdclef2025-spectrograms/pseudo_files.txt'\n\n    model_name = 'efficientnet_b0'\n    pretrained = True\n    in_channels = 1\n\n    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n    epochs = 10  \n    batch_size = 64\n    criterion = 'BCELoss'\n\n    n_fold = 5\n    selected_folds = [0, 1, 2, 3, 4]   \n\n    optimizer = 'AdamW'\n    lr = 5e-4 \n    weight_decay = 1e-5\n  \n    scheduler = 'CosineAnnealingLR'\n    min_lr = 1e-6\n    T_max = epochs\n\n    aug_prob = 0.5  \n    mixup_start_alpha = 0.5\n    mixup_end_alpha = 0.5\n    mixup_enabled = True\n\n    loss_percentage = 3.0\n\n    use_base = False\n    \n    def update_debug_settings(self):\n        if self.debug:\n            self.epochs = 2\n            self.selected_folds = [0]\n\ncfg = CFG()","metadata":{"_uuid":"ed814b68-be00-4949-9372-0032ba7255a0","_cell_guid":"dd022215-2c58-4c5f-af12-de8e5f76511a","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Utilities","metadata":{"_uuid":"c6096590-dc7c-4792-b343-025e311b4b65","_cell_guid":"2cfe5438-f859-42bf-81aa-3600ac9e380a","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def set_seed(seed=42):\n    \"\"\"\n    Set seed for reproducibility\n    \"\"\"\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nset_seed(cfg.seed)","metadata":{"_uuid":"5e723370-e43d-422f-82a5-a1fd475a93a4","_cell_guid":"99ae96b1-3658-41a1-95a6-b657e40e901b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Dataset Preparation and Data Augmentations\nWe'll convert audio to mel spectrograms and apply random augmentations with 50% probability each - including time stretching, pitch shifting, and volume adjustments. This randomized approach creates diverse training samples from the same audio files","metadata":{"_uuid":"7fee228a-fb59-4e34-b311-024d7658223e","_cell_guid":"b5093a40-d548-40ef-92c9-7181c01dda7d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"class BirdCLEFDatasetFromNPY(Dataset):\n    def __init__(self, df, cfg, spectrograms=None, mode=\"train\"):\n        self.df = df\n        self.cfg = cfg\n        self.mode = mode\n\n        self.spectrograms = spectrograms\n        \n        taxonomy_df = pd.read_csv(self.cfg.taxonomy_csv)\n        self.species_ids = taxonomy_df['primary_label'].tolist()\n        self.num_classes = len(self.species_ids)\n        self.label_to_idx = {label: idx for idx, label in enumerate(self.species_ids)}\n\n        if 'filepath' not in self.df.columns:\n            self.df['filepath'] = self.cfg.train_datadir + '/' + self.df.filename\n        \n        if 'samplename' not in self.df.columns:\n            self.df['samplename'] = self.df.filename.map(lambda x: x.split('/')[0] + '-' + x.split('/')[-1].split('.')[0])\n\n        sample_names = set(self.df['samplename'])\n        if self.spectrograms:\n            found_samples = sum(1 for name in sample_names if name in self.spectrograms)\n            print(f\"Found {found_samples} matching spectrograms for {mode} dataset out of {len(self.df)} samples\")\n        \n        if cfg.debug:\n            self.df = self.df.sample(min(1000, len(self.df)), random_state=cfg.seed).reset_index(drop=True)\n\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        samplename = row['samplename']\n        spec = None\n\n        if self.spectrograms and samplename in self.spectrograms:\n            spec = self.spectrograms[samplename]\n\n        if spec is None:\n            #if self.mode == \"train\":  # Only print warning during training\n                print(f\"Warning: Spectrogram for {samplename} not found and could not be generated\")\n\n        spec = torch.tensor(spec, dtype=torch.float32).unsqueeze(0)  # Add channel dimension\n\n        if self.mode == \"train\" and random.random() < self.cfg.aug_prob:\n            spec = self.apply_spec_augmentations(spec)\n        \n        target = self.encode_label(row['primary_label'])\n\n        if 'secondary_labels' in row and row['secondary_labels'] not in [[''], None, np.nan]:\n            if isinstance(row['secondary_labels'], str):\n                secondary_labels = eval(row['secondary_labels'])\n            else:\n                secondary_labels = row['secondary_labels']\n            \n            for label in secondary_labels:\n                if label in self.label_to_idx:\n                    target[self.label_to_idx[label]] = 1.0\n        \n        return {\n            'melspec': spec, \n            'target': torch.tensor(target, dtype=torch.float32),\n            'filename': row['filename']\n        }\n    \n    def apply_spec_augmentations(self, spec):\n        \"\"\"Apply augmentations to spectrogram\"\"\"\n    \n        # Time masking (horizontal stripes)\n        if random.random() < 0.5:\n            num_masks = random.randint(1, 3)\n            for _ in range(num_masks):\n                width = random.randint(5, 20)\n                start = random.randint(0, spec.shape[2] - width)\n                spec[0, :, start:start+width] = 0\n        \n        # Frequency masking (vertical stripes)\n        if random.random() < 0.5:\n            num_masks = random.randint(1, 3)\n            for _ in range(num_masks):\n                height = random.randint(5, 20)\n                start = random.randint(0, spec.shape[1] - height)\n                spec[0, start:start+height, :] = 0\n\n        \n        # Random brightness/contrast\n        if random.random() < 0.5:\n            gain = random.uniform(0.8, 1.2)\n            bias = random.uniform(-0.1, 0.1)\n            spec = spec * gain + bias\n            spec = torch.clamp(spec, 0, 1) \n            \n        return spec\n    \n    def encode_label(self, label):\n        \"\"\"Encode label to one-hot vector\"\"\"\n        target = np.zeros(self.num_classes)\n        if label in self.label_to_idx:\n            target[self.label_to_idx[label]] = 1.0\n        return target","metadata":{"_uuid":"13a990d2-2f7f-4991-9551-1d67cd69e853","_cell_guid":"dad27bd0-77eb-431d-b7d0-1b88e5082407","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def collate_fn(batch):\n    \"\"\"Custom collate function to handle different sized spectrograms\"\"\"\n    batch = [item for item in batch if item is not None]\n    if len(batch) == 0:\n        return {}\n        \n    result = {key: [] for key in batch[0].keys()}\n    \n    for item in batch:\n        for key, value in item.items():\n            result[key].append(value)\n    \n    for key in result:\n        if key == 'target' and isinstance(result[key][0], torch.Tensor):\n            result[key] = torch.stack(result[key])\n        elif key == 'melspec' and isinstance(result[key][0], torch.Tensor):\n            shapes = [t.shape for t in result[key]]\n            if len(set(str(s) for s in shapes)) == 1:\n                result[key] = torch.stack(result[key])\n\n    return result","metadata":{"_uuid":"aed919b6-2734-489b-a149-4e4dc027d9e1","_cell_guid":"d9637055-c289-4623-a598-895600f3639b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training Utilities\nWe are configuring our optimization strategy with the AdamW optimizer, cosine scheduling, and the BCEWithLogitsLoss criterion.","metadata":{"_uuid":"9cf04fdc-87b4-4168-a88e-5a2ea6822acd","_cell_guid":"c710130a-b79d-4098-ab87-a5668c87edca","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def get_optimizer(model, cfg):\n  \n    if cfg.optimizer == 'Adam':\n        optimizer = optim.Adam(\n            model.parameters(),\n            lr=cfg.lr,\n            weight_decay=cfg.weight_decay\n        )\n    elif cfg.optimizer == 'AdamW':\n        optimizer = optim.AdamW(\n            model.parameters(),\n            lr=cfg.lr,\n            weight_decay=cfg.weight_decay\n        )\n    elif cfg.optimizer == 'SGD':\n        optimizer = optim.SGD(\n            model.parameters(),\n            lr=cfg.lr,\n            momentum=0.9,\n            weight_decay=cfg.weight_decay\n        )\n    else:\n        raise NotImplementedError(f\"Optimizer {cfg.optimizer} not implemented\")\n        \n    return optimizer\n\ndef get_scheduler(optimizer, cfg):\n\n    if cfg.scheduler == 'CosineAnnealingLR':\n        scheduler = lr_scheduler.CosineAnnealingLR(\n            optimizer,\n            T_max=cfg.T_max,\n            eta_min=cfg.min_lr\n        )\n    elif cfg.scheduler == 'ReduceLROnPlateau':\n        scheduler = lr_scheduler.ReduceLROnPlateau(\n            optimizer,\n            mode='min',\n            factor=0.5,\n            patience=2,\n            min_lr=cfg.min_lr,\n            verbose=True\n        )\n    elif cfg.scheduler == 'StepLR':\n        scheduler = lr_scheduler.StepLR(\n            optimizer,\n            step_size=cfg.epochs // 3,\n            gamma=0.5\n        )\n    elif cfg.scheduler == 'OneCycleLR':\n        scheduler = None  \n    else:\n        scheduler = None\n        \n    return scheduler\n\ndef get_criterion(cfg):\n \n    if cfg.criterion == 'BCELoss':\n        criterion = nn.BCEWithLogitsLoss(reduction=\"none\")\n    elif cfg.criterion == 'CELoss':\n        criterion = nn.CrossEntropyLoss(reduction=\"none\")\n    elif cfg.criterion == 'FocalBCELoss':\n        criterion = FocalLossBCE()\n    else:\n        raise NotImplementedError(f\"Criterion {cfg.criterion} not implemented\")\n        \n    return criterion","metadata":{"_uuid":"ff7d88b4-f4a9-4b9d-881d-be95e5a7a71f","_cell_guid":"7d84c214-a769-4d7e-b69e-3628168e219e","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model Definition","metadata":{"_uuid":"fe6b0bca-38dc-40f7-a404-9f260c31d9b7","_cell_guid":"bc4beae0-fbbb-4ab2-a17b-7f13bddb5c55","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"class FocalLossBCE(torch.nn.Module):\n    def __init__(\n            self,\n            alpha: float = 0.25,\n            gamma: float = 2,\n            reduction: str = \"none\",\n            bce_weight: float = 0.6,\n            focal_weight: float = 1.4,\n    ):\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.reduction = reduction\n        self.bce = torch.nn.BCEWithLogitsLoss(reduction=reduction)\n        self.bce_weight = bce_weight\n        self.focal_weight = focal_weight\n\n    def forward(self, logits, targets):\n        focal_loss = torchvision.ops.focal_loss.sigmoid_focal_loss(\n            inputs=logits,\n            targets=targets,\n            alpha=self.alpha,\n            gamma=self.gamma,\n            reduction=self.reduction,\n        )\n        #bce_loss = self.bce(logits, targets)\n        #return self.bce_weight * bce_loss + self.focal_weight * focal_loss\n        return focal_loss\n        \nclass BirdCLEFModel(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.cfg = cfg\n        \n        taxonomy_df = pd.read_csv(cfg.taxonomy_csv)\n        cfg.num_classes = len(taxonomy_df)\n        \n        self.backbone = timm.create_model(\n            cfg.model_name,\n            pretrained=cfg.pretrained,\n            in_chans=cfg.in_channels,\n            drop_rate=0.2,\n            drop_path_rate=0.2\n        )\n        \n        if 'efficientnet' in cfg.model_name:\n            backbone_out = self.backbone.classifier.in_features\n            self.backbone.classifier = nn.Identity()\n        elif 'resnet' in cfg.model_name:\n            backbone_out = self.backbone.fc.in_features\n            self.backbone.fc = nn.Identity()\n        else:\n            backbone_out = self.backbone.get_classifier().in_features\n            self.backbone.reset_classifier(0, '')\n        \n        self.pooling = nn.AdaptiveAvgPool2d(1)\n            \n        self.feat_dim = backbone_out\n        \n        self.classifier = nn.Linear(backbone_out, cfg.num_classes)\n\n            \n    def forward(self, x):\n\n        features = self.backbone(x)\n        \n        if isinstance(features, dict):\n            features = features['features']\n            \n        if len(features.shape) == 4:\n            features = self.pooling(features)\n            features = features.view(features.size(0), -1)\n        \n        logits = self.classifier(features)\n            \n        return logits","metadata":{"_uuid":"e015ec5d-d21c-4618-9852-8e56ab8d43b2","_cell_guid":"c5f6c06d-2e28-4249-8a75-b535147d46cf","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training Loop","metadata":{"_uuid":"790665a9-ee4e-4e29-870f-bccf4f66c8f8","_cell_guid":"774edd8e-207a-4c18-986b-3fe6687f6969","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def get_progressive_mixup_alpha(epoch, total_epochs, start_alpha=1.0, end_alpha=0.0):\n    \"\"\"\n    Linearly decays mixup alpha from start_alpha to end_alpha over training.\n    \"\"\"\n    return start_alpha * (1 - epoch / total_epochs) + end_alpha * (epoch / total_epochs)\n\n\ndef mixup_data(x, targets, cfg, epoch):\n    \"\"\"Applies mixup to the data batch with progressive alpha.\"\"\"\n    batch_size = x.size(0)\n\n    mixup_alpha = get_progressive_mixup_alpha(epoch, cfg.epochs, cfg.mixup_start_alpha, cfg.mixup_end_alpha)\n\n    lam = np.random.beta(mixup_alpha, mixup_alpha)\n    indices = torch.randperm(batch_size).to(cfg.device)\n    mixed_x = lam * x + (1 - lam) * x[indices]\n\n    return mixed_x, targets, targets[indices], lam\n\n\ndef mixup_criterion(criterion, pred, y_a, y_b, lam):\n    \"\"\"Applies mixup to the loss function\"\"\"\n    return lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b)\n\n\ndef filter_top_loss_mixup(inputs, outputs, targets, y_a, y_b, lam, criterion, top_percent=3.0):\n    \"\"\"\n    Filters out the top X% samples with highest mixup loss.\n    \"\"\"\n    assert 0.0 <= top_percent <= 100.0\n\n    # Compute per-sample mixup losses (shape: B x num_classes)\n    per_element_losses = mixup_criterion(criterion, outputs, y_a, y_b, lam)  \n\n    # Reduce to per-sample scalar losses by averaging over classes\n    if per_element_losses.ndim > 1:\n        per_sample_losses = per_element_losses.mean(dim=1)  # shape: (B,)\n    else:\n        per_sample_losses = per_element_losses  # already shape (B,)\n\n    # Determine number of samples to keep and get indices\n    keep_ratio = 1.0 - (top_percent / 100.0)\n    keep_count = max(1, int(len(per_sample_losses) * keep_ratio))\n    _, keep_idx = torch.topk(per_sample_losses, keep_count, largest=False)\n\n    # Filter targets and outputs accordingly\n    filtered_outputs = outputs[keep_idx]\n    filtered_targets = targets[keep_idx]\n    filtered_y_a = y_a[keep_idx]\n    filtered_y_b = y_b[keep_idx]\n\n    # Recompute filtered loss (still returns per-element losses)\n    filtered_loss = mixup_criterion(criterion, filtered_outputs, filtered_y_a,\n                                    filtered_y_b, lam).mean()\n\n    return filtered_loss, filtered_outputs, filtered_targets\n\n\ndef train_one_epoch(model, loader, optimizer, criterion, cfg, epoch, scheduler=None):\n\n    device = cfg.device\n    model.train()\n    losses = []\n    all_targets = []\n    all_outputs = []\n    \n    pbar = tqdm(enumerate(loader), total=len(loader), desc=\"Training\")\n    \n    for step, batch in pbar:\n    \n        if isinstance(batch['melspec'], list):\n            batch_outputs = []\n            batch_losses = []\n            \n            for i in range(len(batch['melspec'])):\n                inputs = batch['melspec'][i].unsqueeze(0).to(device)\n                targets = batch['target'][i].unsqueeze(0).to(device)\n                if cfg.mixup_enabled:\n                    mixed_inputs, targets_a, targets_b, lam = mixup_data(inputs, targets, cfg,\n                                                                        epoch)\n                    inputs = mixed_inputs\n                \n                optimizer.zero_grad()\n                outputs = model(inputs)\n                if cfg.mixup_enabled:\n                    loss, outputs, targets = filter_top_loss_mixup(inputs, outputs, targets,\n                                    targets_a, targets_b, lam, criterion, cfg.loss_percentage)\n                else:\n                    loss = criterion(outputs, targets).mean()\n\n                loss.backward()\n                \n                batch_outputs.append(outputs.detach().cpu())\n                batch_losses.append(loss.item())\n            \n            optimizer.step()\n            outputs = torch.cat(batch_outputs, dim=0).numpy()\n            loss = np.mean(batch_losses)\n            targets = batch['target'].numpy()\n            \n        else:\n            inputs = batch['melspec'].to(device)\n            targets = batch['target'].to(device)\n            if cfg.mixup_enabled:\n                mixed_inputs, targets_a, targets_b, lam = mixup_data(inputs, targets, cfg, epoch)\n                inputs = mixed_inputs\n            \n            optimizer.zero_grad()\n            outputs = model(inputs)\n            if cfg.mixup_enabled:\n                loss, outputs, targets = filter_top_loss_mixup(inputs, outputs, targets,\n                                targets_a, targets_b, lam, criterion, cfg.loss_percentage)\n            else:\n                loss = criterion(outputs, targets).mean()\n\n            loss.backward()\n            optimizer.step()\n            \n            outputs = outputs.detach().cpu().numpy()\n            targets = targets.detach().cpu().numpy()\n        \n        if scheduler is not None and isinstance(scheduler, lr_scheduler.OneCycleLR):\n            scheduler.step()\n            \n        all_outputs.append(outputs)\n        all_targets.append(targets)\n        losses.append(loss if isinstance(loss, float) else loss.item())\n        \n        pbar.set_postfix({\n            'train_loss': np.mean(losses[-10:]) if losses else 0,\n            'lr': optimizer.param_groups[0]['lr']\n        })\n    \n    all_outputs = np.concatenate(all_outputs)\n    all_targets = np.concatenate(all_targets)\n    auc = calculate_auc(all_targets, all_outputs)\n    avg_loss = np.mean(losses)\n    \n    return avg_loss, auc\n\ndef validate(model, loader, criterion, device):\n   \n    model.eval()\n    losses = []\n    all_targets = []\n    all_outputs = []\n    \n    with torch.no_grad():\n        for batch in tqdm(loader, desc=\"Validation\"):\n            if isinstance(batch['melspec'], list):\n                batch_outputs = []\n                batch_losses = []\n                \n                for i in range(len(batch['melspec'])):\n                    inputs = batch['melspec'][i].unsqueeze(0).to(device)\n                    target = batch['target'][i].unsqueeze(0).to(device)\n                    \n                    output = model(inputs)\n                    loss = criterion(output, target).mean()\n                    \n                    batch_outputs.append(output.detach().cpu())\n                    batch_losses.append(loss.item())\n                \n                outputs = torch.cat(batch_outputs, dim=0).numpy()\n                loss = np.mean(batch_losses)\n                targets = batch['target'].numpy()\n                \n            else:\n                inputs = batch['melspec'].to(device)\n                targets = batch['target'].to(device)\n                \n                outputs = model(inputs)\n                loss = criterion(outputs, targets).mean()\n                \n                outputs = outputs.detach().cpu().numpy()\n                targets = targets.detach().cpu().numpy()\n            \n            all_outputs.append(outputs)\n            all_targets.append(targets)\n            losses.append(loss if isinstance(loss, float) else loss.item())\n    \n    all_outputs = np.concatenate(all_outputs)\n    all_targets = np.concatenate(all_targets)\n    \n    auc = calculate_auc(all_targets, all_outputs)\n    avg_loss = np.mean(losses)\n    \n    return avg_loss, auc\n\ndef calculate_auc(targets, outputs):\n  \n    num_classes = targets.shape[1]\n    aucs = []\n    \n    probs = 1 / (1 + np.exp(-outputs))\n    \n    for i in range(num_classes):\n        \n        if np.sum(targets[:, i]) > 0:\n            class_auc = roc_auc_score(targets[:, i], probs[:, i])\n            aucs.append(class_auc)\n    \n    return np.mean(aucs) if aucs else 0.0","metadata":{"_uuid":"eae16100-0e41-460c-ab31-888792e45d10","_cell_guid":"2846c682-2d58-490f-93b9-b66912156ec8","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training!","metadata":{"_uuid":"11f5a1a3-2a1b-40e2-80ae-00039182f07e","_cell_guid":"c57663fb-5c88-420a-b39b-b402d787b3ff","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def enable_multiple_gpu(model):\n    return nn.DataParallel(model)\n\ndef generate_train_loader(df, train_idx, spectrograms, cfg):\n    train_df = df.iloc[train_idx].reset_index(drop=True)\n    print(f'Training set: {len(train_df)} samples')\n    train_dataset = BirdCLEFDatasetFromNPY(train_df, cfg, spectrograms=spectrograms, mode='train')\n    # Use normal shuffle for FocalBCELoss to avoid overfitting\n    train_loader = DataLoader(\n        train_dataset,\n        batch_size=cfg.batch_size,\n        shuffle=True,\n        num_workers=cfg.num_workers,\n        pin_memory=True,\n        collate_fn=collate_fn,\n        drop_last=True\n    )\n    \n    return train_loader\n\ndef generate_valid_loader(df, val_idx, spectrograms, cfg):\n    val_df = df.iloc[val_idx].reset_index(drop=True)\n    print(f'Validation set: {len(val_df)} samples')\n    val_dataset = BirdCLEFDatasetFromNPY(val_df, cfg, spectrograms=spectrograms, mode='valid')\n    val_loader = DataLoader(\n        val_dataset, \n        batch_size=cfg.batch_size, \n        shuffle=False, \n        num_workers=cfg.num_workers,\n        pin_memory=True,\n        collate_fn=collate_fn\n    )\n    return val_loader\n    \n\ndef run_training(df, cfg):\n    \"\"\"Training function that can either use pre-computed spectrograms or generate them on-the-fly\"\"\"\n\n    taxonomy_df = pd.read_csv(cfg.taxonomy_csv)\n    species_ids = taxonomy_df['primary_label'].tolist()\n    cfg.num_classes = len(species_ids)\n    \n    if cfg.debug:\n        cfg.update_debug_settings()\n\n    spectrograms = None\n    spectrograms_val = None\n    print(\"Loading pre-computed mel spectrograms from NPY file...\")\n    try:\n        spectrograms = np.load(cfg.spectrogram_npy, allow_pickle=True).item()\n        spectrograms_val = np.load(cfg.spectrogram_val_npy, allow_pickle=True).item()\n        print(f\"Loaded {len(spectrograms)} pre-computed mel spectrograms\")\n    except Exception as e:\n        print(f\"Error loading pre-computed spectrograms: {e}\")\n        print(\"Will generate spectrograms on-the-fly instead.\")\n        \n    skf = StratifiedKFold(n_splits=cfg.n_fold, shuffle=True, random_state=cfg.seed)\n    \n    best_scores = []\n    \n    for fold, (train_idx, val_idx) in enumerate(skf.split(df, df['primary_label'])):\n        if fold not in cfg.selected_folds:\n            continue\n        \n        print(f'\\n{\"=\"*30} Fold {fold} {\"=\"*30}')\n\n        train_loader = generate_train_loader(df, train_idx, spectrograms, cfg)\n        val_loader = generate_valid_loader(df, val_idx, spectrograms_val, cfg)\n\n        model = BirdCLEFModel(cfg).to(cfg.device)\n        if cfg.use_base:\n            checkpoint = torch.load('/kaggle/input/pub-bird25-b-422-ppv15-v2-s-focallossbce/model_0.9605.pth',\n                                    map_location=torch.device(cfg.device))\n            model.load_state_dict(checkpoint['model_state_dict'])\n\n        if cfg.device == \"cuda\" and torch.cuda.device_count() > 1:\n            model = enable_multiple_gpu(model)\n        optimizer = get_optimizer(model, cfg)\n        criterion = get_criterion(cfg)\n        \n        if cfg.scheduler == 'OneCycleLR':\n            scheduler = lr_scheduler.OneCycleLR(\n                optimizer,\n                max_lr=cfg.lr,\n                steps_per_epoch=len(train_loader),\n                epochs=cfg.epochs,\n                pct_start=0.1\n            )\n        else:\n            scheduler = get_scheduler(optimizer, cfg)\n        \n        best_auc = 0\n        best_epoch = 0\n\n        val_loss, val_auc = validate(model, val_loader, criterion, cfg.device)\n        print(f\"Val Loss: {val_loss:.4f}, Val AUC: {val_auc:.4f}\")\n        \n        for epoch in range(cfg.epochs):\n            print(f\"\\nEpoch {epoch+1}/{cfg.epochs}\")\n            \n            train_loss, train_auc = train_one_epoch(\n                model, \n                train_loader, \n                optimizer, \n                criterion, \n                cfg,\n                epoch,\n                scheduler if isinstance(scheduler, lr_scheduler.OneCycleLR) else None\n            )\n            \n            val_loss, val_auc = validate(model, val_loader, criterion, cfg.device)\n\n            if scheduler is not None and not isinstance(scheduler, lr_scheduler.OneCycleLR):\n                if isinstance(scheduler, lr_scheduler.ReduceLROnPlateau):\n                    scheduler.step(val_loss)\n                else:\n                    scheduler.step()\n\n            print(f\"Train Loss: {train_loss:.4f}, Train AUC: {train_auc:.4f}\")\n            print(f\"Val Loss: {val_loss:.4f}, Val AUC: {val_auc:.4f}\")\n            \n            if val_auc > best_auc:\n                best_auc = val_auc\n                best_epoch = epoch + 1\n                print(f\"New best AUC: {best_auc:.4f} at epoch {best_epoch}\")\n\n                torch.save({\n                    'model_state_dict': model.state_dict(),\n                    'optimizer_state_dict': optimizer.state_dict(),\n                    'scheduler_state_dict': scheduler.state_dict() if scheduler else None,\n                    'epoch': epoch,\n                    'val_auc': val_auc,\n                    'train_auc': train_auc,\n                    'cfg': cfg\n                }, f\"model_fold{fold}.pth\")#_epoch{epoch}.pth\")\n            \n        best_scores.append(best_auc)\n        print(f\"\\nBest AUC for fold {fold}: {best_auc:.4f} at epoch {best_epoch}\")\n        \n        # Clear memory\n        del model, optimizer, scheduler, train_loader, val_loader\n        torch.cuda.empty_cache()\n        gc.collect()\n    \n    print(\"\\n\" + \"=\"*60)\n    print(\"Cross-Validation Results:\")\n    for fold, score in enumerate(best_scores):\n        print(f\"Fold {cfg.selected_folds[fold]}: {score:.4f}\")\n    print(f\"Mean AUC: {np.mean(best_scores):.4f}\")\n    print(\"=\"*60)","metadata":{"_uuid":"4fad512c-e0da-4640-bceb-c8ccb0280e9d","_cell_guid":"cfdda655-9144-49c0-bdc7-8d9a8cac1a1a","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def drop_discarded_files(train_df, discarded_txt_path):\n    \"\"\"\n    Removes rows from train_df whose filenames appear in the discarded files txt.\n    \"\"\"\n    prefix_to_strip = '/kaggle/input/birdclef-2025/train_audio/'\n    with open(discarded_txt_path, 'r') as f:\n        discarded_list = [line.strip().replace(prefix_to_strip, '') for line in f.readlines()]\n\n    filtered_df = train_df[~train_df['filename'].isin(discarded_list)].reset_index(drop=True)\n    return filtered_df","metadata":{"_uuid":"23660fb4-62bc-4016-9e94-3f7f3fd872b1","_cell_guid":"345686c3-cef5-47b4-928d-6c3bcca48710","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    import time\n    cfg.selected_folds = [3,4]\n    cfg.criterion = 'FocalBCELoss'\n    cfg.epochs = 15\n    cfg.T_max = 15\n    cfg.batch_size = 32\n    cfg.loss_percentage = 0.0\n    cfg.mixup_start_alpha = 0.3\n    cfg.mixup_end_alpha = 0.3\n    cfg.aug_prob = 0.75\n    cfg.mixup_enabled = True\n    #cfg.model_name = 'tf_efficientnetv2_s.in21k_ft_in1k'\n    #cfg.use_base = True\n    #cfg.debug = True\n\n    print(\"\\nLoading training data...\")\n    train_df = pd.read_csv(cfg.train_csv)\n    train_df = drop_discarded_files(train_df, cfg.discarded_files_random)\n    train_df = drop_discarded_files(train_df, cfg.discarded_files_first)\n    taxonomy_df = pd.read_csv(cfg.taxonomy_csv)\n\n    print(\"\\nStarting training...\")\n    print(\"Using pre-computed mel spectrograms from NPY file\")\n    \n    run_training(train_df, cfg)\n    \n    print(\"\\nTraining complete!\")","metadata":{"_uuid":"a0b7d601-afce-409d-945d-7882677dec40","_cell_guid":"48e68818-68ed-4cb5-8953-59f2f364eea4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}