{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.13"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":11053663,"sourceType":"datasetVersion","datasetId":6886569}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport time\nimport math\nimport random\nimport logging\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport librosa\nimport timm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import train_test_split \nfrom sklearn.metrics import roc_auc_score\nfrom tqdm.auto import tqdm\n\n# Suppress warnings and configure logging\nwarnings.filterwarnings(\"ignore\")\nlogging.basicConfig(level=logging.ERROR)\n\n# Configuration Class\nclass CFG:\n    seed = 42\n    \n    # Set to True for quick debugging (fewer epochs, small subset)\n    debug = False  \n    \n    # Input Paths (Kaggle Input is Read-Only)\n    train_datadir = '/kaggle/input/birdclef-2025/train_audio'\n    train_csv = '/kaggle/input/birdclef-2025/train.csv'\n    taxonomy_csv = '/kaggle/input/birdclef-2025/taxonomy.csv'\n    \n    # Path to pre-computed spectrograms (otherwise use CPU to compute would be very slow)\n    spectrogram_npy = '/kaggle/input/birdclef25-mel-spectrograms/birdclef2025_melspec_5sec_256_256.npy'\n    \n    # Output Path (Must be in /kaggle/working/ to save files)\n    output_dir = '/kaggle/working/model' \n    \n    # Audio & Spectrogram Parameters\n    FS = 32000              # Sampling Rate\n    TARGET_DURATION = 5.0   # Duration in seconds\n    TARGET_SHAPE = (256, 256) # Image Shape (H, W)\n    N_FFT = 1024\n    HOP_LENGTH = 512\n    N_MELS = 128\n    FMIN = 50\n    FMAX = 14000\n    \n    # Model Hyperparameters\n    model_name = 'efficientnet_b0'\n    pretrained = True\n    in_channels = 1         # 1 for grayscale (spectrogram), 3 for RGB\n    num_classes = 0         # Updated dynamically based on taxonomy\n    \n    # Training Hyperparameters\n    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n    epochs = 10\n    batch_size = 32\n    num_workers = 2\n    \n    # Optimizer & Scheduler\n    lr = 5e-4\n    weight_decay = 1e-5\n    min_lr = 1e-6\n    \n    # Validation Strategy (Single Split)\n    val_pct = 0.2  # 20% for Validation, 80% for Training\n    \n    # Data Augmentation\n    aug_prob = 0.5\n    mixup_alpha = 0.5       # Mixup strength\n    \n    # Data Loading Strategy\n    LOAD_DATA = True        # Attempt to load .npy file first\n    \n    def update_debug_settings(self):\n        \"\"\"Adjusts settings for quick debugging\"\"\"\n        if self.debug:\n            print(\"DEBUG MODE ENABLED: Running fewer epochs on a subset.\")\n            self.epochs = 2\n\ncfg = CFG()\n\n\n# Utilities\ndef set_seed(seed=42):\n    \"\"\"Sets the random seed for reproducibility.\"\"\"\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nset_seed(cfg.seed)\n\n\n# Pre-processing Functions\ndef audio2melspec(audio_data, cfg):\n    \"\"\"\n    Converts raw audio waveform to a Log-Mel Spectrogram.\n    Returns a normalized numpy array.\n    \"\"\"\n    # Handle NaNs\n    if np.isnan(audio_data).any():\n        mean_signal = np.nanmean(audio_data)\n        audio_data = np.nan_to_num(audio_data, nan=mean_signal)\n\n    # Compute Mel Spectrogram\n    mel_spec = librosa.feature.melspectrogram(\n        y=audio_data,\n        sr=cfg.FS,\n        n_fft=cfg.N_FFT,\n        hop_length=cfg.HOP_LENGTH,\n        n_mels=cfg.N_MELS,\n        fmin=cfg.FMIN,\n        fmax=cfg.FMAX,\n        power=2.0\n    )\n\n    # Convert to dB (Log scale)\n    mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max)\n    \n    # Min-Max Normalization to [0, 1]\n    mel_spec_norm = (mel_spec_db - mel_spec_db.min()) / (mel_spec_db.max() - mel_spec_db.min() + 1e-8)\n    \n    return mel_spec_norm\n\ndef process_audio_file(audio_path, cfg):\n    \"\"\"\n    Loads an audio file, crops/pads it to 5 seconds, and returns the spectrogram.\n    \"\"\"\n    try:\n        # Load audio\n        audio_data, _ = librosa.load(audio_path, sr=cfg.FS)\n        target_samples = int(cfg.TARGET_DURATION * cfg.FS)\n\n        # Padding if too short\n        if len(audio_data) < target_samples:\n            n_copy = math.ceil(target_samples / len(audio_data))\n            if n_copy > 1:\n                audio_data = np.concatenate([audio_data] * n_copy)\n\n        # Crop the center 5 seconds\n        start_idx = max(0, int(len(audio_data) / 2 - target_samples / 2))\n        end_idx = min(len(audio_data), start_idx + target_samples)\n        center_audio = audio_data[start_idx:end_idx]\n\n        # Final check for padding\n        if len(center_audio) < target_samples:\n            center_audio = np.pad(center_audio, (0, target_samples - len(center_audio)), mode='constant')\n\n        # Convert to Spectrogram\n        mel_spec = audio2melspec(center_audio, cfg)\n        \n        # Resize if necessary (ensure consistent input size)\n        if mel_spec.shape != cfg.TARGET_SHAPE:\n            mel_spec = cv2.resize(mel_spec, cfg.TARGET_SHAPE, interpolation=cv2.INTER_LINEAR)\n\n        return mel_spec.astype(np.float32)\n        \n    except Exception as e:\n        print(f\"Error processing {audio_path}: {e}\")\n        return None\n\n\n# Dataset Class\nclass BirdCLEFDataset(Dataset):\n    def __init__(self, df, cfg, spectrograms=None, mode=\"train\"):\n        self.df = df\n        self.cfg = cfg\n        self.mode = mode\n        self.spectrograms = spectrograms # Pre-loaded .npy dictionary\n        \n        # Load Taxonomy to map labels to integers\n        taxonomy_df = pd.read_csv(self.cfg.taxonomy_csv)\n        self.species_ids = taxonomy_df['primary_label'].tolist()\n        self.num_classes = len(self.species_ids)\n        self.label_to_idx = {label: idx for idx, label in enumerate(self.species_ids)}\n\n        # Prepare file paths\n        if 'filepath' not in self.df.columns:\n            self.df['filepath'] = self.cfg.train_datadir + '/' + self.df.filename\n        \n        # Create 'samplename' key for dictionary lookup\n        if 'samplename' not in self.df.columns:\n            self.df['samplename'] = self.df.filename.map(lambda x: x.split('/')[0] + '-' + x.split('/')[-1].split('.')[0])\n        \n        # Debugging: subset data\n        if cfg.debug and len(self.df) > 1000:\n            self.df = self.df.sample(1000, random_state=cfg.seed).reset_index(drop=True)\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        samplename = row['samplename']\n        spec = None\n\n        # Strategy 1: Load from RAM (Fastest)\n        if self.spectrograms and samplename in self.spectrograms:\n            spec = self.spectrograms[samplename]\n        # Strategy 2: Load from Disk (Slower, processes audio on the fly)\n        elif not self.cfg.LOAD_DATA:\n            spec = process_audio_file(row['filepath'], self.cfg)\n\n        # Fallback for missing data\n        if spec is None:\n            spec = np.zeros(self.cfg.TARGET_SHAPE, dtype=np.float32)\n            \n        # Add Channel Dimension: (H, W) -> (1, H, W)\n        spec = torch.tensor(spec, dtype=torch.float32).unsqueeze(0)\n\n        # Apply Augmentations (Only during training)\n        if self.mode == \"train\" and random.random() < self.cfg.aug_prob:\n            spec = self.apply_spec_augmentations(spec)\n        \n        # Encode Labels (One-Hot Encoding for Multi-Label)\n        target = self.encode_label(row['primary_label'])\n        \n        # Handle Secondary Labels (Background birds)\n        if 'secondary_labels' in row and isinstance(row['secondary_labels'], str):\n            try:\n                secondary_labels = eval(row['secondary_labels'])\n                for label in secondary_labels:\n                    if label in self.label_to_idx:\n                        target[self.label_to_idx[label]] = 1.0\n            except:\n                pass\n        \n        return {\n            'melspec': spec, \n            'target': torch.tensor(target, dtype=torch.float32)\n        }\n    \n    def apply_spec_augmentations(self, spec):\n        \"\"\"Applies SpecAugment: Time masking and Frequency masking\"\"\"\n        # Time masking\n        if random.random() < 0.5:\n            num_masks = random.randint(1, 3)\n            for _ in range(num_masks):\n                width = random.randint(5, 20)\n                start = random.randint(0, spec.shape[2] - width)\n                spec[0, :, start:start+width] = 0\n        \n        # Frequency masking\n        if random.random() < 0.5:\n            num_masks = random.randint(1, 3)\n            for _ in range(num_masks):\n                height = random.randint(5, 20)\n                start = random.randint(0, spec.shape[1] - height)\n                spec[0, start:start+height, :] = 0\n            \n        return spec\n    \n    def encode_label(self, label):\n        \"\"\"Creates a One-Hot vector for the target label\"\"\"\n        target = np.zeros(self.num_classes)\n        if label in self.label_to_idx:\n            target[self.label_to_idx[label]] = 1.0\n        return target\n\ndef collate_fn(batch):\n    \"\"\"Custom collate function to handle batches.\"\"\"\n    batch = [item for item in batch if item is not None]\n    if len(batch) == 0: return {}\n    \n    melspecs = torch.stack([item['melspec'] for item in batch])\n    targets = torch.stack([item['target'] for item in batch])\n    \n    return {'melspec': melspecs, 'target': targets}\n\n\n# Model Architecture\nclass BirdCLEFModel(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.cfg = cfg\n        \n        # Load Backbone using timm (Transfer Learning)\n        self.backbone = timm.create_model(\n            cfg.model_name,\n            pretrained=cfg.pretrained,\n            in_chans=cfg.in_channels,\n            drop_rate=0.2,\n            drop_path_rate=0.2\n        )\n        \n        # Replace the original classification head\n        if 'efficientnet' in cfg.model_name:\n            backbone_out = self.backbone.classifier.in_features\n            self.backbone.classifier = nn.Identity()\n        else:\n            backbone_out = self.backbone.num_features\n            self.backbone.reset_classifier(0, '')\n        \n        # Global Average Pooling\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        \n        # New Classification Head\n        self.classifier = nn.Linear(backbone_out, cfg.num_classes)\n        \n        # Mixup Settings\n        self.mixup_enabled = hasattr(cfg, 'mixup_alpha') and cfg.mixup_alpha > 0\n            \n    def forward(self, x, targets=None):\n        # Apply Mixup during training\n        if self.training and self.mixup_enabled and targets is not None:\n            mixed_x, targets_a, targets_b, lam = self.mixup_data(x, targets)\n            x = mixed_x\n        else:\n            targets_a, targets_b, lam = None, None, None\n        \n        # Feature Extraction\n        features = self.backbone(x)\n        \n        # Handle different output formats from timm\n        if isinstance(features, dict):\n            features = features['features']\n        \n        # Pooling (B, C, H, W) -> (B, C, 1, 1) -> (B, C)\n        if len(features.shape) == 4:\n            features = self.pooling(features)\n            features = features.view(features.size(0), -1)\n        \n        # Classification\n        logits = self.classifier(features)\n        \n        # Return loss directly if using Mixup\n        if self.training and self.mixup_enabled and targets is not None:\n            loss = self.mixup_criterion(nn.BCEWithLogitsLoss(), logits, targets_a, targets_b, lam)\n            return logits, loss\n            \n        return logits\n    \n    def mixup_data(self, x, targets):\n        \"\"\"Mixes two images and their labels.\"\"\"\n        batch_size = x.size(0)\n        lam = np.random.beta(self.cfg.mixup_alpha, self.cfg.mixup_alpha)\n        indices = torch.randperm(batch_size).to(x.device)\n        mixed_x = lam * x + (1 - lam) * x[indices]\n        return mixed_x, targets, targets[indices], lam\n    \n    def mixup_criterion(self, criterion, pred, y_a, y_b, lam):\n        \"\"\"Weighted loss for mixed samples.\"\"\"\n        return lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b)\n\n\n# Training Engine\ndef train_one_epoch(model, loader, optimizer, criterion, device, scheduler=None):\n    model.train()\n    losses = []\n    all_targets = []\n    all_outputs = []\n    \n    pbar = tqdm(loader, desc=\"Training\")\n    \n    for batch in pbar:\n        inputs = batch['melspec'].to(device)\n        targets = batch['target'].to(device)\n        \n        optimizer.zero_grad()\n        \n        # Forward pass (handles mixup internally)\n        outputs = model(inputs, targets)\n        \n        # Calculate loss\n        if isinstance(outputs, tuple):\n            outputs, loss = outputs # Mixup returns (logits, loss)\n        else:\n            loss = criterion(outputs, targets)\n            \n        loss.backward()\n        optimizer.step()\n        \n        if scheduler is not None:\n            scheduler.step()\n            \n        losses.append(loss.item())\n        \n        # Store for AUC calculation\n        all_outputs.append(outputs.detach().cpu().numpy())\n        all_targets.append(targets.detach().cpu().numpy())\n        \n        pbar.set_postfix({'loss': np.mean(losses[-10:])})\n    \n    # Calculate Metrics\n    all_outputs = np.concatenate(all_outputs)\n    all_targets = np.concatenate(all_targets)\n    auc = calculate_auc(all_targets, all_outputs)\n    \n    return np.mean(losses), auc\n\n@torch.no_grad()\ndef validate(model, loader, criterion, device):\n    model.eval()\n    losses = []\n    all_targets = []\n    all_outputs = []\n    \n    for batch in tqdm(loader, desc=\"Validation\"):\n        inputs = batch['melspec'].to(device)\n        targets = batch['target'].to(device)\n        \n        outputs = model(inputs)\n        loss = criterion(outputs, targets)\n        \n        losses.append(loss.item())\n        all_outputs.append(outputs.cpu().numpy())\n        all_targets.append(targets.cpu().numpy())\n        \n    all_outputs = np.concatenate(all_outputs)\n    all_targets = np.concatenate(all_targets)\n    auc = calculate_auc(all_targets, all_outputs)\n    \n    return np.mean(losses), auc\n\ndef calculate_auc(targets, outputs):\n    \"\"\"Calculates Mean ROC-AUC score across all classes.\"\"\"\n    num_classes = targets.shape[1]\n    aucs = []\n    probs = 1 / (1 + np.exp(-outputs)) # Sigmoid\n    \n    for i in range(num_classes):\n        # Only calculate AUC if the class is present in the batch\n        if np.sum(targets[:, i]) > 0:\n            try:\n                class_auc = roc_auc_score(targets[:, i], probs[:, i])\n                aucs.append(class_auc)\n            except:\n                pass\n    return np.mean(aucs) if aucs else 0.0\n\n\n# Main Execution Loop (Single Split)\ndef run_training(df, cfg):\n    # 0. Ensure Output Directory Exists\n    if not os.path.exists(cfg.output_dir):\n        print(f\"Creating output directory: {cfg.output_dir}\")\n        os.makedirs(cfg.output_dir, exist_ok=True)\n\n    # Setup Data Metadata\n    taxonomy_df = pd.read_csv(cfg.taxonomy_csv)\n    cfg.num_classes = len(taxonomy_df)\n    \n    if cfg.debug:\n        cfg.update_debug_settings()\n\n    # Try to load pre-computed spectrograms (Speed optimization)\n    spectrograms = None\n    if cfg.LOAD_DATA:\n        print(\"Attempting to load pre-computed mel spectrograms...\")\n        try:\n            spectrograms = np.load(cfg.spectrogram_npy, allow_pickle=True).item()\n            print(f\"Successfully loaded {len(spectrograms)} spectrograms.\")\n        except Exception as e:\n            print(f\"Failed to load NPY file: {e}\")\n            print(\"Switching to on-the-fly generation (Slower).\")\n            cfg.LOAD_DATA = False\n    \n    print(f'\\n{\"=\"*30} Training Start (Single Split) {\"=\"*30}')\n    \n    # Create Train/Val Split\n    # We use stratified split to ensure all bird classes are represented in validation\n    train_df, val_df = train_test_split(\n        df, \n        test_size=cfg.val_pct, \n        random_state=cfg.seed, \n        stratify=df['primary_label'] \n    )\n    \n    train_df = train_df.reset_index(drop=True)\n    val_df = val_df.reset_index(drop=True)\n    \n    print(f\"Train Size: {len(train_df)} | Val Size: {len(val_df)}\")\n    \n    # 4. Create Datasets & Loaders\n    train_dataset = BirdCLEFDataset(train_df, cfg, spectrograms, mode='train')\n    val_dataset = BirdCLEFDataset(val_df, cfg, spectrograms, mode='valid')\n    \n    train_loader = DataLoader(train_dataset, batch_size=cfg.batch_size, shuffle=True, \n                              num_workers=cfg.num_workers, collate_fn=collate_fn, drop_last=True)\n    val_loader = DataLoader(val_dataset, batch_size=cfg.batch_size, shuffle=False, \n                            num_workers=cfg.num_workers, collate_fn=collate_fn)\n    \n    # Initialize Model & Optimizer\n    model = BirdCLEFModel(cfg).to(cfg.device)\n    optimizer = optim.AdamW(model.parameters(), lr=cfg.lr, weight_decay=cfg.weight_decay)\n    criterion = nn.BCEWithLogitsLoss()\n    \n    # Scheduler (OneCycleLR for faster convergence)\n    scheduler = lr_scheduler.OneCycleLR(\n        optimizer, max_lr=cfg.lr, steps_per_epoch=len(train_loader), \n        epochs=cfg.epochs, pct_start=0.1\n    )\n    \n    # Training Loop\n    best_auc = 0\n    for epoch in range(cfg.epochs):\n        print(f\"Epoch {epoch+1}/{cfg.epochs}\")\n        \n        train_loss, train_auc = train_one_epoch(model, train_loader, optimizer, criterion, cfg.device, scheduler)\n        val_loss, val_auc = validate(model, val_loader, criterion, cfg.device)\n        \n        print(f\"Train Loss: {train_loss:.4f} | Train AUC: {train_auc:.4f}\")\n        print(f\"Val Loss:   {val_loss:.4f} | Val AUC:   {val_auc:.4f}\")\n        \n        # Save Best Model\n        if val_auc > best_auc:\n            best_auc = val_auc\n            save_path = os.path.join(cfg.output_dir, \"best_model.pth\")\n            torch.save({'model_state_dict': model.state_dict(), 'cfg': cfg}, save_path)\n            print(f\"--> Saved Best Model to {save_path} (AUC: {best_auc:.4f})\")\n    \n    # Cleanup\n    del model, optimizer, scheduler, train_loader, val_loader\n    torch.cuda.empty_cache()\n    gc.collect()\n\nif __name__ == \"__main__\":\n    print(f\"Starting Training on {cfg.device}...\")\n    \n    # Load Metadata\n    train_df = pd.read_csv(cfg.train_csv)\n    \n    # Run Training\n    run_training(train_df, cfg)","metadata":{"execution":{"iopub.execute_input":"2025-12-08T10:55:40.431084Z","iopub.status.busy":"2025-12-08T10:55:40.430298Z","iopub.status.idle":"2025-12-08T11:18:29.844479Z","shell.execute_reply":"2025-12-08T11:18:29.843457Z","shell.execute_reply.started":"2025-12-08T10:55:40.431054Z"},"trusted":true},"outputs":[],"execution_count":null}]}