{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"},{"sourceId":11053663,"sourceType":"datasetVersion","datasetId":6886569},{"sourceId":11657133,"sourceType":"datasetVersion","datasetId":7315384}],"dockerImageVersionId":30919,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **BirdCLEF 2025 Training Notebook**\n\nThis is a baseline training pipeline for BirdCLEF 2025 using EfficientNetB0 with PyTorch and Timm(for pretrained EffNet). You can check inference and preprocessing notebooks in the following links: \n\n- [EfficientNet B0 Pytorch [Inference] | BirdCLEF'25](https://www.kaggle.com/code/kadircandrisolu/efficientnet-b0-pytorch-inference-birdclef-25)\n\n  \n- [Transforming Audio-to-Mel Spec. | BirdCLEF'25](https://www.kaggle.com/code/kadircandrisolu/transforming-audio-to-mel-spec-birdclef-25)  \n\nNote that by default this notebook is in Debug Mode, so it will only train the model with 2 epochs, but the [weight](https://www.kaggle.com/datasets/kadircandrisolu/birdclef25-effnetb0-starter-weight) I used in the inference notebook was obtained after 10 epochs of training.\n\n**Features**\n* Implement with Pytorch and Timm\n* Flexible audio processing with both pre-computed and on-the-fly mel spectrograms\n* Stratified 5-fold cross-validation with ensemble capability\n* Mixup training for improved generalization\n* Spectrogram augmentations (time/frequency masking, brightness adjustment)\n* AdamW optimizer with Cosine Annealing LR scheduling\n* Debug mode for quick experimentation with smaller datasets\n\n**Pre-computed Spectrograms**\nFor faster training, you can use pre-computed mel spectrograms from [this dataset](https://www.kaggle.com/datasets/kadircandrisolu/birdclef25-mel-spectrograms) by setting `LOAD_DATA = True`","metadata":{}},{"cell_type":"markdown","source":"## Libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport logging\nimport random\nimport gc\nimport time\nimport cv2\nimport math\nimport warnings\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nimport librosa\nimport librosa.effects\nimport glob\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.auto import tqdm\n\nimport timm\n\nwarnings.filterwarnings(\"ignore\")\nlogging.basicConfig(level=logging.ERROR)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T17:23:43.913637Z","iopub.execute_input":"2025-06-07T17:23:43.914161Z","iopub.status.idle":"2025-06-07T17:23:59.287107Z","shell.execute_reply.started":"2025-06-07T17:23:43.914117Z","shell.execute_reply":"2025-06-07T17:23:59.285603Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Configuration","metadata":{}},{"cell_type":"code","source":"class CFG:\n    seed = 42\n    debug = True  \n    apex = False\n    print_freq = 100\n    num_workers = 2\n    \n    # GENTLE AUGMENTATION PARAMETERS (reduced from aggressive values)\n    noise_aug_prob = 0.4  # Reduced from 0.7 to 0.4\n    noise_samples_dir = '/kaggle/input/background-noise-samples/ESC-50-master/audio/'\n    \n    # GENTLE noise augmentation parameters (much more conservative)\n    white_noise_factor = 0.002    # Reduced from 0.005\n    gaussian_noise_factor = 0.003  # Reduced from 0.01\n    background_noise_factor = 0.1  # Reduced from 0.3\n    time_shift_max = 0.05          # Reduced from 0.2\n    reverb_room_size = 0.3         # Reduced from 0.8\n    reverb_decay = 0.2             # Reduced from 0.5\n    \n    # GENTLE spectrogram augmentation parameters\n    spec_aug_time_mask_param = 8   # Reduced from 24\n    spec_aug_freq_mask_param = 8   # Reduced from 24\n    time_stretch_prob = 0.3        # Reduced from 0.5\n    time_stretch_factors = (0.95, 1.05)  # Reduced from (0.8, 1.2) to (95%-105%)\n    \n    OUTPUT_DIR = '/kaggle/working/'\n\n    train_datadir = '/kaggle/input/birdclef-2025/train_audio'\n    train_csv = '/kaggle/input/birdclef-2025/train.csv'\n    test_soundscapes = '/kaggle/input/birdclef-2025/test_soundscapes'\n    submission_csv = '/kaggle/input/birdclef-2025/sample_submission.csv'\n    taxonomy_csv = '/kaggle/input/birdclef-2025/taxonomy.csv'\n\n    spectrogram_npy = '/kaggle/input/birdclef25-mel-spectrograms/birdclef2025_melspec_5sec_256_256.npy'\n \n    model_name = 'efficientnet_b3'\n    pretrained = True\n    in_channels = 1\n\n    LOAD_DATA = True  \n    FS = 32000\n    TARGET_DURATION = 5.0\n    TARGET_SHAPE = (256, 256)\n    \n    N_FFT = 1024\n    HOP_LENGTH = 512\n    N_MELS = 128\n    FMIN = 50\n    FMAX = 14000\n    \n    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n    epochs = 20\n    batch_size = 16\n    criterion = 'FocalLoss'\n\n    n_fold = 5\n    selected_folds = [0, 1, 2, 3, 4]   \n\n    optimizer = 'AdamW'\n    lr = 3e-4\n    weight_decay = 1e-5\n  \n    scheduler = 'CosineAnnealingLR'\n    min_lr = 1e-6\n    T_max = epochs\n\n    aug_prob = 0.5  # Reduced from 0.7 to 0.5 (more conservative)\n    mixup_alpha = 0.3  # Reduced from 0.5 for gentler mixing\n    \n    train_multiple_models = False\n    models_to_train = [\n        {'name': 'efficientnet_b3', 'selected_folds': [0, 1, 2, 3, 4]},\n        {'name': 'efficientnet_b4', 'selected_folds': [0, 1, 2, 3, 4]},\n    ]\n    \n    def update_debug_settings(self):\n        if self.debug:\n            self.epochs = 2\n            self.selected_folds = [0]\n            self.train_multiple_models = False\n\ncfg = CFG()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T17:23:59.289267Z","iopub.execute_input":"2025-06-07T17:23:59.289789Z","iopub.status.idle":"2025-06-07T17:23:59.304693Z","shell.execute_reply.started":"2025-06-07T17:23:59.289729Z","shell.execute_reply":"2025-06-07T17:23:59.303327Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def set_seed(seed=42):\n    \"\"\"\n    Set seed for reproducibility\n    \"\"\"\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nset_seed(cfg.seed)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T17:23:59.306549Z","iopub.execute_input":"2025-06-07T17:23:59.307011Z","iopub.status.idle":"2025-06-07T17:23:59.340800Z","shell.execute_reply.started":"2025-06-07T17:23:59.306958Z","shell.execute_reply":"2025-06-07T17:23:59.339296Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self, gamma=2.0, alpha=None, reduction='mean'):\n        super(FocalLoss, self).__init__()\n        self.gamma = gamma\n        self.alpha = alpha\n        self.reduction = reduction\n        \n    def forward(self, inputs, targets):\n        BCE_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduction='none')\n        pt = torch.exp(-BCE_loss)\n        F_loss = (1-pt)**self.gamma * BCE_loss\n        \n        if self.reduction == 'mean':\n            return torch.mean(F_loss)\n        else:\n            return F_loss","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T17:23:59.342734Z","iopub.execute_input":"2025-06-07T17:23:59.343378Z","iopub.status.idle":"2025-06-07T17:23:59.353466Z","shell.execute_reply.started":"2025-06-07T17:23:59.343297Z","shell.execute_reply":"2025-06-07T17:23:59.352019Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def add_gentle_white_noise(audio, noise_factor=0.002):\n    \"\"\"Add very gentle white noise\"\"\"\n    noise = np.random.randn(len(audio))\n    augmented_audio = audio + noise_factor * noise\n    return np.clip(augmented_audio, -1.0, 1.0)\n\ndef add_gentle_background_noise(audio, background_noise, noise_factor=0.1):\n    \"\"\"Add gentle background noise (much reduced factor)\"\"\"\n    if len(background_noise) < len(audio):\n        repeats = int(np.ceil(len(audio) / len(background_noise)))\n        background_noise = np.tile(background_noise, repeats)\n    \n    background_noise = background_noise[:len(audio)]\n    augmented_audio = audio + noise_factor * background_noise\n    return np.clip(augmented_audio, -1.0, 1.0)\n\ndef add_gentle_gaussian_noise(audio, noise_factor=0.003):\n    \"\"\"Add gentle Gaussian noise\"\"\"\n    noise = np.random.normal(0, 1, len(audio))\n    augmented_audio = audio + noise_factor * noise\n    return np.clip(augmented_audio, -1.0, 1.0)\n\ndef gentle_time_shift(audio, shift_max=0.05):\n    \"\"\"Very gentle time shifting (5% instead of 20%)\"\"\"\n    shift = np.random.randint(-int(shift_max * len(audio)), int(shift_max * len(audio)))\n    if shift > 0:\n        audio_shifted = np.pad(audio, (shift, 0), mode='constant')[:len(audio)]\n    else:\n        audio_shifted = np.pad(audio, (0, -shift), mode='constant')[-shift:]\n    return audio_shifted\n\ndef gentle_reverb(audio, sr, room_size=0.3, decay=0.2):\n    \"\"\"Add gentle reverb effect\"\"\"\n    try:\n        from scipy.signal import lfilter\n        impulse_length = int(sr * decay)\n        impulse = np.exp(-5 * np.linspace(0, 1, impulse_length)) * np.random.randn(impulse_length)\n        augmented_audio = lfilter(impulse, [1], audio)\n        return np.clip(augmented_audio, -1.0, 1.0)\n    except:\n        return audio  # Return original if reverb fails\n\ndef gentle_time_stretch(audio, factors=(0.95, 1.05)):\n    \"\"\"Very gentle time stretching (5% change instead of 20%)\"\"\"\n    try:\n        factor = np.random.uniform(factors[0], factors[1])\n        return librosa.effects.time_stretch(audio=audio, rate=factor)\n    except:\n        return audio  # Return original if time stretch fails\n\ndef apply_gentle_audio_augmentation(audio, cfg):\n    \"\"\"Apply gentle audio augmentations with controlled probability\"\"\"\n    \n    # Apply gentle time stretching first\n    if random.random() < cfg.time_stretch_prob:\n        audio = gentle_time_stretch(audio, cfg.time_stretch_factors)\n        \n    # Choose augmentation type with lower probability\n    if random.random() < cfg.noise_aug_prob:\n        aug_type = random.choice(['white_noise', 'gaussian_noise', 'background_noise', 'time_shift', 'reverb'])\n        \n        if aug_type == 'white_noise':\n            audio = add_gentle_white_noise(audio, cfg.white_noise_factor)\n        \n        elif aug_type == 'gaussian_noise':\n            audio = add_gentle_gaussian_noise(audio, cfg.gaussian_noise_factor)\n        \n        elif aug_type == 'background_noise':\n            noise_files = glob.glob(os.path.join(cfg.noise_samples_dir, '*.wav'))\n            if noise_files:\n                noise_file = random.choice(noise_files)\n                try:\n                    background_noise, _ = librosa.load(noise_file, sr=cfg.FS)\n                    audio = add_gentle_background_noise(audio, background_noise, cfg.background_noise_factor)\n                except:\n                    audio = add_gentle_white_noise(audio, cfg.white_noise_factor)\n        \n        elif aug_type == 'time_shift':\n            audio = gentle_time_shift(audio, cfg.time_shift_max)\n        \n        elif aug_type == 'reverb':\n            audio = gentle_reverb(audio, cfg.FS, cfg.reverb_room_size, cfg.reverb_decay)\n    \n    return audio","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T17:23:59.354686Z","iopub.execute_input":"2025-06-07T17:23:59.355006Z","iopub.status.idle":"2025-06-07T17:23:59.375205Z","shell.execute_reply.started":"2025-06-07T17:23:59.354978Z","shell.execute_reply":"2025-06-07T17:23:59.373665Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def audio2melspec(audio_data, cfg):\n    \"\"\"Convert audio data to mel spectrogram\"\"\"\n    if np.isnan(audio_data).any():\n        mean_signal = np.nanmean(audio_data)\n        audio_data = np.nan_to_num(audio_data, nan=mean_signal)\n\n    mel_spec = librosa.feature.melspectrogram(\n        y=audio_data,\n        sr=cfg.FS,\n        n_fft=cfg.N_FFT,\n        hop_length=cfg.HOP_LENGTH,\n        n_mels=cfg.N_MELS,\n        fmin=cfg.FMIN,\n        fmax=cfg.FMAX,\n        power=2.0\n    )\n\n    mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max)\n    mel_spec_norm = (mel_spec_db - mel_spec_db.min()) / (mel_spec_db.max() - mel_spec_db.min() + 1e-8)\n    \n    return mel_spec_norm\n\ndef process_audio_file(audio_path, cfg, apply_noise_aug=False, num_segments=3):\n    \"\"\"Process a single audio file with gentle augmentations\"\"\"\n    try:\n        audio_data, _ = librosa.load(audio_path, sr=cfg.FS)\n        \n        target_samples = int(cfg.TARGET_DURATION * cfg.FS)\n        audio_len = len(audio_data)\n        \n        if audio_len < target_samples:\n            n_copy = math.ceil(target_samples / audio_len)\n            if n_copy > 1:\n                audio_data = np.concatenate([audio_data] * n_copy)\n            audio_len = len(audio_data)\n        \n        specs = []\n        for i in range(num_segments):\n            if num_segments == 1:\n                start_idx = max(0, int(audio_len/2 - target_samples/2))\n            else:\n                segment_position = i / (num_segments - 1) if num_segments > 1 else 0.5\n                start_idx = max(0, min(audio_len - target_samples, \n                               int(segment_position * (audio_len - target_samples))))\n            \n            end_idx = min(audio_len, start_idx + target_samples)\n            segment_audio = audio_data[start_idx:end_idx]\n            \n            if len(segment_audio) < target_samples:\n                segment_audio = np.pad(segment_audio, \n                                     (0, target_samples - len(segment_audio)), \n                                     mode='constant')\n            \n            # Apply GENTLE augmentation if requested\n            if apply_noise_aug:\n                segment_audio = apply_gentle_audio_augmentation(segment_audio, cfg)\n            \n            mel_spec = audio2melspec(segment_audio, cfg)\n            \n            if mel_spec.shape != cfg.TARGET_SHAPE:\n                mel_spec = cv2.resize(mel_spec, cfg.TARGET_SHAPE, interpolation=cv2.INTER_LINEAR)\n            \n            specs.append(mel_spec.astype(np.float32))\n        \n        return specs if len(specs) > 1 else specs[0]\n        \n    except Exception as e:\n        print(f\"Error processing {audio_path}: {e}\")\n        return None\n\ndef generate_spectrograms(df, cfg):\n    \"\"\"Generate spectrograms from audio files\"\"\"\n    print(\"Generating mel spectrograms from audio files...\")\n    start_time = time.time()\n\n    all_bird_data = {}\n    errors = []\n\n    for i, row in tqdm(df.iterrows(), total=len(df)):\n        if cfg.debug and i >= 1000:\n            break\n        \n        try:\n            samplename = row['samplename']\n            filepath = row['filepath']\n            \n            mel_spec = process_audio_file(filepath, cfg, num_segments=1)  # Single segment for pre-computation\n            \n            if mel_spec is not None:\n                all_bird_data[samplename] = mel_spec\n            \n        except Exception as e:\n            print(f\"Error processing {row.filepath}: {e}\")\n            errors.append((row.filepath, str(e)))\n\n    end_time = time.time()\n    print(f\"Processing completed in {end_time - start_time:.2f} seconds\")\n    print(f\"Successfully processed {len(all_bird_data)} files out of {len(df)}\")\n    print(f\"Failed to process {len(errors)} files\")\n    \n    return all_bird_data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T17:23:59.376478Z","iopub.execute_input":"2025-06-07T17:23:59.376948Z","iopub.status.idle":"2025-06-07T17:23:59.404401Z","shell.execute_reply.started":"2025-06-07T17:23:59.376905Z","shell.execute_reply":"2025-06-07T17:23:59.402754Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BirdCLEFDatasetFromNPY(Dataset):\n    def __init__(self, df, cfg, spectrograms=None, mode=\"train\"):\n        self.df = df\n        self.cfg = cfg\n        self.mode = mode\n\n        self.spectrograms = spectrograms\n        \n        taxonomy_df = pd.read_csv(self.cfg.taxonomy_csv)\n        self.species_ids = taxonomy_df['primary_label'].tolist()\n        self.num_classes = len(self.species_ids)\n        self.label_to_idx = {label: idx for idx, label in enumerate(self.species_ids)}\n\n        if 'filepath' not in self.df.columns:\n            self.df['filepath'] = self.cfg.train_datadir + '/' + self.df.filename\n        \n        if 'samplename' not in self.df.columns:\n            self.df['samplename'] = self.df.filename.map(lambda x: x.split('/')[0] + '-' + x.split('/')[-1].split('.')[0])\n\n        sample_names = set(self.df['samplename'])\n        if self.spectrograms:\n            found_samples = sum(1 for name in sample_names if name in self.spectrograms)\n            print(f\"Found {found_samples} matching spectrograms for {mode} dataset out of {len(self.df)} samples\")\n        \n        if cfg.debug:\n            self.df = self.df.sample(min(1000, len(self.df)), random_state=cfg.seed).reset_index(drop=True)\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        samplename = row['samplename']\n        spec = None\n        \n        if self.spectrograms and samplename in self.spectrograms:\n            spec = self.spectrograms[samplename]\n            specs = [spec]\n        elif not self.cfg.LOAD_DATA:\n            # Pass flag to apply noise augmentation for training mode\n            num_segments = 3 if self.mode == \"train\" else 1\n            specs = process_audio_file(row['filepath'], self.cfg, \n                                   apply_noise_aug=(self.mode == \"train\"),\n                                   num_segments=num_segments)\n            \n            if specs is None:\n                specs = [np.zeros(self.cfg.TARGET_SHAPE, dtype=np.float32)]\n            elif not isinstance(specs, list):\n                specs = [specs]\n        else:\n            specs = [np.zeros(self.cfg.TARGET_SHAPE, dtype=np.float32)]\n            if self.mode == \"train\":\n                print(f\"Warning: Spectrogram for {samplename} not found and could not be generated\")\n        \n        # Pick a random segment for training, or use the only segment for validation\n        if self.mode == \"train\" and len(specs) > 1:\n            spec = specs[random.randint(0, len(specs)-1)]\n        else:\n            spec = specs[0]\n        \n        spec = torch.tensor(spec, dtype=torch.float32).unsqueeze(0)\n        \n        # Apply gentle spectrogram augmentations\n        if self.mode == \"train\" and random.random() < self.cfg.aug_prob:\n            # Gentle time masking (much smaller masks)\n            if random.random() < 0.5:  # Reduced probability\n                num_masks = random.randint(1, 2)  # Reduced from 1-3\n                for _ in range(num_masks):\n                    width = random.randint(2, self.cfg.spec_aug_time_mask_param)  # Smaller masks\n                    start = random.randint(0, spec.shape[2] - width)\n                    mask_value = random.uniform(0.3, 0.7)  # Don't zero out completely\n                    spec[0, :, start:start+width] *= mask_value\n        \n            # Gentle frequency masking (much smaller masks)\n            if random.random() < 0.5:  # Reduced probability\n                num_masks = random.randint(1, 2)  # Reduced from 1-3\n                for _ in range(num_masks):\n                    height = random.randint(2, self.cfg.spec_aug_freq_mask_param)  # Smaller masks\n                    start = random.randint(0, spec.shape[1] - height)\n                    mask_value = random.uniform(0.3, 0.7)  # Don't zero out completely\n                    spec[0, start:start+height, :] *= mask_value\n        \n            # Gentle brightness/contrast (much more conservative)\n            if random.random() < 0.3:  # Reduced probability\n                gain = random.uniform(0.95, 1.05)  # Much smaller range\n                bias = random.uniform(-0.02, 0.02)  # Much smaller range\n                spec = spec * gain + bias\n                spec = torch.clamp(spec, 0, 1)\n        \n        target = self.encode_label(row['primary_label'])\n        \n        if 'secondary_labels' in row and row['secondary_labels'] not in [[''], None, np.nan]:\n            if isinstance(row['secondary_labels'], str):\n                secondary_labels = eval(row['secondary_labels'])\n            else:\n                secondary_labels = row['secondary_labels']\n            \n            for label in secondary_labels:\n                if label in self.label_to_idx:\n                    target[self.label_to_idx[label]] = 1.0\n        \n        return {\n            'melspec': spec, \n            'target': torch.tensor(target, dtype=torch.float32),\n            'filename': row['filename']\n        }\n    \n    def encode_label(self, label):\n        \"\"\"Encode label to one-hot vector\"\"\"\n        target = np.zeros(self.num_classes)\n        if label in self.label_to_idx:\n            target[self.label_to_idx[label]] = 1.0\n        return target","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T17:23:59.406014Z","iopub.execute_input":"2025-06-07T17:23:59.406544Z","iopub.status.idle":"2025-06-07T17:23:59.439402Z","shell.execute_reply.started":"2025-06-07T17:23:59.406441Z","shell.execute_reply":"2025-06-07T17:23:59.437384Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def collate_fn(batch):\n    \"\"\"Custom collate function to handle different sized spectrograms\"\"\"\n    batch = [item for item in batch if item is not None]\n    if len(batch) == 0:\n        return {}\n        \n    result = {key: [] for key in batch[0].keys()}\n    \n    for item in batch:\n        for key, value in item.items():\n            result[key].append(value)\n    \n    for key in result:\n        if key == 'target' and isinstance(result[key][0], torch.Tensor):\n            result[key] = torch.stack(result[key])\n        elif key == 'melspec' and isinstance(result[key][0], torch.Tensor):\n            shapes = [t.shape for t in result[key]]\n            if len(set(str(s) for s in shapes)) == 1:\n                result[key] = torch.stack(result[key])\n    \n    return result","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T17:23:59.443157Z","iopub.execute_input":"2025-06-07T17:23:59.443698Z","iopub.status.idle":"2025-06-07T17:23:59.471095Z","shell.execute_reply.started":"2025-06-07T17:23:59.443639Z","shell.execute_reply":"2025-06-07T17:23:59.469481Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BirdCLEFModel(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()  # Fixed: removed 'self' parameter\n        self.cfg = cfg\n        \n        taxonomy_df = pd.read_csv(cfg.taxonomy_csv)\n        cfg.num_classes = len(taxonomy_df)\n        \n        self.backbone = timm.create_model(\n            cfg.model_name,\n            pretrained=cfg.pretrained,\n            in_chans=cfg.in_channels,\n            drop_rate=0.2,\n            drop_path_rate=0.2\n        )\n        \n        if 'efficientnet' in cfg.model_name:\n            backbone_out = self.backbone.classifier.in_features\n            self.backbone.classifier = nn.Identity()\n        elif 'resnet' in cfg.model_name:\n            backbone_out = self.backbone.fc.in_features\n            self.backbone.fc = nn.Identity()\n        else:\n            backbone_out = self.backbone.get_classifier().in_features\n            self.backbone.reset_classifier(0, '')\n        \n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.feat_dim = backbone_out\n        self.dropout = nn.Dropout(0.5)\n        self.classifier = nn.Linear(backbone_out, cfg.num_classes)\n        \n        self.mixup_enabled = hasattr(cfg, 'mixup_alpha') and cfg.mixup_alpha > 0\n        if self.mixup_enabled:\n            self.mixup_alpha = cfg.mixup_alpha\n    \n    def forward(self, x, targets=None):\n        \"\"\"Forward pass - THIS METHOD WAS MISSING OR BROKEN\"\"\"\n        \n        if self.training and self.mixup_enabled and targets is not None:\n            mixed_x, targets_a, targets_b, lam = self.mixup_data(x, targets)\n            x = mixed_x\n        else:\n            targets_a, targets_b, lam = None, None, None\n        \n        features = self.backbone(x)\n        \n        if isinstance(features, dict):\n            features = features['features']\n            \n        if len(features.shape) == 4:\n            features = self.pooling(features)\n            features = features.view(features.size(0), -1)\n        \n        # Apply dropout\n        features = self.dropout(features)\n        \n        logits = self.classifier(features)\n        \n        if self.training and self.mixup_enabled and targets is not None:\n            loss = self.mixup_criterion(F.binary_cross_entropy_with_logits, \n                                       logits, targets_a, targets_b, lam)\n            return logits, loss\n            \n        return logits\n    \n    def mixup_data(self, x, targets):\n        \"\"\"Gentler mixup with reduced alpha\"\"\"\n        batch_size = x.size(0)\n        lam = np.random.beta(self.mixup_alpha, self.mixup_alpha)\n        \n        # Ensure lambda is not too extreme (gentle mixing)\n        lam = max(0.2, min(0.8, lam))  # Clamp between 0.2 and 0.8\n        \n        indices = torch.randperm(batch_size).to(x.device)\n        mixed_x = lam * x + (1 - lam) * x[indices]\n        \n        return mixed_x, targets, targets[indices], lam\n    \n    def mixup_criterion(self, criterion, pred, y_a, y_b, lam):\n        \"\"\"Applies mixup to the loss function\"\"\"\n        return lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T17:23:59.472988Z","iopub.execute_input":"2025-06-07T17:23:59.473342Z","iopub.status.idle":"2025-06-07T17:23:59.495570Z","shell.execute_reply.started":"2025-06-07T17:23:59.473312Z","shell.execute_reply":"2025-06-07T17:23:59.494200Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_optimizer(model, cfg):\n  \n    if cfg.optimizer == 'Adam':\n        optimizer = optim.Adam(\n            model.parameters(),\n            lr=cfg.lr,\n            weight_decay=cfg.weight_decay\n        )\n    elif cfg.optimizer == 'AdamW':\n        optimizer = optim.AdamW(\n            model.parameters(),\n            lr=cfg.lr,\n            weight_decay=cfg.weight_decay\n        )\n    elif cfg.optimizer == 'SGD':\n        optimizer = optim.SGD(\n            model.parameters(),\n            lr=cfg.lr,\n            momentum=0.9,\n            weight_decay=cfg.weight_decay\n        )\n    else:\n        raise NotImplementedError(f\"Optimizer {cfg.optimizer} not implemented\")\n        \n    return optimizer\n\ndef get_scheduler(optimizer, cfg):\n   \n    if cfg.scheduler == 'CosineAnnealingLR':\n        scheduler = lr_scheduler.CosineAnnealingLR(\n            optimizer,\n            T_max=cfg.T_max,\n            eta_min=cfg.min_lr\n        )\n    elif cfg.scheduler == 'ReduceLROnPlateau':\n        scheduler = lr_scheduler.ReduceLROnPlateau(\n            optimizer,\n            mode='min',\n            factor=0.5,\n            patience=2,\n            min_lr=cfg.min_lr,\n            verbose=True\n        )\n    elif cfg.scheduler == 'StepLR':\n        scheduler = lr_scheduler.StepLR(\n            optimizer,\n            step_size=cfg.epochs // 3,\n            gamma=0.5\n        )\n    elif cfg.scheduler == 'OneCycleLR':\n        scheduler = lr_scheduler.OneCycleLR(\n            optimizer,\n            max_lr=cfg.lr,\n            steps_per_epoch=1000,  # This will be updated in run_training\n            epochs=cfg.epochs,\n            pct_start=0.1\n        )\n    else:\n        scheduler = None\n        \n    return scheduler\n\ndef get_criterion(cfg):\n    if cfg.criterion == 'FocalLoss':\n        return FocalLoss(gamma=2.0)\n    elif cfg.criterion == 'BCEWithLogitsLoss':\n        return nn.BCEWithLogitsLoss()\n    else:\n        raise NotImplementedError(f\"Criterion {cfg.criterion} not implemented\")\n        \n    return criterion","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T17:23:59.496969Z","iopub.execute_input":"2025-06-07T17:23:59.497361Z","iopub.status.idle":"2025-06-07T17:23:59.522066Z","shell.execute_reply.started":"2025-06-07T17:23:59.497324Z","shell.execute_reply":"2025-06-07T17:23:59.520736Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_one_epoch(model, loader, optimizer, criterion, device, scheduler=None):\n    \n    model.train()\n    losses = []\n    all_targets = []\n    all_outputs = []\n    \n    pbar = tqdm(enumerate(loader), total=len(loader), desc=\"Training\")\n    \n    for step, batch in pbar:\n    \n        if isinstance(batch['melspec'], list):\n            batch_outputs = []\n            batch_losses = []\n            \n            for i in range(len(batch['melspec'])):\n                inputs = batch['melspec'][i].unsqueeze(0).to(device)\n                target = batch['target'][i].unsqueeze(0).to(device)\n                \n                optimizer.zero_grad()\n                output = model(inputs)\n                loss = criterion(output, target)\n                loss.backward()\n                \n                batch_outputs.append(output.detach().cpu())\n                batch_losses.append(loss.item())\n            \n            optimizer.step()\n            outputs = torch.cat(batch_outputs, dim=0).numpy()\n            loss = np.mean(batch_losses)\n            targets = batch['target'].numpy()\n            \n        else:\n            inputs = batch['melspec'].to(device)\n            targets = batch['target'].to(device)\n            \n            optimizer.zero_grad()\n            outputs = model(inputs)\n            \n            if isinstance(outputs, tuple):\n                outputs, loss = outputs  \n            else:\n                loss = criterion(outputs, targets)\n                \n            loss.backward()\n            optimizer.step()\n            \n            outputs = outputs.detach().cpu().numpy()\n            targets = targets.detach().cpu().numpy()\n        \n        if scheduler is not None and isinstance(scheduler, lr_scheduler.OneCycleLR):\n            scheduler.step()\n            \n        all_outputs.append(outputs)\n        all_targets.append(targets)\n        losses.append(loss if isinstance(loss, float) else loss.item())\n        \n        pbar.set_postfix({\n            'train_loss': np.mean(losses[-10:]) if losses else 0,\n            'lr': optimizer.param_groups[0]['lr']\n        })\n    \n    all_outputs = np.concatenate(all_outputs)\n    all_targets = np.concatenate(all_targets)\n    auc = calculate_auc(all_targets, all_outputs)\n    avg_loss = np.mean(losses)\n    \n    return avg_loss, auc\n\ndef validate(model, loader, criterion, device):\n   \n    model.eval()\n    losses = []\n    all_targets = []\n    all_outputs = []\n    \n    with torch.no_grad():\n        for batch in tqdm(loader, desc=\"Validation\"):\n            if isinstance(batch['melspec'], list):\n                batch_outputs = []\n                batch_losses = []\n                \n                for i in range(len(batch['melspec'])):\n                    inputs = batch['melspec'][i].unsqueeze(0).to(device)\n                    target = batch['target'][i].unsqueeze(0).to(device)\n                    \n                    output = model(inputs)\n                    loss = criterion(output, target)\n                    \n                    batch_outputs.append(output.detach().cpu())\n                    batch_losses.append(loss.item())\n                \n                outputs = torch.cat(batch_outputs, dim=0).numpy()\n                loss = np.mean(batch_losses)\n                targets = batch['target'].numpy()\n                \n            else:\n                inputs = batch['melspec'].to(device)\n                targets = batch['target'].to(device)\n                \n                outputs = model(inputs)\n                loss = criterion(outputs, targets)\n                \n                outputs = outputs.detach().cpu().numpy()\n                targets = targets.detach().cpu().numpy()\n            \n            all_outputs.append(outputs)\n            all_targets.append(targets)\n            losses.append(loss if isinstance(loss, float) else loss.item())\n    \n    all_outputs = np.concatenate(all_outputs)\n    all_targets = np.concatenate(all_targets)\n    \n    auc = calculate_auc(all_targets, all_outputs)\n    avg_loss = np.mean(losses)\n    \n    return avg_loss, auc\n\ndef calculate_auc(targets, outputs):\n  \n    num_classes = targets.shape[1]\n    aucs = []\n    \n    probs = 1 / (1 + np.exp(-outputs))\n    \n    for i in range(num_classes):\n        \n        if np.sum(targets[:, i]) > 0:\n            class_auc = roc_auc_score(targets[:, i], probs[:, i])\n            aucs.append(class_auc)\n    \n    return np.mean(aucs) if aucs else 0.0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T17:23:59.523470Z","iopub.execute_input":"2025-06-07T17:23:59.523977Z","iopub.status.idle":"2025-06-07T17:23:59.554304Z","shell.execute_reply.started":"2025-06-07T17:23:59.523940Z","shell.execute_reply":"2025-06-07T17:23:59.552910Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def run_training(df, cfg):\n    \"\"\"Training function that can either use pre-computed spectrograms or generate them on-the-fly\"\"\"\n\n    taxonomy_df = pd.read_csv(cfg.taxonomy_csv)\n    species_ids = taxonomy_df['primary_label'].tolist()\n    cfg.num_classes = len(species_ids)\n    \n    if cfg.debug:\n        cfg.update_debug_settings()\n\n    spectrograms = None\n    if cfg.LOAD_DATA:\n        print(\"Loading pre-computed mel spectrograms from NPY file...\")\n        try:\n            spectrograms = np.load(cfg.spectrogram_npy, allow_pickle=True).item()\n            print(f\"Loaded {len(spectrograms)} pre-computed mel spectrograms\")\n        except Exception as e:\n            print(f\"Error loading pre-computed spectrograms: {e}\")\n            print(\"Will generate spectrograms on-the-fly instead.\")\n            cfg.LOAD_DATA = False\n    \n    if not cfg.LOAD_DATA:\n        print(\"Will generate spectrograms on-the-fly during training.\")\n        if 'filepath' not in df.columns:\n            df['filepath'] = cfg.train_datadir + '/' + df.filename\n        if 'samplename' not in df.columns:\n            df['samplename'] = df.filename.map(lambda x: x.split('/')[0] + '-' + x.split('/')[-1].split('.')[0])\n        \n    skf = StratifiedKFold(n_splits=cfg.n_fold, shuffle=True, random_state=cfg.seed)\n    \n    best_scores = []\n    \n    for fold, (train_idx, val_idx) in enumerate(skf.split(df, df['primary_label'])):\n        if fold not in cfg.selected_folds:\n            continue\n            \n        print(f'\\n{\"=\"*30} Fold {fold} {\"=\"*30}')\n        \n        train_df = df.iloc[train_idx].reset_index(drop=True)\n        val_df = df.iloc[val_idx].reset_index(drop=True)\n        \n        print(f'Training set: {len(train_df)} samples')\n        print(f'Validation set: {len(val_df)} samples')\n        \n        train_dataset = BirdCLEFDatasetFromNPY(train_df, cfg, spectrograms=spectrograms, mode='train')\n        val_dataset = BirdCLEFDatasetFromNPY(val_df, cfg, spectrograms=spectrograms, mode='valid')\n        \n        train_loader = DataLoader(\n            train_dataset, \n            batch_size=cfg.batch_size, \n            shuffle=True, \n            num_workers=cfg.num_workers,\n            pin_memory=True,\n            collate_fn=collate_fn,\n            drop_last=True\n        )\n        \n        val_loader = DataLoader(\n            val_dataset, \n            batch_size=cfg.batch_size, \n            shuffle=False, \n            num_workers=cfg.num_workers,\n            pin_memory=True,\n            collate_fn=collate_fn\n        )\n        \n        model = BirdCLEFModel(cfg).to(cfg.device)\n        optimizer = get_optimizer(model, cfg)\n        criterion = get_criterion(cfg)\n        \n        if cfg.scheduler == 'OneCycleLR':\n            scheduler = lr_scheduler.OneCycleLR(\n                optimizer,\n                max_lr=cfg.lr,\n                steps_per_epoch=len(train_loader),\n                epochs=cfg.epochs,\n                pct_start=0.1\n            )\n        else:\n            scheduler = get_scheduler(optimizer, cfg)\n        \n        best_auc = 0\n        best_epoch = 0\n        \n        for epoch in range(cfg.epochs):\n            print(f\"\\nEpoch {epoch+1}/{cfg.epochs}\")\n            \n            train_loss, train_auc = train_one_epoch(\n                model, \n                train_loader, \n                optimizer, \n                criterion, \n                cfg.device,\n                scheduler if isinstance(scheduler, lr_scheduler.OneCycleLR) else None\n            )\n            \n            val_loss, val_auc = validate(model, val_loader, criterion, cfg.device)\n\n            if scheduler is not None and not isinstance(scheduler, lr_scheduler.OneCycleLR):\n                if isinstance(scheduler, lr_scheduler.ReduceLROnPlateau):\n                    scheduler.step(val_loss)\n                else:\n                    scheduler.step()\n\n            print(f\"Train Loss: {train_loss:.4f}, Train AUC: {train_auc:.4f}\")\n            print(f\"Val Loss: {val_loss:.4f}, Val AUC: {val_auc:.4f}\")\n            \n            if val_auc > best_auc:\n                best_auc = val_auc\n                best_epoch = epoch + 1\n                print(f\"New best AUC: {best_auc:.4f} at epoch {best_epoch}\")\n\n                # Save model with more metadata\n                torch.save({\n                    'model_state_dict': model.state_dict(),\n                    'optimizer_state_dict': optimizer.state_dict(),\n                    'scheduler_state_dict': scheduler.state_dict() if scheduler else None,\n                    'epoch': epoch,\n                    'val_auc': val_auc,\n                    'train_auc': train_auc,\n                    'cfg': cfg,\n                    'model_name': cfg.model_name,\n                    'fold': fold\n                }, f\"{cfg.model_name}_fold{fold}.pth\")\n        \n        best_scores.append(best_auc)\n        print(f\"\\nBest AUC for fold {fold}: {best_auc:.4f} at epoch {best_epoch}\")\n        \n        # Clear memory\n        del model, optimizer, scheduler, train_loader, val_loader\n        torch.cuda.empty_cache()\n        gc.collect()\n    \n    print(\"\\n\" + \"=\"*60)\n    print(\"Cross-Validation Results:\")\n    for i, fold in enumerate(cfg.selected_folds[:len(best_scores)]):\n        print(f\"Fold {fold}: {best_scores[i]:.4f}\")\n    print(f\"Mean AUC: {np.mean(best_scores):.4f}\")\n    print(\"=\"*60)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T17:23:59.555462Z","iopub.execute_input":"2025-06-07T17:23:59.555790Z","iopub.status.idle":"2025-06-07T17:23:59.583153Z","shell.execute_reply.started":"2025-06-07T17:23:59.555761Z","shell.execute_reply":"2025-06-07T17:23:59.581713Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict_with_tta(model, mel_spec, device, tta_count=5):\n    \"\"\"Predict with test-time augmentation\"\"\"\n    all_preds = []\n    \n    # Original prediction\n    mel_spec_tensor = torch.tensor(mel_spec, dtype=torch.float32).unsqueeze(0).unsqueeze(0)\n    mel_spec_tensor = mel_spec_tensor.to(device)\n    \n    with torch.no_grad():\n        output = model(mel_spec_tensor)\n        prob = torch.sigmoid(output).cpu().numpy().squeeze()\n        all_preds.append(prob)\n    \n    # TTA: Horizontal flip (time reversal)\n    mel_spec_h = np.flip(mel_spec, axis=1)\n    mel_spec_tensor = torch.tensor(mel_spec_h, dtype=torch.float32).unsqueeze(0).unsqueeze(0)\n    mel_spec_tensor = mel_spec_tensor.to(device)\n    \n    with torch.no_grad():\n        output = model(mel_spec_tensor)\n        prob = torch.sigmoid(output).cpu().numpy().squeeze()\n        all_preds.append(prob)\n    \n    # TTA: Vertical flip (frequency inversion)\n    mel_spec_v = np.flip(mel_spec, axis=0)\n    mel_spec_tensor = torch.tensor(mel_spec_v, dtype=torch.float32).unsqueeze(0).unsqueeze(0)\n    mel_spec_tensor = mel_spec_tensor.to(device)\n    \n    with torch.no_grad():\n        output = model(mel_spec_tensor)\n        prob = torch.sigmoid(output).cpu().numpy().squeeze()\n        all_preds.append(prob)\n        \n    # Average predictions\n    final_pred = np.mean(all_preds, axis=0)\n    return final_pred\n\ndef train_multiple_models():\n    \"\"\"Train multiple models for ensemble\"\"\"\n    models_to_train = cfg.models_to_train\n    \n    for model_config in models_to_train:\n        print(f\"Training {model_config['name']}...\")\n        \n        # Update config\n        cfg.model_name = model_config['name']\n        cfg.selected_folds = model_config['selected_folds']\n        \n        # Train model\n        run_training(train_df, cfg)\n        \n        # Clear memory\n        torch.cuda.empty_cache()\n        gc.collect()\n        \n    print(\"All models trained!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T17:23:59.584463Z","iopub.execute_input":"2025-06-07T17:23:59.584948Z","iopub.status.idle":"2025-06-07T17:23:59.609060Z","shell.execute_reply.started":"2025-06-07T17:23:59.584897Z","shell.execute_reply":"2025-06-07T17:23:59.607715Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    import time\n    \n    print(\"\\nLoading training data...\")\n    train_df = pd.read_csv(cfg.train_csv)\n    taxonomy_df = pd.read_csv(cfg.taxonomy_csv)\n\n    print(\"\\nStarting training...\")\n    print(f\"LOAD_DATA is set to {cfg.LOAD_DATA}\")\n    if cfg.LOAD_DATA:\n        print(\"Using pre-computed mel spectrograms from NPY file\")\n    else:\n        print(\"Will generate spectrograms on-the-fly during training\")\n    \n    if cfg.train_multiple_models:\n        train_multiple_models()\n    else:\n        run_training(train_df, cfg)\n    \n    print(\"\\nTraining complete!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T17:23:59.610446Z","iopub.execute_input":"2025-06-07T17:23:59.610838Z","iopub.status.idle":"2025-06-07T17:39:56.204556Z","shell.execute_reply.started":"2025-06-07T17:23:59.610798Z","shell.execute_reply":"2025-06-07T17:39:56.202747Z"}},"outputs":[],"execution_count":null}]}