{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"},{"sourceId":11053663,"sourceType":"datasetVersion","datasetId":6886569}],"dockerImageVersionId":31012,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport logging\nimport random\nimport gc\nimport time\nimport cv2\nimport math\nimport warnings\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nimport librosa\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.auto import tqdm\n\nimport timm\n\nwarnings.filterwarnings(\"ignore\")\nlogging.basicConfig(level=logging.ERROR)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-19T11:30:49.510151Z","iopub.execute_input":"2025-04-19T11:30:49.510892Z","iopub.status.idle":"2025-04-19T11:30:49.517038Z","shell.execute_reply.started":"2025-04-19T11:30:49.510861Z","shell.execute_reply":"2025-04-19T11:30:49.516295Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":16},{"cell_type":"code","source":"#自定义类\nimport torch\nimport torch.nn as nn\n\nclass CBAM(nn.Module):\n def __init__(self, channel, reduction=16, spatial_kernel=7):\n super(CBAM, self).__init__()\n \n # channel attention 压缩H,W为1\n self.max_pool = nn.AdaptiveMaxPool2d(1)\n self.avg_pool = nn.AdaptiveAvgPool2d(1)\n \n # shared MLP\n self.mlp = nn.Sequential(\n # Conv2d比Linear方便操作\n # nn.Linear(channel, channel // reduction, bias=False)\n nn.Conv2d(channel, channel // reduction, 1, bias=False),\n # inplace=True直接替换，节省内存\n nn.ReLU(inplace=True),\n # nn.Linear(channel // reduction, channel,bias=False)\n nn.Conv2d(channel // reduction, channel, 1, bias=False)\n )\n \n # spatial attention\n self.conv = nn.Conv2d(2, 1, kernel_size=spatial_kernel,\n padding=spatial_kernel // 2, bias=False)\n self.sigmoid = nn.Sigmoid()\n \n def forward(self, x):\n max_out = self.mlp(self.max_pool(x))\n avg_out = self.mlp(self.avg_pool(x))\n channel_out = self.sigmoid(max_out + avg_out)\n x = channel_out * x\n \n max_out, _ = torch.max(x, dim=1, keepdim=True)\n avg_out = torch.mean(x, dim=1, keepdim=True)\n spatial_out = self.sigmoid(self.conv(torch.cat([max_out, avg_out], dim=1)))\n x = spatial_out * x\n return x\n\n\n\nclass GeM(nn.Module):\n def __init__(self, p_init=3.0, eps=1e-6):\n super().__init__()\n self.p = nn.Parameter(torch.tensor(p_init, dtype=torch.float32))\n self.eps = eps\n\n def forward(self, x):\n p = self.p.clamp(min=1e-6)\n x = x.clamp(min=self.eps)\n return x.pow(p).mean(dim=(-2, -1)).pow(1./p)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-19T11:30:49.518223Z","iopub.execute_input":"2025-04-19T11:30:49.518415Z","iopub.status.idle":"2025-04-19T11:30:49.532835Z","shell.execute_reply.started":"2025-04-19T11:30:49.5184Z","shell.execute_reply":"2025-04-19T11:30:49.532118Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":17},{"cell_type":"code","source":"class CFG:\n seed = 42\n apex = False\n print_freq = 100\n num_workers = 2\n\n OUTPUT_DIR = '/kaggle/working/'\n\n train_datadir = '/kaggle/input/birdclef-2025/train_audio'\n train_csv = '/kaggle/input/birdclef-2025/train.csv'\n test_soundscapes = '/kaggle/input/birdclef-2025/test_soundscapes'\n submission_csv = '/kaggle/input/birdclef-2025/sample_submission.csv'\n taxonomy_csv = '/kaggle/input/birdclef-2025/taxonomy.csv'\n\n spectrogram_npy = '/kaggle/input/birdclef25-mel-spectrograms/birdclef2025_melspec_5sec_256_256.npy'\n\n model_name = 'efficientnet_b0'\n pretrained = True\n in_channels = 1\n\n LOAD_DATA = True\n FS = 32000\n TARGET_DURATION = 5.0\n TARGET_SHAPE = (256, 256)\n\n N_FFT = 1024\n HOP_LENGTH = 512\n N_MELS = 128\n FMIN = 50\n FMAX = 14000\n\n device = 'cuda' if torch.cuda.is_available() else 'cpu'\n\n criterion = 'BCEWithLogitsLoss'\n\n n_fold = 2\n selected_folds = [0, 1, 2, 3, 4]\n\n optimizer = 'AdamW'\n lr = 5e-4\n weight_decay = 1e-5\n\n scheduler = 'CosineAnnealingLR'\n min_lr = 1e-6\n\n\n # 以下为需要调整的参数\n #debug模式为 1Fold 2epochs 小批量训练\n debug = False\n\n # 模型参数，其余参数为自学习\n epochs = 10\n T_max = epochs #无需调整\n batch_size = 32\n\n # mixup启用\n mixup_alpha = 0.5 # cfg.mixup_alpha = 0.5 启用mixup, =-1 关闭\n\n # 频谱增强启用\n time_masking = True\n frequency_masking = True\n\n def update_debug_settings(self):\n if self.debug:\n self.epochs = 5\n self.selected_folds = [0]\n\n\ncfg = CFG()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-19T11:30:49.533528Z","iopub.execute_input":"2025-04-19T11:30:49.533727Z","iopub.status.idle":"2025-04-19T11:30:49.549183Z","shell.execute_reply.started":"2025-04-19T11:30:49.533713Z","shell.execute_reply":"2025-04-19T11:30:49.548406Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":18},{"cell_type":"code","source":"# 需要调整的参数\n#debug模式为 1Fold 2epochs 小批量训练\ncfg.debug = False\n\n# 模型参数，其余参数为自学习\ncfg.epochs = 10#图\ncfg.batch_size = 32#表\n\n#消融表\n# mixup启用\ncfg.mixup_alpha = 0.5 # cfg.mixup_alpha = 0.5 启用mixup, =-1 关闭\n\n# 频谱增强启用\ncfg.time_masking = False\ncfg.frequency_masking = True","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-19T11:30:49.550502Z","iopub.execute_input":"2025-04-19T11:30:49.550686Z","iopub.status.idle":"2025-04-19T11:30:49.566025Z","shell.execute_reply.started":"2025-04-19T11:30:49.550672Z","shell.execute_reply":"2025-04-19T11:30:49.565438Z"}},"outputs":[],"execution_count":19},{"cell_type":"code","source":"#main\n\n\ndef set_seed(seed=42):\n \"\"\"\n Set seed for reproducibility\n \"\"\"\n random.seed(seed)\n os.environ[\"PYTHONHASHSEED\"] = str(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.cuda.manual_seed_all(seed)\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = False\n\n\nset_seed(cfg.seed)\n\n\ndef audio2melspec(audio_data, cfg):\n \"\"\"Convert audio data to mel spectrogram\"\"\"\n if np.isnan(audio_data).any():\n mean_signal = np.nanmean(audio_data)\n audio_data = np.nan_to_num(audio_data, nan=mean_signal)\n\n mel_spec = librosa.feature.melspectrogram(\n y=audio_data,\n sr=cfg.FS,\n n_fft=cfg.N_FFT,\n hop_length=cfg.HOP_LENGTH,\n n_mels=cfg.N_MELS,\n fmin=cfg.FMIN,\n fmax=cfg.FMAX,\n power=2.0\n )\n\n mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max)\n mel_spec_norm = (mel_spec_db - mel_spec_db.min()) / (mel_spec_db.max() - mel_spec_db.min() + 1e-8)\n\n return mel_spec_norm\n\n\ndef process_audio_file(audio_path, cfg):\n \"\"\"Process a single audio file to get the mel spectrogram\"\"\"\n try:\n audio_data, _ = librosa.load(audio_path, sr=cfg.FS)\n\n target_samples = int(cfg.TARGET_DURATION * cfg.FS)\n\n if len(audio_data) < target_samples:\n n_copy = math.ceil(target_samples / len(audio_data))\n if n_copy > 1:\n audio_data = np.concatenate([audio_data] * n_copy)\n\n # Extract center 5 seconds\n start_idx = max(0, int(len(audio_data) / 2 - target_samples / 2))\n end_idx = min(len(audio_data), start_idx + target_samples)\n center_audio = audio_data[start_idx:end_idx]\n\n if len(center_audio) < target_samples:\n center_audio = np.pad(center_audio,\n (0, target_samples - len(center_audio)),\n mode='constant')\n\n mel_spec = audio2melspec(center_audio, cfg)\n\n if mel_spec.shape != cfg.TARGET_SHAPE:\n mel_spec = cv2.resize(mel_spec, cfg.TARGET_SHAPE, interpolation=cv2.INTER_LINEAR)\n\n return mel_spec.astype(np.float32)\n\n except Exception as e:\n print(f\"Error processing {audio_path}: {e}\")\n return None\n\n\ndef generate_spectrograms(df, cfg):\n \"\"\"Generate spectrograms from audio files\"\"\"\n print(\"Generating mel spectrograms from audio files...\")\n start_time = time.time()\n\n all_bird_data = {}\n errors = []\n\n for i, row in tqdm(df.iterrows(), total=len(df)):\n if cfg.debug and i >= 1000:\n break\n\n try:\n samplename = row['samplename']\n filepath = row['filepath']\n\n mel_spec = process_audio_file(filepath, cfg)\n\n if mel_spec is not None:\n all_bird_data[samplename] = mel_spec\n\n except Exception as e:\n print(f\"Error processing {row.filepath}: {e}\")\n errors.append((row.filepath, str(e)))\n\n end_time = time.time()\n print(f\"Processing completed in {end_time - start_time:.2f} seconds\")\n print(f\"Successfully processed {len(all_bird_data)} files out of {len(df)}\")\n print(f\"Failed to process {len(errors)} files\")\n\n return all_bird_data\n\n\nclass BirdCLEFDatasetFromNPY(Dataset):\n def __init__(self, df, cfg, spectrograms=None, mode=\"train\"):\n self.df = df\n self.cfg = cfg\n self.mode = mode\n\n self.spectrograms = spectrograms\n\n taxonomy_df = pd.read_csv(self.cfg.taxonomy_csv)\n self.species_ids = taxonomy_df['primary_label'].tolist()\n self.num_classes = len(self.species_ids)\n self.label_to_idx = {label: idx for idx, label in enumerate(self.species_ids)}\n\n if 'filepath' not in self.df.columns:\n self.df['filepath'] = self.cfg.train_datadir + '/' + self.df.filename\n\n if 'samplename' not in self.df.columns:\n self.df['samplename'] = self.df.filename.map(\n lambda x: x.split('/')[0] + '-' + x.split('/')[-1].split('.')[0])\n\n sample_names = set(self.df['samplename'])\n if self.spectrograms:\n found_samples = sum(1 for name in sample_names if name in self.spectrograms)\n print(f\"Found {found_samples} matching spectrograms for {mode} dataset out of {len(self.df)} samples\")\n\n if cfg.debug:\n self.df = self.df.sample(min(1000, len(self.df)), random_state=cfg.seed).reset_index(drop=True)\n\n def __len__(self):\n return len(self.df)\n\n def __getitem__(self, idx):\n row = self.df.iloc[idx]\n samplename = row['samplename']\n spec = None\n\n if self.spectrograms and samplename in self.spectrograms:\n spec = self.spectrograms[samplename]\n elif not self.cfg.LOAD_DATA:\n spec = process_audio_file(row['filepath'], self.cfg)\n\n if spec is None:\n spec = np.zeros(self.cfg.TARGET_SHAPE, dtype=np.float32)\n if self.mode == \"train\": # Only print warning during training\n print(f\"Warning: Spectrogram for {samplename} not found and could not be generated\")\n\n spec = torch.tensor(spec, dtype=torch.float32).unsqueeze(0) # Add channel dimension\n\n if self.mode == \"train\":\n spec = self.apply_spec_augmentations(spec)\n\n target = self.encode_label(row['primary_label'])\n\n if 'secondary_labels' in row and row['secondary_labels'] not in [[''], None, np.nan]:\n if isinstance(row['secondary_labels'], str):\n secondary_labels = eval(row['secondary_labels'])\n else:\n secondary_labels = row['secondary_labels']\n\n for label in secondary_labels:\n if label in self.label_to_idx:\n target[self.label_to_idx[label]] = 1.0\n\n return {\n 'melspec': spec,\n 'target': torch.tensor(target, dtype=torch.float32),\n 'filename': row['filename']\n }\n\n def apply_spec_augmentations(self, spec):\n \"\"\"Apply augmentations to spectrogram\"\"\"\n\n # Time masking (horizontal stripes)\n if random.random() < 0.3 and cfg.time_masking:\n num_masks = random.randint(1, 3)\n for _ in range(num_masks):\n width = random.randint(5, 20)\n start = random.randint(0, spec.shape[2] - width)\n spec[0, :, start:start + width] = 0\n\n # Frequency masking (vertical stripes)\n if random.random() < 0.3 and cfg.frequency_masking:\n num_masks = random.randint(1, 3)\n for _ in range(num_masks):\n height = random.randint(5, 20)\n start = random.randint(0, spec.shape[1] - height)\n spec[0, start:start + height, :] = 0\n\n # Random brightness/contrast\n '''if random.random() < 0.3:\n gain = random.uniform(0.8, 1.2)\n bias = random.uniform(-0.1, 0.1)\n spec = spec * gain + bias\n spec = torch.clamp(spec, 0, 1)'''\n\n return spec\n\n def apply_audio_augmentations(self, audio):\n \"\"\"\n 对音频进行数据增强\n 参数:\n audio (np.ndarray): 音频波形数据\n cfg: 配置对象\n\n 返回:\n np.ndarray: 增强后的音频波形\n \"\"\"\n # 复制音频以防止修改原始数据\n augmented_audio = audio.copy()\n\n # 1. 时间伸缩 (Time stretching)\n if np.random.rand() < 0.5:\n stretch_rate = np.random.uniform(0.8, 1.2)\n augmented_audio = librosa.effects.time_stretch(augmented_audio, rate=stretch_rate)\n\n # 确保音频长度一致\n if len(augmented_audio) > cfg.DURATION * cfg.FS:\n augmented_audio = augmented_audio[:cfg.DURATION * cfg.FS]\n elif len(augmented_audio) < cfg.DURATION * cfg.FS:\n # 填充不足的部分\n pad_len = cfg.DURATION * cfg.FS - len(augmented_audio)\n augmented_audio = np.pad(augmented_audio, (0, pad_len), 'constant')\n\n # 2. 音高偏移 (Pitch shifting)\n if np.random.rand() < 0.5:\n n_steps = np.random.randint(-3, 4)\n augmented_audio = librosa.effects.pitch_shift(augmented_audio, sr=cfg.FS, n_steps=n_steps)\n\n # 3. 添加背景噪声\n if np.random.rand() < 0.3:\n noise_factor = np.random.uniform(0.001, 0.02)\n noise = np.random.randn(len(augmented_audio))\n augmented_audio = augmented_audio + noise_factor * noise\n\n # 4. 音量变化\n if np.random.rand() < 0.5:\n gain = np.random.uniform(0.8, 1.2)\n augmented_audio = augmented_audio * gain\n\n # 剪裁幅度，确保在[-1, 1]范围内\n augmented_audio = np.clip(augmented_audio, -1, 1)\n\n return augmented_audio\n\n def encode_label(self, label):\n \"\"\"Encode label to one-hot vector\"\"\"\n target = np.zeros(self.num_classes)\n if label in self.label_to_idx:\n target[self.label_to_idx[label]] = 1.0\n return target\n\n\ndef collate_fn(batch):\n \"\"\"Custom collate function to handle different sized spectrograms\"\"\"\n batch = [item for item in batch if item is not None]\n if len(batch) == 0:\n return {}\n\n result = {key: [] for key in batch[0].keys()}\n\n for item in batch:\n for key, value in item.items():\n result[key].append(value)\n\n for key in result:\n if key == 'target' and isinstance(result[key][0], torch.Tensor):\n result[key] = torch.stack(result[key])\n elif key == 'melspec' and isinstance(result[key][0], torch.Tensor):\n shapes = [t.shape for t in result[key]]\n if len(set(str(s) for s in shapes)) == 1:\n result[key] = torch.stack(result[key])\n\n return result\n\n\nclass BirdCLEFModel(nn.Module):\n def __init__(self, cfg):\n super().__init__()\n self.cfg = cfg\n\n taxonomy_df = pd.read_csv(cfg.taxonomy_csv)\n cfg.num_classes = len(taxonomy_df)\n\n self.backbone = timm.create_model(\n cfg.model_name,\n pretrained=cfg.pretrained,\n in_chans=cfg.in_channels,\n drop_rate=0.2,\n drop_path_rate=0.2\n )\n\n if 'efficientnet' in cfg.model_name:\n backbone_out = self.backbone.classifier.in_features\n self.backbone.classifier = nn.Identity()\n elif 'resnet' in cfg.model_name:\n backbone_out = self.backbone.fc.in_features\n self.backbone.fc = nn.Identity()\n else:\n backbone_out = self.backbone.get_classifier().in_features\n self.backbone.reset_classifier(0, '')\n\n self.cbam = CBAM(backbone_out)\n self.pool = GeM()\n # self.pooling = nn.AdaptiveAvgPool2d(1)\n\n self.feat_dim = backbone_out\n\n # self.classifier = nn.Linear(backbone_out, cfg.num_classes)\n # 分类头\n self.classifier = nn.Sequential(\n nn.Linear(backbone_out, 512),\n nn.BatchNorm1d(512),\n nn.SiLU(),\n nn.Dropout(0.3),\n nn.Linear(512, cfg.num_classes)\n )\n\n self.mixup_enabled = hasattr(cfg, 'mixup_alpha') and cfg.mixup_alpha > 0\n if self.mixup_enabled:\n self.mixup_alpha = cfg.mixup_alpha\n\n def forward(self, x, targets):\n\n if self.training and self.mixup_enabled and targets is not None:\n mixed_x, targets_a, targets_b, lam = self.mixup_data(x, targets)\n x = mixed_x\n else:\n targets_a, targets_b, lam = None, None, None\n\n features = self.backbone(x)\n\n if isinstance(features, dict):\n features = features['features']\n\n if len(features.shape) == 4:\n features = self.cbam(features)\n features = self.pool(features)\n features = features.view(features.size(0), -1)\n\n logits = self.classifier(features)\n\n if self.training and self.mixup_enabled and targets is not None:\n loss = self.mixup_criterion(F.binary_cross_entropy_with_logits,\n logits, targets_a, targets_b, lam)\n return logits, loss\n\n return logits\n\n def mixup_data(self, x, targets):\n \"\"\"Applies mixup to the data batch\"\"\"\n batch_size = x.size(0)\n\n lam = np.random.beta(self.mixup_alpha, self.mixup_alpha)\n\n indices = torch.randperm(batch_size).to(x.device)\n\n mixed_x = lam * x + (1 - lam) * x[indices]\n\n return mixed_x, targets, targets[indices], lam\n\n def mixup_criterion(self, criterion, pred, y_a, y_b, lam):\n \"\"\"Applies mixup to the loss function\"\"\"\n return lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b)\n\n\ndef get_optimizer(model, cfg):\n if cfg.optimizer == 'Adam':\n optimizer = optim.Adam(\n model.parameters(),\n lr=cfg.lr,\n weight_decay=cfg.weight_decay\n )\n elif cfg.optimizer == 'AdamW':\n optimizer = optim.AdamW(\n model.parameters(),\n lr=cfg.lr,\n weight_decay=cfg.weight_decay\n )\n elif cfg.optimizer == 'SGD':\n optimizer = optim.SGD(\n model.parameters(),\n lr=cfg.lr,\n momentum=0.9,\n weight_decay=cfg.weight_decay\n )\n else:\n raise NotImplementedError(f\"Optimizer {cfg.optimizer} not implemented\")\n\n return optimizer\n\n\ndef get_scheduler(optimizer, cfg):\n if cfg.scheduler == 'CosineAnnealingLR':\n scheduler = lr_scheduler.CosineAnnealingLR(\n optimizer,\n T_max=cfg.T_max,\n eta_min=cfg.min_lr\n )\n elif cfg.scheduler == 'ReduceLROnPlateau':\n scheduler = lr_scheduler.ReduceLROnPlateau(\n optimizer,\n mode='min',\n factor=0.5,\n patience=2,\n min_lr=cfg.min_lr,\n verbose=True\n )\n elif cfg.scheduler == 'StepLR':\n scheduler = lr_scheduler.StepLR(\n optimizer,\n step_size=cfg.epochs // 3,\n gamma=0.5\n )\n elif cfg.scheduler == 'OneCycleLR':\n scheduler = None\n else:\n scheduler = None\n\n return scheduler\n\n\ndef get_criterion(cfg):\n if cfg.criterion == 'BCEWithLogitsLoss':\n criterion = nn.BCEWithLogitsLoss()\n else:\n raise NotImplementedError(f\"Criterion {cfg.criterion} not implemented\")\n\n return criterion\n\n\ndef train_one_epoch(model, loader, optimizer, criterion, device, scheduler=None):\n model.train()\n losses = []\n all_targets = []\n all_outputs = []\n\n pbar = tqdm(enumerate(loader), total=len(loader), desc=\"Training\")\n\n for step, batch in pbar:\n\n if isinstance(batch['melspec'], list):\n batch_outputs = []\n batch_losses = []\n\n for i in range(len(batch['melspec'])):\n inputs = batch['melspec'][i].unsqueeze(0).to(device)\n target = batch['target'][i].unsqueeze(0).to(device)\n\n optimizer.zero_grad()\n output = model(inputs,target)\n loss = criterion(output, target)\n loss.backward()\n\n batch_outputs.append(output.detach().cpu())\n batch_losses.append(loss.item())\n\n optimizer.step()\n outputs = torch.cat(batch_outputs, dim=0).numpy()\n loss = np.mean(batch_losses)\n targets = batch['target'].numpy()\n\n else:\n inputs = batch['melspec'].to(device)\n targets = batch['target'].to(device)\n\n optimizer.zero_grad()\n outputs = model(inputs,targets)\n\n if isinstance(outputs, tuple):\n outputs, loss = outputs\n else:\n loss = criterion(outputs, targets)\n\n loss.backward()\n optimizer.step()\n\n outputs = outputs.detach().cpu().numpy()\n targets = targets.detach().cpu().numpy()\n\n if scheduler is not None and isinstance(scheduler, lr_scheduler.OneCycleLR):\n scheduler.step()\n\n all_outputs.append(outputs)\n all_targets.append(targets)\n losses.append(loss if isinstance(loss, float) else loss.item())\n\n pbar.set_postfix({\n 'train_loss': np.mean(losses[-10:]) if losses else 0,\n 'lr': optimizer.param_groups[0]['lr']\n })\n\n all_outputs = np.concatenate(all_outputs)\n all_targets = np.concatenate(all_targets)\n auc = calculate_auc(all_targets, all_outputs)\n avg_loss = np.mean(losses)\n\n return avg_loss, auc\n\n\ndef validate(model, loader, criterion, device):\n model.eval()\n losses = []\n all_targets = []\n all_outputs = []\n\n with torch.no_grad():\n for batch in tqdm(loader, desc=\"Validation\"):\n if isinstance(batch['melspec'], list):\n batch_outputs = []\n batch_losses = []\n\n for i in range(len(batch['melspec'])):\n inputs = batch['melspec'][i].unsqueeze(0).to(device)\n target = batch['target'][i].unsqueeze(0).to(device)\n\n output = model(inputs, target)\n loss = criterion(output, target)\n\n batch_outputs.append(output.detach().cpu())\n batch_losses.append(loss.item())\n\n outputs = torch.cat(batch_outputs, dim=0).numpy()\n loss = np.mean(batch_losses)\n targets = batch['target'].numpy()\n\n else:\n inputs = batch['melspec'].to(device)\n targets = batch['target'].to(device)\n\n outputs = model(inputs, targets)\n loss = criterion(outputs, targets)\n\n outputs = outputs.detach().cpu().numpy()\n targets = targets.detach().cpu().numpy()\n\n all_outputs.append(outputs)\n all_targets.append(targets)\n losses.append(loss if isinstance(loss, float) else loss.item())\n\n all_outputs = np.concatenate(all_outputs)\n all_targets = np.concatenate(all_targets)\n\n auc = calculate_auc(all_targets, all_outputs)\n avg_loss = np.mean(losses)\n\n return avg_loss, auc\n\n\ndef calculate_auc(targets, outputs):\n num_classes = targets.shape[1]\n aucs = []\n\n probs = 1 / (1 + np.exp(-outputs))\n\n for i in range(num_classes):\n\n if np.sum(targets[:, i]) > 0:\n class_auc = roc_auc_score(targets[:, i], probs[:, i])\n aucs.append(class_auc)\n\n return np.mean(aucs) if aucs else 0.0\n\n\ndef run_training(df, cfg):\n \"\"\"Training function that can either use pre-computed spectrograms or generate them on-the-fly\"\"\"\n\n taxonomy_df = pd.read_csv(cfg.taxonomy_csv)\n species_ids = taxonomy_df['primary_label'].tolist()\n cfg.num_classes = len(species_ids)\n\n if cfg.debug:\n cfg.update_debug_settings()\n\n spectrograms = None\n if cfg.LOAD_DATA:\n print(\"Loading pre-computed mel spectrograms from NPY file...\")\n try:\n spectrograms = np.load(cfg.spectrogram_npy, allow_pickle=True).item()\n print(f\"Loaded {len(spectrograms)} pre-computed mel spectrograms\")\n except Exception as e:\n print(f\"Error loading pre-computed spectrograms: {e}\")\n print(\"Will generate spectrograms on-the-fly instead.\")\n cfg.LOAD_DATA = False\n\n if not cfg.LOAD_DATA:\n print(\"Will generate spectrograms on-the-fly during training.\")\n if 'filepath' not in df.columns:\n df['filepath'] = cfg.train_datadir + '/' + df.filename\n if 'samplename' not in df.columns:\n df['samplename'] = df.filename.map(lambda x: x.split('/')[0] + '-' + x.split('/')[-1].split('.')[0])\n\n skf = StratifiedKFold(n_splits=cfg.n_fold, shuffle=True, random_state=cfg.seed)\n\n best_scores = []\n\n for fold, (train_idx, val_idx) in enumerate(skf.split(df, df['primary_label'])):\n if fold not in cfg.selected_folds:\n continue\n\n print(f'\\n{\"=\" * 30} Fold {fold} {\"=\" * 30}')\n\n train_df = df.iloc[train_idx].reset_index(drop=True)\n val_df = df.iloc[val_idx].reset_index(drop=True)\n\n print(f'Training set: {len(train_df)} samples')\n print(f'Validation set: {len(val_df)} samples')\n\n train_dataset = BirdCLEFDatasetFromNPY(train_df, cfg, spectrograms=spectrograms, mode='train')\n val_dataset = BirdCLEFDatasetFromNPY(val_df, cfg, spectrograms=spectrograms, mode='valid')\n\n train_loader = DataLoader(\n train_dataset,\n batch_size=cfg.batch_size,\n shuffle=True,\n num_workers=cfg.num_workers,\n pin_memory=True,\n collate_fn=collate_fn,\n drop_last=True\n )\n\n val_loader = DataLoader(\n val_dataset,\n batch_size=cfg.batch_size,\n shuffle=False,\n num_workers=cfg.num_workers,\n pin_memory=True,\n collate_fn=collate_fn\n )\n\n model = BirdCLEFModel(cfg).to(cfg.device)\n optimizer = get_optimizer(model, cfg)\n criterion = get_criterion(cfg)\n\n if cfg.scheduler == 'OneCycleLR':\n scheduler = lr_scheduler.OneCycleLR(\n optimizer,\n max_lr=cfg.lr,\n steps_per_epoch=len(train_loader),\n epochs=cfg.epochs,\n pct_start=0.1\n )\n else:\n scheduler = get_scheduler(optimizer, cfg)\n\n best_auc = 0\n best_epoch = 0\n\n for epoch in range(cfg.epochs):\n print(f\"\\nEpoch {epoch + 1}/{cfg.epochs}\")\n\n train_loss, train_auc = train_one_epoch(\n model,\n train_loader,\n optimizer,\n criterion,\n cfg.device,\n scheduler if isinstance(scheduler, lr_scheduler.OneCycleLR) else None\n )\n\n val_loss, val_auc = validate(model, val_loader, criterion, cfg.device)\n\n if scheduler is not None and not isinstance(scheduler, lr_scheduler.OneCycleLR):\n if isinstance(scheduler, lr_scheduler.ReduceLROnPlateau):\n scheduler.step(val_loss)\n else:\n scheduler.step()\n\n print(f\"Train Loss: {train_loss:.4f}, Train AUC: {train_auc:.4f}\")\n print(f\"Val Loss: {val_loss:.4f}, Val AUC: {val_auc:.4f}\")\n\n if val_auc > best_auc:\n best_auc = val_auc\n best_epoch = epoch + 1\n print(f\"New best AUC: {best_auc:.4f} at epoch {best_epoch}\")\n\n torch.save({\n 'model_state_dict': model.state_dict(),\n 'optimizer_state_dict': optimizer.state_dict(),\n 'scheduler_state_dict': scheduler.state_dict() if scheduler else None,\n 'epoch': epoch,\n 'val_auc': val_auc,\n 'train_auc': train_auc,\n 'cfg': cfg\n }, f\"model_fold{fold}.pth\")\n\n best_scores.append(best_auc)\n print(f\"\\nBest AUC for fold {fold}: {best_auc:.4f} at epoch {best_epoch}\")\n\n # Clear memory\n del model, optimizer, scheduler, train_loader, val_loader\n torch.cuda.empty_cache()\n gc.collect()\n\n print(\"\\n\" + \"=\" * 60)\n print(\"Cross-Validation Results:\")\n for fold, score in enumerate(best_scores):\n print(f\"Fold {cfg.selected_folds[fold]}: {score:.4f}\")\n print(f\"Mean AUC: {np.mean(best_scores):.4f}\")\n print(\"=\" * 60)\n\n\nif __name__ == \"__main__\":\n import time\n\n print(\"\\nLoading training data...\")\n train_df = pd.read_csv(cfg.train_csv)\n taxonomy_df = pd.read_csv(cfg.taxonomy_csv)\n\n print(\"\\nStarting training...\")\n print(f\"LOAD_DATA is set to {cfg.LOAD_DATA}\")\n if cfg.LOAD_DATA:\n print(\"Using pre-computed mel spectrograms from NPY file\")\n else:\n print(\"Will generate spectrograms on-the-fly during training\")\n\n run_training(train_df, cfg)\n\n print(\"\\nTraining complete!\")\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-04-19T11:31:21.69285Z","iopub.execute_input":"2025-04-19T11:31:21.693152Z","iopub.status.idle":"2025-04-19T12:10:54.000817Z","shell.execute_reply.started":"2025-04-19T11:31:21.693133Z","shell.execute_reply":"2025-04-19T12:10:53.999671Z"},"jupyter":{"source_hidden":true}},"outputs":[{"name":"stdout","text":"\nLoading training data...\n\nStarting training...\nLOAD_DATA is set to True\nUsing pre-computed mel spectrograms from NPY file\nLoading pre-computed mel spectrograms from NPY file...\nLoaded 28564 pre-computed mel spectrograms\n\n============================== Fold 0 ==============================\nTraining set: 14282 samples\nValidation set: 14282 samples\nFound 14282 matching spectrograms for train dataset out of 14282 samples\nFound 14282 matching spectrograms for valid dataset out of 14282 samples\n\nEpoch 1/10\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Training: 0%| | 0/446 [00:00