{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"},{"sourceId":11840064,"sourceType":"datasetVersion","datasetId":7438986},{"sourceId":11855918,"sourceType":"datasetVersion","datasetId":7449695},{"sourceId":11855938,"sourceType":"datasetVersion","datasetId":7439602}],"dockerImageVersionId":31011,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import Modules","metadata":{}},{"cell_type":"code","source":"!pip install /kaggle/input/birdclef-dependencies/audiomentations-0.33.0-py3-none-any.whl","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport librosa\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchaudio\nimport os\nimport random\nimport shutil\nimport logging\nimport pickle\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_curve, auc, confusion_matrix, precision_score, recall_score, f1_score, roc_auc_score\nfrom tqdm import tqdm\nfrom audiomentations import Compose, AddGaussianNoise\nfrom torch.cuda.amp import autocast, GradScaler\nimport timm\nimport joblib\nimport warnings\nimport glob\nimport time\n\nwarnings.filterwarnings(\"ignore\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Logging Setup","metadata":{}},{"cell_type":"code","source":"logging.basicConfig(\n    level=logging.INFO,\n    format='%(asctime)s - %(levelname)s - %(message)s',\n    handlers=[logging.FileHandler('birdclef.log'), logging.StreamHandler()]\n)\nlogger = logging.getLogger(__name__)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Configuration","metadata":{}},{"cell_type":"code","source":"class Config:\n    train_dir = \"/kaggle/input/birdclef-2025/train_audio\"\n    train_csv = \"/kaggle/input/birdclef-2025/train.csv\"\n    test_soundscapes = \"/kaggle/input/birdclef-2025/test_soundscapes\"\n    sample_csv = \"/kaggle/input/birdclef-2025/sample_submission.csv\"\n    weights = {\n        'tf_efficientnet_b3': \"/kaggle/input/efficientnet-weights/tf_efficientnet_b3.pth\",\n        'convnext_small': \"/kaggle/input/convnext-weights/convnext_small.pth\"\n    }\n    feature_dir = \"/kaggle/working/features\"\n    model_weights_dir = \"/kaggle/working/model_weights\"\n    checkpoint_dir = \"/kaggle/working/checkpoints\"\n    sr = 32000\n    num_classes = 206\n    n_folds = 5\n    n_fft = 1024\n    hop_length = 512\n    n_mels = 128\n    n_mels_low = 80\n    fmin = 50\n    fmax = 16000\n    chunk_duration = 10\n    batch_size = 4\n    epochs = 10\n    lr = 5e-5\n    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    early_stop_patience = 2\n    tta_chunks = 1\n    max_pseudo_samples = 800\n    dropout = 0.3\n    label_smoothing = 0.1\n    seed = 42\n    eval_timeout = 300","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Helper Functions","metadata":{}},{"cell_type":"code","source":"def set_seed(seed=Config.seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    logger.info(f\"Seed set to {seed}\")\n\ndef get_disk_usage(path=\"/kaggle/working\"):\n    total_size = sum(os.path.getsize(os.path.join(dirpath, f)) for dirpath, _, filenames in os.walk(path) for f in filenames)\n    return total_size / (1024**3)  # GiB\n\ndef save_checkpoint(model_name, fold, state, model, optimizer):\n    os.makedirs(Config.checkpoint_dir, exist_ok=True)\n    checkpoint_path = os.path.join(Config.checkpoint_dir, f\"checkpoint_{model_name}_fold_{fold}.pkl\")\n    state.update({\n        'model_state': model.state_dict(),\n        'optimizer_state': optimizer.state_dict()\n    })\n    with open(checkpoint_path, 'wb') as f:\n        pickle.dump(state, f)\n    logger.info(f\"Saved checkpoint: {checkpoint_path}\")\n\ndef load_checkpoint(model_name, fold):\n    checkpoint_path = os.path.join(Config.checkpoint_dir, f\"checkpoint_{model_name}_fold_{fold}.pkl\")\n    if os.path.exists(checkpoint_path):\n        with open(checkpoint_path, 'rb') as f:\n            logger.info(f\"Loaded checkpoint: {checkpoint_path}\")\n            return pickle.load(f)\n    return None","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Audio Augmentation","metadata":{}},{"cell_type":"code","source":"train_augment = Compose([AddGaussianNoise(min_amplitude=0.001, max_amplitude=0.015, p=0.5)])\ntta_augment = Compose([AddGaussianNoise(min_amplitude=0.001, max_amplitude=0.01, p=0.5)])\n\ndef spec_augment(spec, max_time_mask=0.1, max_freq_mask=0.1):\n    spec = torch.tensor(spec, dtype=torch.float32)\n    time_steps, freq_bins = spec.shape[-1], spec.shape[-2]\n    for _ in range(2):\n        t = int(random.uniform(0, max_time_mask) * time_steps)\n        t0 = random.randint(0, time_steps - t)\n        spec[:, :, t0:t0+t] = 0\n    for _ in range(2):\n        f = int(random.uniform(0, max_freq_mask) * freq_bins)\n        f0 = random.randint(0, freq_bins - f)\n        spec[:, f0:f0+f, :] = 0\n    return spec.numpy()\n\ndef mixup(x, y, alpha=0.2):\n    lam = np.random.beta(alpha, alpha)\n    index = torch.randperm(x.size(0)).to(x.device)\n    mixed_x = lam * x + (1 - lam) * x[index]\n    return mixed_x, y, y[index], lam","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"code","source":"def extract_features(audio, sr=Config.sr, n_mels=Config.n_mels, hop_length=Config.hop_length):\n    transform = torchaudio.transforms.MelSpectrogram(\n        sample_rate=sr, n_fft=Config.n_fft, hop_length=hop_length,\n        n_mels=n_mels, f_min=Config.fmin, f_max=Config.fmax\n    )\n    mel_sp = transform(torch.tensor(audio, dtype=torch.float32)).numpy()\n    mel_sp = np.clip(mel_sp, a_min=1e-10, a_max=1e10)\n    mel_sp = librosa.power_to_db(mel_sp, ref=1.0)\n    mel_sp = np.clip(mel_sp, a_min=-100, a_max=100)\n    chroma = librosa.feature.chroma_stft(y=audio, sr=sr, n_fft=Config.n_fft, hop_length=hop_length)\n    spectral_contrast = librosa.feature.spectral_contrast(y=audio, sr=sr, n_fft=Config.n_fft, hop_length=hop_length)\n    return mel_sp, chroma, spectral_contrast\n\ndef transform_features(mel_sp, chroma, spectral_contrast):\n    mel_sp = (mel_sp - mel_sp.min()) / (mel_sp.max() - mel_sp.min() + 1e-10)\n    chroma = (chroma - chroma.min()) / (chroma.max() - chroma.min() + 1e-10)\n    spectral_contrast = (spectral_contrast - spectral_contrast.min()) / (spectral_contrast.max() - spectral_contrast.min() + 1e-10)\n    mel_delta = librosa.feature.delta(mel_sp)\n    mel_delta2 = librosa.feature.delta(mel_sp, order=2)\n    mel_delta = np.where(np.isnan(mel_delta) | np.isinf(mel_delta), 0.0, mel_delta)\n    mel_delta2 = np.where(np.isnan(mel_delta2) | np.isinf(mel_delta2), 0.0, mel_delta2)\n    features = np.stack([mel_sp, chroma, spectral_contrast, mel_delta, mel_delta2], axis=0)\n    features = np.where(np.isnan(features) | np.isinf(features), 0.0, features)\n    return features\n\ndef process_audio(row, mode=\"train\"):\n    idx, row = row\n    audio_path = row['filename']\n    feature_path = os.path.join(Config.feature_dir, f\"{idx}_{mode}.npz\")\n    if os.path.exists(feature_path):\n        logger.info(f\"Feature exists: {feature_path}\")\n        return feature_path\n    try:\n        audio, _ = librosa.load(audio_path, sr=Config.sr)\n        if np.all(audio == 0) or np.std(audio) < 1e-6:\n            raise ValueError(\"Invalid audio data\")\n        audio = audio * 1024\n        if mode == \"train\":\n            audio = train_augment(samples=audio, sample_rate=Config.sr)\n        min_len = Config.chunk_duration * Config.sr\n        if len(audio) < min_len:\n            audio = np.tile(audio, int(np.ceil(min_len / len(audio))))\n        audio = audio[:min_len]\n        mel_sp, chroma, spectral_contrast = extract_features(audio, n_mels=Config.n_mels)\n        mel_sp_low, _, _ = extract_features(audio, n_mels=Config.n_mels_low)\n        features = transform_features(mel_sp, chroma, spectral_contrast)\n        if mode == \"train\" and random.random() < 0.5:\n            features[0] = spec_augment(features[0][np.newaxis, :, :], max_time_mask=0.1, max_freq_mask=0.1)[0]\n            features[3] = librosa.feature.delta(features[0])\n            features[4] = librosa.feature.delta(features[0], order=2)\n        features = np.concatenate([features, mel_sp_low[np.newaxis, :, :]], axis=0)[:, :, :641]\n        np.savez_compressed(feature_path, features=features, allow_pickle=False)\n    except Exception as e:\n        logger.warning(f\"Error processing {audio_path}: {e}\")\n        features = np.zeros((6, Config.n_mels, 641), dtype=np.float32)\n        np.savez_compressed(feature_path, features=features, allow_pickle=False)\n    return feature_path\n\ndef precompute_features(df, mode=\"train\"):\n    os.makedirs(Config.feature_dir, exist_ok=True)\n    if df.empty:\n        logger.warning(f\"Empty DataFrame for {mode} features, returning empty DataFrame\")\n        return df\n    feature_paths = joblib.Parallel(n_jobs=4, backend='loky', verbose=10)(\n        joblib.delayed(process_audio)(row, mode) for row in df.iterrows()\n    )\n    df['feature_path'] = feature_paths\n    return df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Dataset  ","metadata":{}},{"cell_type":"code","source":"sample_submission = pd.read_csv(Config.sample_csv)\nall_classes = sample_submission.columns[1:].tolist()\nlabel_mapper = {label: idx for idx, label in enumerate(all_classes)}\nrev_mapper = {idx: label for label, idx in label_mapper.items()}\n\nclass BirdClefDataset(torch.utils.data.Dataset):\n    def __init__(self, df, mode=\"train\"):\n        self.df = df\n        self.mode = mode\n        if mode == \"train\" and not df.empty:\n            class_counts = self.df['primary_label'].value_counts()\n            oversample_df = pd.concat([\n                self.df[self.df['primary_label'] == label].sample(n=20, replace=True, random_state=Config.seed)\n                for label in class_counts[class_counts < 20].index\n            ])\n            self.df = pd.concat([self.df, oversample_df], ignore_index=True)\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        feature_path = row['feature_path']\n        try:\n            features = np.load(feature_path, allow_pickle=False)['features'].astype(np.float32)\n            if np.any(np.isnan(features)) or np.any(np.isinf(features)):\n                logger.warning(f\"NaN/Inf in features at {feature_path}\")\n                features = np.zeros((6, Config.n_mels, 641), dtype=np.float32)\n        except Exception as e:\n            logger.warning(f\"Error loading {feature_path}: {e}\")\n            features = np.zeros((6, Config.n_mels, 641), dtype=np.float32)\n        if self.mode == \"train\":\n            return torch.tensor(features, dtype=torch.float32), torch.tensor(label_mapper[row['primary_label']], dtype=torch.long)\n        return torch.tensor(features, dtype=torch.float32)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class BirdClefModel(nn.Module):\n    def __init__(self, model_name, weights_path):\n        super().__init__()\n        self.model_name = model_name\n        self.base_model = timm.create_model(\n            model_name=model_name,\n            num_classes=Config.num_classes,\n            pretrained=False,\n            in_chans=6,\n            drop_rate=Config.dropout\n        )\n        if os.path.exists(weights_path):\n            state_dict = torch.load(weights_path, map_location=Config.device)\n            if 'convnext' in model_name:\n                if 'stem.0.weight' in state_dict and state_dict['stem.0.weight'].shape[1] != 6:\n                    logger.info(f\"Adjusting stem.0.weight for {model_name} from {state_dict['stem.0.weight'].shape} to 6 input channels\")\n                    conv_stem_weight = state_dict['stem.0.weight']\n                    new_conv_stem_weight = torch.zeros(\n                        conv_stem_weight.shape[0], 6, conv_stem_weight.shape[2], conv_stem_weight.shape[3],\n                        device=Config.device\n                    )\n                    new_conv_stem_weight[:, :3, :, :] = conv_stem_weight\n                    new_conv_stem_weight[:, 3:, :, :] = conv_stem_weight[:, :3, :, :].mean(dim=1, keepdim=True)\n                    state_dict['stem.0.weight'] = new_conv_stem_weight\n            else:\n                if 'conv_stem.weight' in state_dict and state_dict['conv_stem.weight'].shape[1] != 6:\n                    logger.info(f\"Adjusting conv_stem.weight for {model_name} from {state_dict['conv_stem.weight'].shape} to 6 input channels\")\n                    conv_stem_weight = state_dict['conv_stem.weight']\n                    new_conv_stem_weight = torch.zeros(\n                        conv_stem_weight.shape[0], 6, conv_stem_weight.shape[2], conv_stem_weight.shape[3],\n                        device=Config.device\n                    )\n                    new_conv_stem_weight[:, :3, :, :] = conv_stem_weight\n                    new_conv_stem_weight[:, 3:, :, :] = conv_stem_weight[:, :3, :, :].mean(dim=1, keepdim=True)\n                    state_dict['conv_stem.weight'] = new_conv_stem_weight\n            exclude_keys = ['classifier.weight', 'classifier.bias', 'fc.weight', 'fc.bias', 'head.fc.weight', 'head.fc.bias']\n            state_dict = {k: v for k, v in state_dict.items() if k not in exclude_keys}\n            missing_keys, unexpected_keys = self.base_model.load_state_dict(state_dict, strict=False)\n            logger.info(f\"Loaded weights from {weights_path}. Missing keys: {missing_keys}, Unexpected keys: {unexpected_keys}\")\n            with torch.no_grad():\n                if hasattr(self.base_model, 'classifier'):\n                    nn.init.xavier_uniform_(self.base_model.classifier.weight)\n                    nn.init.zeros_(self.base_model.classifier.bias)\n                elif hasattr(self.base_model, 'fc'):\n                    nn.init.xavier_uniform_(self.base_model.fc.weight)\n                    nn.init.zeros_(self.base_model.fc.bias)\n                elif hasattr(self.base_model, 'head') and hasattr(self.base_model.head, 'fc'):\n                    nn.init.xavier_uniform_(self.base_model.head.fc.weight)\n                    nn.init.zeros_(self.base_model.head.fc.bias)\n        else:\n            logger.warning(f\"Weights not found at {weights_path}. Using random initialization.\")\n            with torch.no_grad():\n                if hasattr(self.base_model, 'classifier'):\n                    nn.init.xavier_uniform_(self.base_model.classifier.weight)\n                    nn.init.zeros_(self.base_model.classifier.bias)\n                elif hasattr(self.base_model, 'fc'):\n                    nn.init.xavier_uniform_(self.base_model.fc.weight)\n                    nn.init.zeros_(self.base_model.fc.bias)\n                elif hasattr(self.base_model, 'head') and hasattr(self.base_model.head, 'fc'):\n                    nn.init.xavier_uniform_(self.base_model.head.fc.weight)\n                    nn.init.zeros_(self.base_model.head.fc.bias)\n\n    def forward(self, x):\n        return self.base_model(x)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# FocalLoss","metadata":{}},{"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self, gamma=2.0, label_smoothing=0.1):\n        super().__init__()\n        self.gamma = gamma\n        self.label_smoothing = label_smoothing\n\n    def forward(self, inputs, targets):\n        log_probs = F.log_softmax(inputs, dim=-1)\n        targets_one_hot = F.one_hot(targets, num_classes=Config.num_classes).float()\n        targets_smooth = targets_one_hot * (1 - self.label_smoothing) + self.label_smoothing / Config.num_classes\n        ce_loss = -torch.sum(targets_smooth * log_probs, dim=-1)\n        pt = torch.exp(-ce_loss)\n        focal_loss = (1 - pt) ** self.gamma * ce_loss\n        return focal_loss.mean()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training and Validation","metadata":{}},{"cell_type":"code","source":"def train_epoch(model, loader, criterion, optimizer, scaler, fold, epoch, model_name):\n    model.train()\n    running_loss = 0.0\n    preds, labels = [], []\n    for x, y in tqdm(loader, desc=f\"Fold {fold + 1}/{Config.n_folds}, Epoch {epoch + 1}/{Config.epochs}, Model: {model_name} - Training\"):\n        x, y = x.to(Config.device), y.to(Config.device)\n        optimizer.zero_grad()\n        with autocast():\n            if random.random() < 0.5:\n                x, y_a, y_b, lam = mixup(x, y)\n                outputs = model(x)\n                loss = lam * criterion(outputs, y_a) + (1 - lam) * criterion(outputs, y_b)\n            else:\n                outputs = model(x)\n                loss = criterion(outputs, y)\n            if torch.isnan(loss) or torch.isinf(loss):\n                logger.warning(\"NaN/Inf loss detected, skipping batch\")\n                continue\n        scaler.scale(loss).backward()\n        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n        scaler.step(optimizer)\n        scaler.update()\n        running_loss += loss.item()\n        probs = torch.softmax(outputs, dim=1).detach().cpu().numpy()\n        preds.append(probs)\n        labels.append(y.cpu().numpy())\n    return running_loss / len(loader), np.concatenate(preds), np.concatenate(labels)\n\ndef validate_epoch(model, loader, criterion, fold, epoch, model_name):\n    model.eval()\n    running_loss = 0.0\n    preds, labels = [], []\n    with torch.no_grad():\n        for x, y in tqdm(loader, desc=f\"Fold {fold + 1}/{Config.n_folds}, Epoch {epoch + 1}/{Config.epochs}, Model: {model_name} - Validating\"):\n            x, y = x.to(Config.device), y.to(Config.device)\n            outputs = model(x)\n            if torch.any(torch.isnan(outputs)) or torch.any(torch.isinf(outputs)):\n                logger.warning(\"NaN/Inf in model outputs, skipping batch\")\n                continue\n            loss = criterion(outputs, y)\n            running_loss += loss.item()\n            probs = torch.softmax(outputs, dim=1)\n            probs = probs / (probs.sum(dim=1, keepdim=True) + 1e-10)\n            probs = probs.detach().cpu().numpy()\n            if not np.allclose(probs.sum(axis=1), 1.0, atol=1e-5):\n                logger.warning(\"Probabilities do not sum to 1, normalizing\")\n                probs = probs / (probs.sum(axis=1, keepdim=True) + 1e-10)\n            preds.append(probs)\n            labels.append(y.cpu().numpy())\n    return running_loss / len(loader), np.concatenate(preds), np.concatenate(labels)\n\ndef compute_metrics(labels, preds, probs):\n    cm = confusion_matrix(labels, preds)\n    precision = precision_score(labels, preds, average='macro', zero_division=0)\n    recall = recall_score(labels, preds, average='macro', zero_division=0)\n    f1 = f1_score(labels, preds, average='macro', zero_division=0)\n    specificity = np.mean([(cm.sum() - cm[i, :].sum() - cm[:, i].sum() + cm[i, i]) / \n                          (cm.sum() - cm[i, :].sum() + 1e-12) for i in range(len(np.unique(labels)))])\n    return recall, specificity, precision, f1, cm","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"def predict_test_ensemble(test_df, device, model_configs, fold_aucs, folds=Config.n_folds):\n    logger.info(f\"Starting prediction with test_df size: {len(test_df)} samples\")\n    if test_df.empty:\n        logger.warning(\"Test DataFrame is empty (likely no test soundscape files in training phase), returning uniform probabilities matching sample_submission\")\n        return np.full((len(sample_submission), Config.num_classes), 1.0 / Config.num_classes)\n    \n    predictions = []\n    total_weight = sum(fold_aucs[model['name']][-1] for model in model_configs)\n    weights = [fold_aucs[model['name']][-1] / total_weight for model in model_configs]\n    test_loader = torch.utils.data.DataLoader(\n        BirdClefDataset(test_df, mode=\"test\"),\n        batch_size=Config.batch_size * 2,\n        shuffle=False,\n        num_workers=2\n    )\n    logger.info(f\"Test loader created with {len(test_loader)} batches\")\n    \n    for model_config, weight in zip(model_configs, weights):\n        model_name = model_config['name']\n        weights_path = model_config['weights_path']\n        model_preds = []\n        for fold in range(folds):\n            model = BirdClefModel(model_name, weights_path).to(device)\n            weight_path = os.path.join(Config.model_weights_dir, f\"{model_name}_fold_{fold}.pth\")\n            if os.path.exists(weight_path):\n                model.load_state_dict(torch.load(weight_path))\n                logger.info(f\"Loaded fold {fold + 1} weights from {weight_path}\")\n            else:\n                logger.warning(f\"Weight path {weight_path} not found, skipping fold {fold + 1}\")\n                continue\n            model.eval()\n            fold_preds = []\n            with torch.no_grad():\n                for features in tqdm(test_loader, desc=f\"Predicting Fold {fold + 1}/{folds}, Model: {model_name}\"):\n                    features = features.to(device)\n                    tta_preds = []\n                    for _ in range(Config.tta_chunks):\n                        outputs = model(features)\n                        if torch.any(torch.isnan(outputs)) or torch.any(torch.isinf(outputs)):\n                            logger.warning(f\"NaN/Inf in test outputs for Fold {fold + 1}, Model: {model_name}, skipping batch\")\n                            continue\n                        probs = torch.softmax(outputs, dim=1)\n                        probs = probs / (probs.sum(dim=1, keepdim=True) + 1e-10)\n                        probs = probs.cpu().numpy()\n                        if np.any(np.isnan(probs)) or np.any(np.isinf(probs)):\n                            logger.warning(f\"NaN/Inf in probabilities for Fold {fold + 1}, Model: {model_name}, skipping batch\")\n                            continue\n                        tta_preds.append(probs)\n                    if tta_preds:\n                        fold_preds.append(np.mean(tta_preds, axis=0))\n            if fold_preds:\n                model_preds.append(np.concatenate(fold_preds))\n            else:\n                logger.warning(f\"No predictions for Fold {fold + 1}, Model: {model_name}\")\n        if model_preds:\n            predictions.append(np.mean(model_preds, axis=0) * weight)\n        else:\n            logger.warning(f\"No predictions for Model: {model_name}\")\n    \n    if predictions:\n        test_preds = np.sum(predictions, axis=0)\n    else:\n        logger.warning(\"No predictions generated, returning uniform probabilities matching sample_submission\")\n        test_preds = np.full((len(sample_submission), Config.num_classes), 1.0 / Config.num_classes)\n    \n    logger.info(f\"Generated test_preds with shape: {test_preds.shape}\")\n    if test_preds.shape[0] != len(sample_submission):\n        logger.warning(f\"Shape mismatch: test_preds has {test_preds.shape[0]} rows, sample_submission has {len(sample_submission)} rows\")\n        test_preds = np.full((len(sample_submission), Config.num_classes), 1.0 / Config.num_classes)\n        logger.info(\"Replaced test_preds with uniform probabilities to match sample_submission\")\n    \n    if np.any(np.isnan(test_preds)) or np.any(np.isinf(test_preds)):\n        logger.warning(\"NaN/Inf in test_preds, replacing with uniform probabilities\")\n        test_preds = np.full((len(sample_submission), Config.num_classes), 1.0 / Config.num_classes)\n    \n    return test_preds","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"set_seed()\nlogger.info(\"Starting BirdCLEF 2025 pipeline\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Preprocess data","metadata":{}},{"cell_type":"code","source":"logger.info(\"Preprocessing training data\")\ndata_df_path = '/kaggle/working/train_data.csv'\nif os.path.exists(data_df_path):\n    logger.info(f\"Loading preprocessed data from {data_df_path}\")\n    data_df = pd.read_csv(data_df_path)\nelse:\n    data_df = pd.read_csv(Config.train_csv)\n    data_df['filename'] = data_df['filename'].apply(lambda x: os.path.join(Config.train_dir, x))\n    logger.info(\"Computing features\")\n    data_df = precompute_features(data_df)\n    skf = StratifiedKFold(n_splits=Config.n_folds, shuffle=True, random_state=Config.seed)\n    data_df['kfold'] = -1\n    for fold, (_, val_idx) in enumerate(skf.split(data_df, data_df['primary_label'])):\n        data_df.iloc[val_idx, data_df.columns.get_loc('kfold')] = fold\n    data_df.to_csv(data_df_path, index=False)\n    logger.info(f\"Saved preprocessed data to {data_df_path}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training\n","metadata":{}},{"cell_type":"code","source":"logger.info(\"Starting training\")\nmodels = [{'name': k, 'weights_path': v} for k, v in Config.weights.items()]\nfold_aucs = {m['name']: [] for m in models}\nall_metrics = {m['name']: [] for m in models}\npreliminary_results = []\n\nfor model_config in models:\n    model_name = model_config['name']\n    weights_path = model_config['weights_path']\n    logger.info(f\"Training model: {model_name}\")\n    logger.info(f\"Checking initial disk usage: {get_disk_usage():.2f} GiB\")\n    if not os.path.exists(weights_path):\n        logger.error(f\"Pretrained weights not found at {weights_path}, cannot proceed with {model_name}\")\n        continue\n    for fold in range(Config.n_folds):\n        logger.info(f\"Processing Fold {fold + 1}/{Config.n_folds}\")\n        weight_path = os.path.join(Config.model_weights_dir, f\"{model_name}_fold_{fold}.pth\")\n        checkpoint = load_checkpoint(model_name, fold)\n        if checkpoint and os.path.exists(weight_path):\n            logger.info(f\"Skipping Fold {fold + 1} (checkpoint and weights exist)\")\n            fold_aucs[model_name].append(checkpoint['fold_aucs'][model_name][-1])\n            all_metrics[model_name].append(checkpoint['metrics'][model_name][-1])\n            preliminary_results.append(checkpoint['preliminary_results'][-1])\n            continue\n        elif checkpoint:\n            logger.warning(f\"Checkpoint exists for Fold {fold + 1}, but weights {weight_path} missing, retraining\")\n        train_df = data_df[data_df['kfold'] != fold].reset_index(drop=True)\n        val_df = data_df[data_df['kfold'] == fold].reset_index(drop=True)\n        train_ds = BirdClefDataset(train_df)\n        val_ds = BirdClefDataset(val_df)\n        train_loader = torch.utils.data.DataLoader(\n            train_ds,\n            batch_size=Config.batch_size,\n            shuffle=True,\n            num_workers=2\n        )\n        val_loader = torch.utils.data.DataLoader(\n            val_ds,\n            batch_size=Config.batch_size,\n            shuffle=False,\n            num_workers=2\n        )\n        \n        model = BirdClefModel(model_name, weights_path).to(Config.device)\n        criterion = FocalLoss().to(Config.device)\n        optimizer = torch.optim.Adam(model.parameters(), lr=Config.lr)\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=Config.epochs)\n        scaler = GradScaler()\n        \n        best_auc = 0\n        early_stop_counter = 0\n        try:\n            for epoch in range(Config.epochs):\n                logger.info(f\"Disk usage before epoch {epoch + 1}: {get_disk_usage():.2f} GiB\")\n                train_loss, train_preds, train_labels = train_epoch(model, train_loader, criterion, optimizer, scaler, fold, epoch, model_name)\n                val_loss, val_preds, val_labels = validate_epoch(model, val_loader, criterion, fold, epoch, model_name)\n                unique_labels = np.unique(val_labels)\n                if len(unique_labels) < Config.num_classes:\n                    valid_indices = [i for i in unique_labels if i < Config.num_classes]\n                    if not valid_indices:\n                        logger.warning(f\"No valid labels in Fold {fold + 1}, Epoch {epoch + 1}, skipping AUC\")\n                        auc_val = 0.0\n                    else:\n                        val_preds_subset = val_preds[:, valid_indices]\n                        val_preds_subset = val_preds_subset / (val_preds_subset.sum(axis=1, keepdims=True) + 1e-10)\n                        auc_val = roc_auc_score(val_labels, val_preds_subset, multi_class='ovr', average='macro')\n                else:\n                    auc_val = roc_auc_score(val_labels, val_preds, multi_class='ovr', average='macro')\n                preds = np.argmax(val_preds, axis=1)\n                metrics = compute_metrics(val_labels, preds, val_preds)\n                logger.info(f\"Fold {fold + 1}/{Config.n_folds}, Epoch {epoch + 1}/{Config.epochs}, Model: {model_name} - Val AUC: {auc_val:.4f}, Sensitivity: {metrics[0]:.4f}, Specificity: {metrics[1]:.4f}, Precision: {metrics[2]:.4f}, F1: {metrics[3]:.4f}\")\n                \n                if auc_val > best_auc:\n                    best_auc = auc_val\n                    early_stop_counter = 0\n                    os.makedirs(Config.model_weights_dir, exist_ok=True)\n                    torch.save(model.state_dict(), weight_path)\n                    logger.info(f\"Saved best model weights for AUC: {best_auc:.4f} at {weight_path}\")\n                    if os.path.exists(weight_path):\n                        logger.info(f\"Confirmed weights saved: {weight_path}, size: {os.path.getsize(weight_path) / (1024**2):.2f} MB\")\n                    else:\n                        logger.error(f\"Failed to save weights at {weight_path}\")\n                else:\n                    early_stop_counter += 1\n                    if early_stop_counter >= Config.early_stop_patience:\n                        logger.info(f\"Early stopping at Fold {fold + 1}, Epoch {epoch + 1}\")\n                        break\n                \n                scheduler.step()\n            \n            # Save weights at the end of training if not saved\n            if not os.path.exists(weight_path):\n                logger.warning(f\"No weights saved during training for Fold {fold + 1}, saving final model\")\n                torch.save(model.state_dict(), weight_path)\n                logger.info(f\"Saved final model weights at {weight_path}\")\n                if os.path.exists(weight_path):\n                    logger.info(f\"Confirmed final weights saved: {weight_path}, size: {os.path.getsize(weight_path) / (1024**2):.2f} MB\")\n                else:\n                    logger.error(f\"Failed to save final weights at {weight_path}\")\n        except Exception as e:\n            logger.error(f\"Error during training Fold {fold + 1}, Model: {model_name}: {e}\")\n            continue\n        \n        fold_aucs[model_name].append(best_auc)\n        all_metrics[model_name].append(metrics)\n        preliminary_results.append({\n            'Model': model_name, 'Fold': fold, 'AUC': best_auc,\n            **dict(zip(['Sensitivity', 'Specificity', 'Precision', 'F1'], metrics[:4]))\n        })\n        save_checkpoint(model_name, fold, {\n            'fold_aucs': fold_aucs, 'metrics': all_metrics, 'preliminary_results': preliminary_results\n        }, model, optimizer)\n        logger.info(f\"Disk usage after Fold {fold + 1}: {get_disk_usage():.2f} GiB\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Evaluation\n","metadata":{}},{"cell_type":"code","source":"logger.info(\"Generating ROC curves and confusion matrices\")\nfor model_name in tqdm([m['name'] for m in models], desc=\"Evaluating Models\", leave=True, dynamic_ncols=True):\n    plt.figure(figsize=(10, 8))\n    mean_fpr = np.linspace(0, 1, 100)\n    tprs, aucs = [], []\n    for fold in range(Config.n_folds):\n        start_time = time.time()\n        logger.info(f\"Computing ROC for Model: {model_name}, Fold {fold + 1}/{Config.n_folds}\")\n        labels, probs = [], []\n        val_df = data_df[data_df['kfold'] == fold]\n        logger.info(f\"Validation set size: {len(val_df)} samples\")\n        val_ds = BirdClefDataset(val_df)\n        val_loader = torch.utils.data.DataLoader(\n            val_ds,\n            batch_size=Config.batch_size,\n            shuffle=False,\n            num_workers=2\n        )\n        model = BirdClefModel(model_name, Config.weights[model_name]).to(Config.device)\n        weight_path = os.path.join(Config.model_weights_dir, f\"{model_name}_fold_{fold}.pth\")\n        if os.path.exists(weight_path):\n            model.load_state_dict(torch.load(weight_path))\n            logger.info(f\"Loaded fold {fold + 1} weights from {weight_path}\")\n        else:\n            logger.warning(f\"Weight path {weight_path} not found, skipping fold\")\n            continue\n        model.eval()\n        with torch.no_grad():\n            for x, y in tqdm(val_loader, desc=f\"Evaluating Fold {fold + 1}/{Config.n_folds}, Model: {model_name}\", leave=False, dynamic_ncols=True, mininterval=1.0):\n                x = x.to(Config.device)\n                outputs = model(x)\n                probs_batch = torch.softmax(outputs, dim=1).cpu().numpy()\n                probs.append(probs_batch)\n                labels.append(y.numpy())\n        if not probs or not labels:\n            logger.warning(f\"No valid data for ROC in Fold {fold + 1}, Model: {model_name}\")\n            continue\n        labels, probs = np.concatenate(labels), np.concatenate(probs)\n        logger.info(f\"Number of unique labels: {len(np.unique(labels))}\")\n        unique_labels = np.unique(labels)\n        if len(unique_labels) < Config.num_classes:\n            valid_indices = [i for i in unique_labels if i < Config.num_classes]\n            if not valid_indices:\n                logger.warning(f\"No valid labels for ROC in Fold {fold + 1}, Model: {model_name}\")\n                continue\n            probs = probs[:, valid_indices]\n            probs = probs / (probs.sum(axis=1, keepdims=True) + 1e-10)\n            labels = np.array([i if i in valid_indices else valid_indices[0] for i in labels])\n        try:\n            fpr, tpr, _ = roc_curve(labels, probs, multi_class='ovr')\n            tprs.append(np.interp(mean_fpr, fpr, tpr))\n            auc_val = auc(mean_fpr, tprs[-1])\n            aucs.append(auc_val)\n            plt.plot(mean_fpr, tprs[-1], alpha=0.3, label=f'Fold {fold + 1} (AUC = {auc_val:.2f})')\n            logger.info(f\"ROC computed for Fold {fold + 1}, Model: {model_name}, AUC: {auc_val:.2f}\")\n        except Exception as e:\n            logger.warning(f\"Error computing ROC for Fold {fold + 1}, Model: {model_name}: {e}\")\n            continue\n        if time.time() - start_time > Config.eval_timeout:\n            logger.warning(f\"Timeout after {Config.eval_timeout}s for Fold {fold + 1}, Model: {model_name}, skipping remaining\")\n            break\n    if tprs:\n        mean_tpr = np.mean(tprs, axis=0)\n        plt.plot(mean_fpr, mean_tpr, 'b', label=f'Mean ROC (AUC = {auc(mean_fpr, mean_tpr):.2f})')\n        plt.fill_between(mean_fpr, mean_tpr - np.std(tprs, axis=0), mean_tpr + np.std(tprs, axis=0), color='b', alpha=0.2)\n    plt.plot([0, 1], [0, 1], 'r--')\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title(f'ROC Curves ({model_name})')\n    plt.legend(loc='lower right')\n    plt.savefig(f\"roc_{model_name}.png\")\n    plt.close()\n    logger.info(f\"Saved ROC curve for {model_name} to roc_{model_name}.png\")\n\n    for fold in tqdm(range(Config.n_folds), desc=f\"Generating Confusion Matrices for {model_name}\", leave=False, dynamic_ncols=True):\n        cm = all_metrics[model_name][fold][4]\n        cm_norm = cm / (cm.sum(axis=1, keepdims=True) + 1e-10)\n        plt.figure(figsize=(8, 6))\n        sns.heatmap(cm_norm, annot=False, cmap='Blues')\n        plt.title(f'Confusion Matrix ({model_name}, Fold {fold + 1})')\n        plt.savefig(f\"cm_{model_name}_fold_{fold + 1}.png\")\n        plt.close()\n        logger.info(f\"Saved confusion matrix for {model_name}, Fold {fold + 1} to cm_{model_name}_fold_{fold + 1}.png\")\n\nfinal_results = [\n    {\n        'Model': model_name,\n        'Avg AUC': np.mean(fold_aucs[model_name]),\n        **{k: np.mean([m[i] for m in all_metrics[model_name]]) for i, k in enumerate(['Sensitivity', 'Specificity', 'Precision', 'F1'])}\n    }\n    for model_name in [m['name'] for m in models]\n]\npd.DataFrame(final_results).to_csv('final_results.csv', index=False)\nlogger.info(\"Saved final results to final_results.csv\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"logger.info(\"Generating submission\")\ntest_files = glob.glob(os.path.join(Config.test_soundscapes, \"*.ogg\"))\nlogger.info(f\"Found {len(test_files)} test soundscape files\")\nif test_files:\n    test_df = pd.DataFrame({'filename': test_files})\n    test_df = precompute_features(test_df, mode=\"test\")\n    test_preds = predict_test_ensemble(test_df, Config.device, models, fold_aucs)\nelse:\n    logger.warning(\"No test soundscape files found (training phase), creating mock submission with uniform probabilities\")\n    test_preds = np.full((len(sample_submission), Config.num_classes), 1.0 / Config.num_classes)\n\nlogger.info(f\"sample_submission shape: {sample_submission.shape}, test_preds shape: {test_preds.shape}\")\nif test_preds.shape[0] != len(sample_submission) or test_preds.shape[1] != Config.num_classes:\n    logger.warning(f\"Shape mismatch: test_preds {test_preds.shape}, expected ({len(sample_submission)}, {Config.num_classes})\")\n    test_preds = np.full((len(sample_submission), Config.num_classes), 1.0 / Config.num_classes)\n    logger.info(\"Replaced test_preds with uniform probabilities to match sample_submission\")\n\nif np.any(np.isnan(test_preds)) or np.any(np.isinf(test_preds)):\n    logger.warning(\"NaN/Inf in test_preds, replacing with uniform probabilities\")\n    test_preds = np.full((len(sample_submission), Config.num_classes), 1.0 / Config.num_classes)\n\nsample_submission.iloc[:, 1:] = test_preds\nsample_submission.to_csv(\"submission.csv\", index=False)\nlogger.info(\"Created submission.csv\")\n\n# Cleanup\nshutil.rmtree(Config.feature_dir, ignore_errors=True)\nshutil.rmtree(Config.model_weights_dir, ignore_errors=True)\nlogger.info(f\"Final disk usage: {get_disk_usage():.2f} GiB\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}