{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"},{"sourceId":12060471,"sourceType":"datasetVersion","datasetId":7591012}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-04T20:27:48.260237Z","iopub.execute_input":"2025-06-04T20:27:48.260777Z","iopub.status.idle":"2025-06-04T20:28:29.502820Z","shell.execute_reply.started":"2025-06-04T20:27:48.260747Z","shell.execute_reply":"2025-06-04T20:28:29.501608Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import librosa\nimport torch\nimport torchaudio\nimport matplotlib.pyplot as plt\nimport ast\nimport numpy as np\nimport timm\nimport shutil\nimport random\n\nfrom torch import nn\nfrom torchvision.ops import sigmoid_focal_loss\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.preprocessing import MultiLabelBinarizer\nfrom sklearn.metrics import roc_auc_score\nfrom tqdm.notebook import tqdm\nfrom pathlib import Path\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T20:28:29.507168Z","iopub.execute_input":"2025-06-04T20:28:29.507421Z","iopub.status.idle":"2025-06-04T20:28:45.263008Z","shell.execute_reply.started":"2025-06-04T20:28:29.507401Z","shell.execute_reply":"2025-06-04T20:28:45.262173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Constants\nSAMPLE_RATE = 32000\nDURATION = 5  # seconds\nN_MELS = 128\nHOP_LENGTH = 512\nN_FFT = 2048\nNUM_CLASSES = 206\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T20:28:45.263856Z","iopub.execute_input":"2025-06-04T20:28:45.264394Z","iopub.status.idle":"2025-06-04T20:28:45.273111Z","shell.execute_reply.started":"2025-06-04T20:28:45.264334Z","shell.execute_reply":"2025-06-04T20:28:45.272165Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_audio(file_path, duration=DURATION, sr=SAMPLE_RATE):\n    y, _ = librosa.load(file_path, sr=sr, mono=True)\n    if len(y) < sr * duration:\n        y = np.pad(y, (0, sr * duration - len(y)))\n    else:\n        y = y[:sr * duration]\n    return y\n\ndef audio_to_logmel(y, sr=SAMPLE_RATE):\n    mel = librosa.feature.melspectrogram(y=y, sr=sr, n_fft=N_FFT, hop_length=HOP_LENGTH, n_mels=N_MELS)\n    logmel = librosa.power_to_db(mel)\n   \n    logmel = (logmel - logmel.mean(axis=1, keepdims=True)) / (logmel.std(axis=1, keepdims=True) + 1e-6)\n    return logmel\n\ndef spec_augment(mel_spectrogram, time_mask_param=40, freq_mask_param=15, num_masks=1):\n    \"\"\"\n    Apply SpecAugment-style time and frequency masking to a mel-spectrogram.\n    Input:\n        mel_spectrogram: np.array of shape (n_mels, time_steps)\n    Output:\n        Augmented spectrogram (same shape)\n    \"\"\"\n    augmented = mel_spectrogram.copy()\n    n_mels, time_steps = augmented.shape\n\n    # Frequency masking\n    for _ in range(num_masks):\n        f = random.randint(0, freq_mask_param)\n        f0 = random.randint(0, max(1, n_mels - f))\n        augmented[f0:f0+f, :] = 0.0\n\n    # Time masking\n    for _ in range(num_masks):\n        t = random.randint(0, time_mask_param)\n        t0 = random.randint(0, max(1, time_steps - t))\n        augmented[:, t0:t0+t] = 0.0\n\n    return augmented\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T20:28:45.274176Z","iopub.execute_input":"2025-06-04T20:28:45.274533Z","iopub.status.idle":"2025-06-04T20:28:45.308967Z","shell.execute_reply.started":"2025-06-04T20:28:45.274502Z","shell.execute_reply":"2025-06-04T20:28:45.308164Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BirdDataset(Dataset):\n    def __init__(self, df, mlb, augment=False, mixup=False, noise_std=0.0):\n        self.df = df.reset_index(drop=True)\n        self.mlb = mlb\n        self.augment = augment\n        self.mixup = mixup \n        self.noise_std = noise_std\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n\n        # Select correct path depending on source\n        if row['source'] == 'train':\n            file_path = os.path.join(AUDIO_PATH_LABELED, row['filename'])\n        else:\n            file_path = os.path.join(AUDIO_PATH_PSEUDO, row['filename'])\n\n        y = load_audio(file_path)\n        melspec = audio_to_logmel(y)  # (128, T)\n\n        # Normalize before augmentation\n        melspec = (melspec - melspec.mean(axis=1, keepdims=True)) / (melspec.std(axis=1, keepdims=True) + 1e-6)\n\n        # Apply SpecAugment if enabled\n        if self.augment:\n            melspec = spec_augment(melspec)\n\n        melspec = torch.tensor(melspec).unsqueeze(0).float()  # (1, 128, T)\n\n        # Build label vector\n        label_list = row['labels']\n        label_vec = np.zeros(len(self.mlb.classes_), dtype=np.float32)\n\n        if row.get(\"source\", \"train\") == \"pseudo\":\n            # Pseudo-labeled: uniform weight summing to 0.5\n            weight = 0.5 / len(label_list)\n            for lbl in label_list:\n                label_vec[self.mlb.classes_.tolist().index(lbl)] = weight\n        elif len(label_list) == 1:\n            # Single label: full weight\n            label_vec[self.mlb.classes_.tolist().index(label_list[0])] = 1.0\n        else:\n            # Primary: 0.5, others: shared 0.5\n            primary = label_list[0]\n            secondaries = label_list[1:]\n            label_vec[self.mlb.classes_.tolist().index(primary)] = 0.5\n            for lbl in secondaries:\n                label_vec[self.mlb.classes_.tolist().index(lbl)] = 0.5 / len(secondaries)\n        # Adding gaussian Noice        \n        if self.noise_std > 0:\n            melspec += np.random.normal(0, self.noise_std, size=melspec.shape)\n\n        # MIXUP LOGIC\n        if self.mixup and random.random() < 0.5:\n            # Sample a second example randomly\n            idx2 = random.randint(0, len(self.df) - 1)\n            row2 = self.df.iloc[idx2]\n            if row2['source'] == 'train':\n                file_path2 = os.path.join(AUDIO_PATH_LABELED, row2['filename'])\n            else:\n                file_path2 = os.path.join(AUDIO_PATH_PSEUDO, row2['filename'])\n\n            y2 = load_audio(file_path2)\n            mel2 = audio_to_logmel(y2)\n            mel2 = (mel2 - mel2.mean(axis=1, keepdims=True)) / (mel2.std(axis=1, keepdims=True) + 1e-6)\n\n            if self.augment:\n                mel2 = spec_augment(mel2)\n\n            # Build label for second sample\n            label_vec2 = np.zeros(len(self.mlb.classes_), dtype=np.float32)\n            label_list2 = row2['labels']\n            if row2.get(\"source\", \"train\") == \"pseudo\":\n                weight = 0.5 / len(label_list2)\n                for lbl in label_list2:\n                    label_vec2[self.mlb.classes_.tolist().index(lbl)] = weight\n            elif len(label_list2) == 1:\n                label_vec2[self.mlb.classes_.tolist().index(label_list2[0])] = 1.0\n            else:\n                primary = label_list2[0]\n                secondaries = label_list2[1:]\n                label_vec2[self.mlb.classes_.tolist().index(primary)] = 0.5\n                for lbl in secondaries:\n                    label_vec2[self.mlb.classes_.tolist().index(lbl)] = 0.5 / len(secondaries)\n\n            # Mix the two examples\n            lam = np.random.beta(0.4, 0.4)  # Mixup strength\n            melspec = lam * melspec + (1 - lam) * mel2\n            label_vec = lam * label_vec + (1 - lam) * label_vec2\n\n        # Convert to tensor\n        # Ensure melspec is shape [1, 128, T] before passing to model\n        if melspec.ndim == 2:\n            melspec = torch.tensor(melspec).unsqueeze(0).float()\n        else:\n            melspec = torch.tensor(melspec).float()\n\n        label_vec = torch.tensor(label_vec).float()\n        return melspec, label_vec\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class EfficientNetFrozen(nn.Module):\n    \"\"\"\n    EfficientNet‐B0 backbone where:\n      - All layers up through blocks.5 are frozen.\n      - Only backbone.blocks.6 and backbone.bn2 are trainable.\n      - The classifier head is trainable.\n    \"\"\"\n    def __init__(self, model_name=\"efficientnet_b0\", n_classes=206, unfreeze_blocks=[\"blocks.6\"]):\n        super().__init__()\n        # 1) Load pretrained EfficientNet‐B0, no head:\n        self.backbone = timm.create_model(\n            model_name,\n            pretrained=True,\n            in_chans=1,\n            num_classes=0\n        )\n        # 2) First, freeze everything:\n        for param in self.backbone.parameters():\n            param.requires_grad = False\n\n        # 3) Unfreeze only the specified blocks (e.g. \"blocks.6\") and final batchnorm (\"bn2\"):\n        #    If you want to unfreeze more, add them by name in unfreeze_blocks.\n        for name, param in self.backbone.named_parameters():\n            # “blocks.6.” is the final MBConv block in B0; “bn2” is the last batchnorm\n            if any([name.startswith(block_name) for block_name in unfreeze_blocks]) \\\n               or name.startswith(\"bn2\"):\n                param.requires_grad = True\n\n        # 4) Build a new classifier head on top of pooled features:\n        num_features = self.backbone.num_features  # should be 1280 for B0\n        self.classifier = nn.Linear(num_features, n_classes)\n        # Ensure classifier is trainable:\n        for param in self.classifier.parameters():\n            param.requires_grad = True\n\n    def forward(self, x):\n        feats = self.backbone(x)     # → (B, 1280)\n        logits = self.classifier(feats)  # → (B, 206)\n        return logits\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T20:28:45.341484Z","iopub.execute_input":"2025-06-04T20:28:45.341804Z","iopub.status.idle":"2025-06-04T20:28:45.363781Z","shell.execute_reply.started":"2025-06-04T20:28:45.341777Z","shell.execute_reply":"2025-06-04T20:28:45.362993Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def criterion(logits, targets):\n    return sigmoid_focal_loss(inputs=logits, targets=targets, alpha=0.25, gamma=2.0, reduction=\"mean\")\n\ndef train_model(model, train_dl, val_dl, optimizer, criterion, num_epochs=10, patience=3, fold=0):\n    model.to(DEVICE)\n\n    best_auc = 0\n    epochs_since_improvement = 0\n    best_model_state = None\n\n    # sanity check: Print trainable layers (debugging)\n    print(\"✅ Trainable parameters:\")\n    for name, param in model.named_parameters():\n        if param.requires_grad:\n            print(f\"  {name}\")\n\n    for epoch in range(num_epochs):\n        model.train()\n        train_loss = 0.0\n        for xb, yb in tqdm(train_dl, desc=f\"Epoch {epoch+1} - Training\"):\n            xb, yb = xb.to(DEVICE), yb.to(DEVICE)\n            optimizer.zero_grad()\n            outputs = model(xb)\n            loss = criterion(outputs, yb)\n            loss.backward()\n            optimizer.step()\n            train_loss += loss.item()\n\n        model.eval()\n        val_loss = 0.0\n        all_preds = []\n        all_targets = []\n        with torch.no_grad():\n            for xb, yb in tqdm(val_dl, desc=f\"Epoch {epoch+1} - Validation\"):\n                xb, yb = xb.to(DEVICE), yb.to(DEVICE).float()\n                outputs = model(xb)\n                loss = criterion(outputs, yb)\n                val_loss += loss.item()\n                all_preds.append(torch.sigmoid(outputs).cpu().numpy())\n                all_targets.append(yb.cpu().numpy())\n\n        all_preds = np.vstack(all_preds)\n        all_targets = np.vstack(all_targets)\n\n        aucs = []\n        binarized_targets = (all_targets >= 0.5).astype(int)\n\n        for i in range(binarized_targets.shape[1]):\n            if np.sum(binarized_targets[:, i]) > 0:\n                try:\n                    auc = roc_auc_score(binarized_targets[:, i], all_preds[:, i])\n                    aucs.append(auc)\n                except ValueError:\n                    continue\n\n        macro_auc = np.mean(aucs) if aucs else 0.0\n\n        print(f\"Epoch {epoch+1}: Train Loss = {train_loss/len(train_dl):.4f} | Val Loss = {val_loss/len(val_dl):.4f} | Val Macro ROC-AUC = {macro_auc:.4f}\")\n\n        # Early stopping logic\n        if macro_auc > best_auc:\n            best_auc = macro_auc\n            epochs_since_improvement = 0\n            best_model_path = f\"efficientnet_b0_frozen_fold{fold}_best.pth\"\n            torch.save(model.state_dict(), best_model_path)\n            best_model_state = model.state_dict()\n        else:\n            epochs_since_improvement += 1\n            if epochs_since_improvement >= patience:\n                print(f\"⏹️ Early stopping triggered after {epoch+1} epochs.\")\n                break\n\n    # Load best model state before returning\n    if best_model_state:\n        model.load_state_dict(best_model_state)\n\n    return best_auc","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T20:28:45.364618Z","iopub.execute_input":"2025-06-04T20:28:45.364842Z","iopub.status.idle":"2025-06-04T20:28:45.388214Z","shell.execute_reply.started":"2025-06-04T20:28:45.364825Z","shell.execute_reply":"2025-06-04T20:28:45.387417Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# === Combine Primary + Secondary Labels ===\ndef combine_labels(row):\n    if pd.isna(row['secondary_labels']):\n        secondary = []\n    else:\n        secondary = ast.literal_eval(row['secondary_labels'])\n        # remove empty strings from parsed list\n        secondary = [s for s in secondary if s.strip() != '']\n    return [row['primary_label']] + secondary\n\n# ===Convert one-hot encoded columns to label indices ===\ndef extract_labels(row):\n    return [f\"class_{i}\" for i, v in enumerate(row[1:].values) if v > 0.9]\n\n# === Setup Paths ===\nDATA_PATH = \"/kaggle/input/birdclef-2025\"\nAUDIO_PATH_LABELED = \"/kaggle/input/birdclef-2025/train_audio\"\nAUDIO_PATH_PSEUDO  = \"/kaggle/input/birdclef-2025/train_soundscapes\"\n\n\n# === Load Data ===\ndf = pd.read_csv(os.path.join(DATA_PATH, \"train.csv\"))\n\ndf['labels'] = df.apply(combine_labels, axis=1)\ndf['source'] = 'train'\n\npseudo_df = pd.read_csv(\"/kaggle/input/pseudo-labels-new/pseudo_labels.csv\")\n\npseudo_df['labels'] = pseudo_df.apply(extract_labels, axis=1)\npseudo_df['source'] = 'pseudo'\n\n# Add dummy filename column to satisfy dataset requirements\npseudo_df['filename'] = pseudo_df['row_id'].apply(lambda x: x.split('_')[0] + '.ogg')\n\n# Retain only columns needed by BirdDataset\npseudo_df = pseudo_df[['filename', 'labels', 'source']]\n\n# Match schema with df\ndf = pd.concat([df, pseudo_df], ignore_index=True)\n\n# Collect labeled files in format: '1139490/CSA36385.ogg'\nlabeled_files = set()\nfor species_folder in os.listdir(AUDIO_PATH_LABELED):\n    species_path = os.path.join(AUDIO_PATH_LABELED, species_folder)\n    if os.path.isdir(species_path):\n        for file in os.listdir(species_path):\n            labeled_files.add(f\"{species_folder}/{file}\")\n\npseudo_files = set(os.listdir(AUDIO_PATH_PSEUDO))\n\ndef is_valid_file(row):\n    if row['source'] == 'train':\n        return row['filename'] in labeled_files\n    else:\n        return row['filename'] in pseudo_files\n\ndf = df[df.apply(is_valid_file, axis=1)].reset_index(drop=True)\nprint(\"✅ Filtered df shape:\", df.shape)\nprint(\"📊 Remaining sources:\", df['source'].value_counts())\n\n# === Fit MultiLabelBinarizer ===\nmlb = MultiLabelBinarizer()\nmlb.fit(df['labels'])  # 🔥 Only once, on full label set\n\nNUM_CLASSES = len(mlb.classes_)\n\nNUM_FOLDS = 2\nskf = StratifiedKFold(n_splits=NUM_FOLDS, shuffle=True, random_state=42)\n\ndf_labeled = df[(df['source'] == 'train') & (df['primary_label'].notna())].copy()\nif df_labeled.empty:\n    raise ValueError(\"❌ No labeled data left after filtering. Check your audio folders and filenames.\")\n\n# Get all pseudo-labeled rows (added after split)\ndf_pseudo = df[df['source'] == 'pseudo'].copy()\nfold_aucs = []","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T20:28:45.389085Z","iopub.execute_input":"2025-06-04T20:28:45.389393Z","iopub.status.idle":"2025-06-04T20:28:46.641143Z","shell.execute_reply.started":"2025-06-04T20:28:45.389346Z","shell.execute_reply":"2025-06-04T20:28:46.640132Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fold_aucs = []\nbest_fold_index = -1\nbest_auc_across_folds = -1.0\n\nfor fold, (train_idx, val_idx) in enumerate(skf.split(df_labeled, df_labeled['primary_label'])):\n    print(f\"\\n===== Fold {fold + 1} / {NUM_FOLDS} =====\")\n\n    # Build fold splits from labeled data\n    train_df = df_labeled.iloc[train_idx].reset_index(drop=True)\n    val_df   = df_labeled.iloc[val_idx].reset_index(drop=True)\n\n    # Add pseudo-labeled data to training set\n    train_df = pd.concat([train_df, df_pseudo], ignore_index=True)\n\n    # Datasets and loaders\n    train_ds = BirdDataset(train_df, mlb, augment = False, mixup = True, noise_std = 0.1)\n    val_ds   = BirdDataset(val_df, mlb, augment = False, mixup = False, noise_std = 0.0)\n\n    train_dl = DataLoader(train_ds, batch_size=16, shuffle=True, num_workers=2)\n    val_dl   = DataLoader(val_ds, batch_size=16, shuffle=False, num_workers=2)\n\n    # Model and optimizer\n    model = EfficientNetFrozen(model_name='efficientnet_b0', n_classes=NUM_CLASSES, unfreeze_blocks=[\"blocks.5\", \"blocks.6\"]).to(DEVICE)\n\n    trainable_params = list(model.classifier.parameters()) + [\n        p for n, p in model.backbone.named_parameters() if p.requires_grad\n    ]\n    optimizer = torch.optim.Adam(trainable_params, lr=1e-4, weight_decay=1e-5)\n\n    # Train model with early stopping and save best checkpoint\n    fold_auc = train_model(\n        model, train_dl, val_dl,\n        optimizer=optimizer,\n        criterion=criterion,\n        num_epochs=10,\n        patience=3,\n        fold=fold\n    )\n\n    fold_aucs.append(fold_auc)\n\n    # Track the best model across folds\n    if fold_auc > best_auc_across_folds:\n        best_auc_across_folds = fold_auc\n        best_fold_index = fold\n\n# === After all folds complete ===\n\nprint(f\"\\n✅ Fold AUCs: {fold_aucs}\")\nprint(f\"📊 Average ROC-AUC across folds: {np.mean(fold_aucs):.4f}\")\n\n# Copy the best model to a general name\nsrc_path = f\"efficientnet_b0_frozen_fold{best_fold_index}_best.pth\"\ndst_path = \"efficientnet_b0_frozen_overall_best.pth\"\nshutil.copy(src_path, dst_path)\n\nprint(f\"🏆 Best model was from fold {best_fold_index} with AUC = {best_auc_across_folds:.4f}\")\nprint(f\"📦 Saved as: {dst_path}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T20:31:24.251061Z","iopub.execute_input":"2025-06-04T20:31:24.251354Z","iopub.status.idle":"2025-06-04T20:31:24.310549Z","shell.execute_reply.started":"2025-06-04T20:31:24.251331Z","shell.execute_reply":"2025-06-04T20:31:24.309533Z"}},"outputs":[],"execution_count":null}]}