{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"},{"sourceId":243997534,"sourceType":"kernelVersion"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport pandas.api.types\n\nimport sklearn.metrics\n\n\nclass ParticipantVisibleError(Exception):\n    pass\n\n\ndef score(solution: pd.DataFrame, submission: pd.DataFrame, row_id_column_name: str) -> float:\n    '''\n    Version of macro-averaged ROC-AUC score that ignores all classes that have no true positive labels.\n    '''\n    del solution[row_id_column_name]\n    del submission[row_id_column_name]\n\n    if not pandas.api.types.is_numeric_dtype(submission.values):\n        bad_dtypes = {x: submission[x].dtype  for x in submission.columns if not pandas.api.types.is_numeric_dtype(submission[x])}\n        raise ParticipantVisibleError(f'Invalid submission data types found: {bad_dtypes}')\n\n    solution_sums = solution.sum(axis=0)\n    scored_columns = list(solution_sums[solution_sums > 0].index.values)\n    assert len(scored_columns) > 0\n\n    return kaggle_metric_utilities.safe_call_score(sklearn.metrics.roc_auc_score, solution[scored_columns].values, submission[scored_columns].values, average='macro')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nimport librosa\nimport numpy as np\nimport seaborn as sns\n\nannotation_path = '/kaggle/input/birdclef-2025/train.csv'\ntrain_df = pd.read_csv('/kaggle/input/birdclef-2025/train.csv')\ntrain_soundscape_dir = '/kaggle/input/birdclef-2025/train_soundscapes/'\ntrain_audio_dir = '/kaggle/input/birdclef-2025/train_audio/'\ntest_soundscape_dir = '/kaggle/input/birdclef-2025/test_soundscapes/'","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Exploratory Data Analysis","metadata":{}},{"cell_type":"code","source":"train_df.head(5)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Number of unique labels is: \", end=\"\")\nprint(len(train_df['primary_label'].unique()))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.info()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"count_unique_by_rating = train_df.groupby('rating')['primary_label'].nunique()\nplt.title(\"Count of unique primary_labels by rating\")\nplt.bar(count_unique_by_rating.index, count_unique_by_rating.values, width=0.2)\nplt.xlabel('Rating')\nplt.ylabel('Count of unique primary_label')\nplt.show()\ncount_unique_by_rating","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.set_option('display.max_colwidth', None)\npivot_table = pd.crosstab(train_df['common_name'], train_df['rating'])\npivot_table","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.groupby('common_name').size().sort_values(ascending=True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Transformation (Audio -> Images)","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset\nfrom torchvision import datasets\nfrom torchaudio.transforms import MelSpectrogram\nfrom torchaudio.transforms import Spectrogram\nfrom torchaudio import load\nimport torchaudio\nimport torch.nn as nn\nimport torchaudio.transforms as T\nfrom torch.utils.data import DataLoader\nimport torchvision.models as models\nimport torch.nn.functional as F\nfrom tqdm import tqdm","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def WindowingMelSpec(mel_spec, mel_spec_freq, win_length:int=100, hop_length:int=50, n_windows_limit:int=600):\n    \"\"\"    \n        Windowing spectrogram with 50% overlap.\n        Window length = sample_rate * seconds\n        Overlap 50%\n        Max number of window should be 500 windows which should cover the given setting for 5 minutes audio\n    \n        Default setting: 10ms = 1 mel_spec frame = 10ms * 32_000 samples so if we want 1 second window frame with 0.5 overlap\n        we should do (1000ms / 10ms) = 100 frames with Hop = (100 * 0.5) = 50\n        5 min = 300_000 ms so max_windows = 300_000ms / hop_duration = 300_000 / (50 * 10) = 600 windows\n    \"\"\"\n    # Input tensor mel_spec = (mel_spec_freq, num_of_frames)\n    # Output tensor\n    windows = []\n    num_frames = mel_spec.shape[-1]\n    mel_spec = mel_spec.squeeze()\n\n    for start in range(0, num_frames - win_length + 1, hop_length):\n        end = start + win_length\n        window = mel_spec[:, start:end]\n        windows.append(window)\n\n    n_windows = min(len(windows), n_windows_limit)\n\n    sample_windows = torch.zeros((n_windows_limit, mel_spec_freq, win_length))\n    for i in range(n_windows):\n        sample_windows[i] = windows[i]\n\n    return sample_windows, n_windows\n\nclass WindowingModule(nn.Module):\n    def __init__(self, mel_spec_freq=128, win_length=100, hop_length=50, n_windows_limit=600):\n        super().__init__()\n        self.mel_spec_freq = mel_spec_freq\n        self.win_length = win_length\n        self.hop_length = hop_length\n        self.n_windows_limit = n_windows_limit\n\n    def forward(self, mel_spec):\n        windows, n_windows = WindowingMelSpec(\n            mel_spec,\n            mel_spec_freq=self.mel_spec_freq,\n            win_length=self.win_length,\n            hop_length=self.hop_length,\n            n_windows_limit=self.n_windows_limit\n        )\n        return windows, n_windows\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SAMPLE_RATE = 32_000\nMEL_SPEC_WINDOW = SAMPLE_RATE * 0.025\nMEL_SPEC_HOP = SAMPLE_RATE * 0.01\nN_MELS = 128\nNFFT = 1024\n\n\n    \nmel_transform = nn.Sequential(\n    T.Preemphasis(),\n    T.MelSpectrogram(\n        sample_rate = SAMPLE_RATE,\n        n_fft=NFFT,\n        win_length=int(MEL_SPEC_WINDOW),\n        hop_length=int(MEL_SPEC_HOP),\n        n_mels=N_MELS\n    ),\n    T.AmplitudeToDB(stype='power'),\n    WindowingModule(mel_spec_freq=N_MELS)\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class SoundscapeData(Dataset):\n    def __init__(self, audio_dir, transform=None, transform_target=None):\n        self.audio_dir = audio_dir\n        self.audio_paths = os.listdir(audio_dir)\n        self.transform = transform\n\n    def __getitem__(self, idx):\n        \"\"\"\n            Return windowed mel spectrograms and the number of valid windows\n        \"\"\"\n        audio, rate = torchaudio.load(self.audio_dir + self.audio_paths[idx])\n\n        if self.transform:\n            audio, n_windows = self.transform(audio)\n            return audio, n_windows\n\n        return audio\n        \n    def __len__(self):\n        return len(self.audio_dir)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport torch\nfrom torch.utils.data import Dataset\n\nclass LabeledData(Dataset):\n    def __init__(self, annotation_df, audio_dir, transform=None, transform_target=None):\n        self.annotation_df = annotation_df\n        self.transform = transform\n        self.audio_dir = audio_dir\n\n        self.unique_labels = sorted(annotation_df['primary_label'].unique())\n        self.label_to_id = {label: idx for idx, label in enumerate(self.unique_labels)}\n        self.transform_target = transform_target\n\n    def __getitem__(self, idx):\n        audio_path = os.path.join(self.audio_dir, self.annotation_df.iloc[idx]['filename'])\n        audio, rate = torchaudio.load(audio_path)\n\n        label_str = self.annotation_df.iloc[idx]['primary_label']\n        label_id = self.label_to_id[label_str]\n\n        if self.transform:\n            audio, n_windows = self.transform(audio)\n\n        if self.transform_target:\n            label_id = self.transform_target(label_id)\n\n        return audio, n_windows, label_id\n\n    def __len__(self):\n        return len(self.annotation_df)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mel_soundscape_dataset = SoundscapeData(train_soundscape_dir, transform=mel_transform)\nmel_train_dataset = LabeledData(train_df, train_audio_dir, transform=mel_transform)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(mel_train_dataset[100][0].shape)\nprint(mel_soundscape_dataset[100][0].shape)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Fine-tuning","metadata":{}},{"cell_type":"code","source":"def collate_fn(batch):\n    mel_specs = []\n    labels = []\n    for mel_spec, n_windows, label in batch:\n        mel_specs.append(mel_spec[:])\n        labels.append(label)\n\n    train = torch.stack(mel_specs)\n    return (train, labels)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torch.utils.data import WeightedRandomSampler\n\nlabels = train_df['primary_label'].values\n\nunique_labels = np.unique(labels)\nlabel_to_id = {label: idx for idx, label in enumerate(unique_labels)}\n\nint_labels = np.array([label_to_id[label] for label in labels])\n\nclass_counts = np.bincount(int_labels)\nclass_weights = 1. / class_counts\nsample_weights = class_weights[int_labels]\n\nsampler = WeightedRandomSampler(weights=sample_weights, num_samples=len(sample_weights), replacement=True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mel_train_loader = DataLoader(mel_train_dataset, batch_size=50, collate_fn=collate_fn, sampler=sampler)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X, Y = next(iter(mel_train_loader))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"encoder = AudioEncoder()\nprojection_head = ProjectionHead(in_dim=512)\nmodel = SimCLR(encoder, projection_head)\nmodel.load_state_dict(torch.load('/kaggle/working/simclr_epoch_10.pt'))\n\nclass AudioClassifier(nn.Module):\n    def __init__(self, encoder, num_classes):\n        super().__init__()\n        self.encoder = encoder\n        self.num_classes = nn.Linear(encoder.out_dim, num_classes)\n\n    def forward(self, x):\n        h = self.encoder(x)\n        return self.classifier(h)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_losses = []\nepochs = 100\n\nfor epoch in epochs:\n    model.train()\n    \n    for step, data, label in enumerate(tqdm(mel_train_loader, desc=f\"Epoch {epoch + 1} batch size: {augmented_soundscape_pairs_loader.batch_size}\")):\n        ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(mel_train_dataset)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Training setupa\nloss = []\nepochs = 50","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Self-supervised learning with Simple Contrastive Learning","metadata":{}},{"cell_type":"code","source":"def collate_soundscape(batch):\n    mel_windows_list, n_windows_list = zip(*batch)\n\n    batch_tensor = torch.stack(mel_windows_list)\n    n_windows_tensor = torch.tensor(n_windows_list)\n\n\n    return batch_tensor, n_windows_tensor","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torchvision import models\n\nclass AudioEncoder(nn.Module):\n    def __init__(self):\n        super().__init__()\n        base_model = models.resnet18()\n        # Replace first layer to accept 1-channel audio (mel spectrogram)\n        base_model.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False)\n        self.encoder = nn.Sequential(*list(base_model.children())[:-1])  # Exclude FC layer\n        self.out_dim = base_model.fc.in_features  # 512\n\n    def forward(self, x):  # x: (batch, 1, 128, 100)\n        x = self.encoder(x)\n        return x.view(x.size(0), -1)  # (batch, 512)\n\nclass ProjectionHead(nn.Module):\n    def __init__(self, in_dim, hidden_dim=512, out_dim=128):\n        super().__init__()\n        self.proj = nn.Sequential(\n            nn.Linear(in_dim, hidden_dim),\n            nn.ReLU(),\n            nn.Linear(hidden_dim, out_dim)\n        )\n\n    def forward(self, h):\n        return self.proj(h)\n\nclass SimCLR(nn.Module):\n    def __init__(self, encoder, projection_head):\n        super().__init__()\n        self.encoder = encoder\n        self.projection_head = projection_head\n\n    def forward(self, x):\n        h = self.encoder(x)\n        z = self.projection_head(h)\n        return h, z","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import random\n\nclass LightAugmentAudioModule(nn.Module):\n    def __init__(self, augment_prob=1):\n        super().__init__()\n        self.augment_prob = augment_prob\n        \n    def forward(self, audio):\n        if not self.training or random.random() > self.augment_prob:\n            return audio\n            \n        original_shape = audio.shape\n        if audio.dim() == 1:\n            audio = audio.unsqueeze(0)\n        \n        # Only fast augmentations\n        # Add noise\n        if random.random() < 0.4:\n            noise = torch.randn_like(audio) * random.uniform(0.001, 0.005)\n            audio = audio + noise\n        \n        # Volume scaling\n        if random.random() < 0.5:\n            audio = audio * random.uniform(0.8, 1.2)\n        \n        # Time masking\n        if random.random() < 0.3:\n            seq_len = audio.shape[-1]\n            mask_length = int(seq_len * random.uniform(0.01, 0.1))\n            if mask_length > 0:\n                start_idx = random.randint(0, max(1, seq_len - mask_length))\n                audio[..., start_idx:start_idx + mask_length] = 0\n        \n        # Polarity flip\n        if random.random() < 0.1:\n            audio = -audio\n        \n        audio = torch.clamp(audio, -1.0, 1.0)\n        \n        if len(original_shape) == 1:\n            audio = audio.squeeze(0)\n            \n        return audio","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class AugmentedSoundscapePairData(Dataset):\n    def __init__(self, audio_dir, transform=None):\n        self.audio_dir = audio_dir\n        self.audio_paths = os.listdir(audio_dir)\n        self.mel_transform = mel_transform\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.audio_paths)\n        \n    def __getitem__(self, idx):\n        raw_audio, rate = torchaudio.load(self.audio_dir + self.audio_paths[idx])\n\n        augmented_mel_spec1, n_windows1 = self.transform(raw_audio)\n        augmented_mel_spec2, n_windows2 = self.transform(raw_audio)\n        \n        return augmented_mel_spec1, augmented_mel_spec2, n_windows1","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"augmented_mel_transform = nn.Sequential(\n    LightAugmentAudioModule(),\n    mel_transform\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"augmented_soundscape_pairs_data = AugmentedSoundscapePairData(train_soundscape_dir, transform=augmented_mel_transform)\n\n# batch_size * tuples([max_windows, mel_freq, win_size], [max_windows, mel_freq, win_size], valid_windows)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def simclr_collate_fn(batch):\n    view1, view2 = [], []\n\n    for mel1, mel2, valid in batch:\n        view1.append(mel1[:valid])\n        view2.append(mel2[:valid])\n\n    view1 = torch.cat(view1, dim=0).unsqueeze(1)  # (N, 1, 128, 100)\n    view2 = torch.cat(view2, dim=0).unsqueeze(1)\n    return view1.float(), view2.float()            # be sure they’re float32\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"augmented_soundscape_pairs_loader = DataLoader(augmented_soundscape_pairs_data, batch_size=1, shuffle=True, collate_fn=simclr_collate_fn)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class NTXentLoss(nn.Module):\n    def __init__(self, temperature=0.5):\n        super().__init__()\n        self.temperature = temperature\n\n    def forward(self, z1, z2):\n        z1 = F.normalize(z1, dim=1)\n        z2 = F.normalize(z2, dim=1)\n        N = z1.size(0)\n        z = torch.cat([z1, z2], dim=0)  # [2N, dim]\n\n        similarity_matrix = F.cosine_similarity(z.unsqueeze(1), z.unsqueeze(0), dim=2)  # [2N, 2N]\n\n        labels = torch.arange(N, device=z.device)\n        labels = torch.cat([labels + N, labels], dim=0)\n\n        # mask out self-similarities\n        mask = torch.eye(2 * N, device=z.device).bool()\n        similarity_matrix = similarity_matrix.masked_fill(mask, -9e15)\n\n        positives = similarity_matrix[torch.arange(2 * N), labels]\n        loss = -positives / self.temperature + torch.logsumexp(similarity_matrix / self.temperature, dim=1)\n        return loss.mean()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport torch\nfrom torch.optim.lr_scheduler import StepLR\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nencoder = AudioEncoder().to(device)\nprojection_head = ProjectionHead(in_dim=encoder.out_dim).to(device)\nmodel = SimCLR(encoder, projection_head).to(device)\n\ncriterion = NTXentLoss(temperature=0.5)\noptimizer = torch.optim.Adam(model.parameters(), lr=6e-4)\n\ncheckpoint_path = \"/kaggle/working/simclr_epoch_3.pt\"\ncurrent_epoch = 0\n\nif os.path.exists(checkpoint_path):\n    checkpoint = torch.load(checkpoint_path, map_location=device)\n    \n    if isinstance(checkpoint, dict) and 'model_state_dict' in checkpoint:\n        model.load_state_dict(checkpoint['model_state_dict'])\n        current_epoch = checkpoint['epoch']\n        print(\"Loaded model weights from checkpoint.\")\n    else:\n        model.load_state_dict(checkpoint)\n        print(\"Loaded model weights only (raw state_dict).\")\nelse:\n    print(\"No checkpoint found, starting from scratch.\")\n\nprint(f\"Current epoch: {current_epoch}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from datetime import datetime\n\nepochs = 20\ntrain_losses = []\n\nfor epoch in range(current_epoch, current_epoch + epochs):\n    model.train()\n    total_loss = 0\n    cum_loss = 0\n\n    for step, (view1, view2) in enumerate(tqdm(augmented_soundscape_pairs_loader, desc=f\"Epoch {epoch + 1}\")):\n        view1, view2 = view1.to(device), view2.to(device)\n\n        _, z1 = model(view1)\n        _, z2 = model(view2)\n\n        loss = criterion(z1, z2)\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        total_loss += loss.item()\n        cum_loss += loss.item()\n        if step % 1000 == 0:\n            show_loss = None\n            if step == 0:\n                show_loss = cum_loss / 1\n            else:\n                show_loss = cum_loss / step\n                \n            print(f\"[{datetime.now().strftime('%H:%M:%S')}] Epoch {epoch+1} Step {step} Cumulative Average Loss: {show_loss:.4f}\")\n\n    \n    torch.save({\n        'epoch': epoch + 1,\n        'model_state_dict': model.state_dict(),\n        'optimizer_state_dict': optimizer.state_dict(),\n    }, f\"/kaggle/working/simclr_epoch_{epoch+1}.pt\")\n    print(f\"Saved checkpoint at epoch {epoch+1}\")\n        \n    avg_loss = total_loss / len(augmented_soundscape_pairs_loader)\n    train_losses.append(avg_loss)\n    print(f\"Epoch {epoch+1} - Avg Loss: {avg_loss:.4f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"torch.save({\n    'model_state_dict': model.state_dict(),\n    'optimizer_state_dict': optimizer.state_dict(),\n}, \"/kaggle/working/simclr_audio.pth\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.plot(range(1, 4 + 1), train_losses, marker='o')\nplt.title(\"Training Loss per Epoch\")\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Loss\")\nplt.grid(True)\nplt.show()","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Fine-tuning the SimCLR model with labeled data","metadata":{}}]}