{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"},{"sourceId":11006697,"sourceType":"datasetVersion","datasetId":6852193},{"sourceId":167220511,"sourceType":"kernelVersion"},{"sourceId":189366517,"sourceType":"kernelVersion"},{"sourceId":227215575,"sourceType":"kernelVersion"}],"dockerImageVersionId":30919,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Training notebook ResNet","metadata":{}},{"cell_type":"markdown","source":"* Inference Notebook [here.](https://www.kaggle.com/code/loekie/inference-notebook-event-detection)\n* Dataset Creation [here](https://www.kaggle.com/code/myso1987/birdclef2025-1-crop-audio-5s)","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/usr/lib/kaggle_metric_utilities')\nsys.path.append('/kaggle/usr/lib/birdclef-roc-auc')\nimport os\nimport random\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import tqdm\n\nimport torch\nimport torch.optim as optim\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, Dataset\nimport torchaudio\nimport torchaudio.transforms as AT\nfrom torchvision import models\n\nfrom metric import score\nimport csv\nimport os","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T10:35:40.785825Z","iopub.execute_input":"2025-06-02T10:35:40.786072Z","iopub.status.idle":"2025-06-02T10:35:49.428677Z","shell.execute_reply.started":"2025-06-02T10:35:40.786050Z","shell.execute_reply":"2025-06-02T10:35:49.427729Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Configurations","metadata":{}},{"cell_type":"code","source":"root_path = \"../input/birdclef-2025/\" \ninput_path = '/kaggle/input/birdclef-2025-train-data/train_raw5/'\n\nclass_labels = sorted(os.listdir('../input/birdclef-2025/train_audio/'))\n\ntrain_meta = pd.read_csv(root_path + 'train.csv')\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T10:35:49.438464Z","iopub.execute_input":"2025-06-02T10:35:49.438649Z","iopub.status.idle":"2025-06-02T10:35:49.677170Z","shell.execute_reply.started":"2025-06-02T10:35:49.438631Z","shell.execute_reply":"2025-06-02T10:35:49.676265Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"wav_sec = 5\nsample_rate = 32000\nmin_segment = sample_rate*wav_sec\n\nepochs = 10","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T10:35:49.678114Z","iopub.execute_input":"2025-06-02T10:35:49.678424Z","iopub.status.idle":"2025-06-02T10:35:49.681665Z","shell.execute_reply.started":"2025-06-02T10:35:49.678400Z","shell.execute_reply":"2025-06-02T10:35:49.681026Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Different mel spectrogram configurations for creating different models\nmel_configs = [\n    {\"n_fft\": 1536, \"hop_length\": 512,  \"n_mels\": 64, \"f_max\": 16000},\n    {\"n_fft\": 2048, \"hop_length\": 512,  \"n_mels\": 64, \"f_max\": 16000},\n    {\"n_fft\": 1536, \"hop_length\": 512,  \"n_mels\": 136, \"f_max\": 16000},\n    {\"n_fft\": 1280, \"hop_length\": 512,  \"n_mels\": 64, \"f_max\": 16000},\n    {\"n_fft\": 2048, \"hop_length\": 512,  \"n_mels\": 128, \"f_max\": 16000},    \n    ]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T10:35:49.682509Z","iopub.execute_input":"2025-06-02T10:35:49.682801Z","iopub.status.idle":"2025-06-02T10:35:49.700370Z","shell.execute_reply.started":"2025-06-02T10:35:49.682767Z","shell.execute_reply":"2025-06-02T10:35:49.699758Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Evaluation and logging functions","metadata":{}},{"cell_type":"code","source":"# No change compared to original\ndef cal_score(label, pred):\n    label = np.concatenate(label)\n    pred = np.concatenate(pred)\n\n    label_df = pd.DataFrame(label>0.5, columns=class_labels)\n    pred_df = pd.DataFrame(pred, columns=class_labels)\n    label_df['id'] = np.arange(len(label_df))\n    pred_df['id'] = np.arange(len(pred_df))\n\n    return score(label_df, pred_df, row_id_column_name='id')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T10:35:49.701058Z","iopub.execute_input":"2025-06-02T10:35:49.701251Z","iopub.status.idle":"2025-06-02T10:35:49.715971Z","shell.execute_reply.started":"2025-06-02T10:35:49.701234Z","shell.execute_reply":"2025-06-02T10:35:49.715127Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# We made this function for easy comparison between models\ndef log_metrics_to_csv(csv_path, name, epoch, auc, auc_val, loss, loss_val):\n    # Boolean if file exists\n    file_exists = os.path.isfile(csv_path)\n\n    # Create writer\n    with open(csv_path, mode='a', newline='') as file:\n        writer = csv.writer(file)\n\n        # If file does not exist, write headers\n        if not file_exists:\n            writer.writerow(['name', 'epoch', 'auc', 'auc_val', 'loss', 'loss_val'])\n        \n        writer.writerow([name, epoch, auc, auc_val, loss, loss_val])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# No change compared to original\nclass BirdclefDataset(Dataset):\n    def __init__(self, df, mel_transform, mode='train'):\n        self.df = df\n        self.mode = mode\n        self.mel_transform = mel_transform\n\n    def normalize_std(self, spec, eps=1e-23):\n        mean = torch.mean(spec)\n        std = torch.std(spec)\n        return torch.where(std == 0, spec-mean, (spec - mean) / (std+eps))\n                \n    def __getitem__(self, index):\n        sig, _ = torchaudio.load(uri=input_path+self.df.iloc[index].filename,backend=\"soundfile\")\n        sig = sig / torch.max(torch.abs(sig))\n        sig = sig + 1.5849e-05*(torch.rand(1, min_segment)-0.5) \n        melspec = self.mel_transform(sig)\n        melspec = torch.log(melspec + 1e-6)\n        melspec = self.normalize_std(melspec)\n\n        target = self.df.iloc[index].primary_label\n        y = np.array([1 if item == target else 0 for item in class_labels])\n        \n        return melspec, y\n    \n    def __len__(self):\n        return len(self.df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T10:35:49.718013Z","iopub.execute_input":"2025-06-02T10:35:49.718240Z","iopub.status.idle":"2025-06-02T10:35:49.735269Z","shell.execute_reply.started":"2025-06-02T10:35:49.718221Z","shell.execute_reply":"2025-06-02T10:35:49.734518Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# No change compared to original\nclass Model_resnet34(nn.Module):\n    def __init__(self, pretrained=False):\n        super().__init__()\n        model = models.resnet34(pretrained=pretrained)\n        num_ftrs = model.fc.in_features\n        model.fc = nn.Linear(num_ftrs, len(class_labels))\n        self.model = model\n\n    def forward(self, x):\n        x = torch.cat((x,x,x),1)\n        x = self.model(x)\n        return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-17T22:24:05.929356Z","iopub.execute_input":"2025-05-17T22:24:05.929709Z","iopub.status.idle":"2025-05-17T22:24:05.934576Z","shell.execute_reply.started":"2025-05-17T22:24:05.929678Z","shell.execute_reply":"2025-05-17T22:24:05.933538Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training loop","metadata":{}},{"cell_type":"markdown","source":"We made the most changes to the training loop. In the original notebook by MYSO, the training is done on one configuration: the configuration is specified in the beginning of the notebook. ","metadata":{}},{"cell_type":"code","source":"# Train test split\ntrain_df, val_df = train_test_split(train_meta, test_size=0.2, random_state=42)\n\nfor i, cfg in enumerate(mel_configs):\n    print(f\"\\nTraining model {i+1}/{len(mel_configs)} with config: {cfg}\")\n\n    # For every set of configurations, make a melspectrogram object\n    mel_transform = AT.MelSpectrogram(\n        sample_rate=sample_rate,\n        n_fft=cfg[\"n_fft\"],\n        win_length=cfg[\"n_fft\"],  # assuming win_length = n_fft\n        hop_length=cfg[\"hop_length\"],\n        center=True,\n        f_min=20,\n        f_max=cfg[\"f_max\"],\n        pad_mode=\"reflect\",\n        power=2.0,\n        norm='slaney',\n        n_mels=cfg[\"n_mels\"],\n        mel_scale=\"htk\",\n    )\n\n    # Make dataset\n    train_dataset = BirdclefDataset(train_df, mel_transform, mode='train')\n    val_dataset = BirdclefDataset(val_df, mel_transform, mode='val')\n\n    train_loader = DataLoader(train_dataset, batch_size=24, shuffle=True, num_workers=2, drop_last=True)\n    val_loader = DataLoader(val_dataset, batch_size=24, shuffle=False, num_workers=1, drop_last=True)\n   \n    # Model and optimizer\n    model = Model_resnet34(pretrained=True).to(device)\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(), lr=0.001)\n    \n    # Training loop\n    for epoch in range(epochs):\n        model.train()\n        pred_train = []\n        label_train = []\n        running_loss = 0.0\n        for melspecs, labels in tqdm(train_loader):\n            melspecs, labels = melspecs.to(device), labels.to(device)\n            optimizer.zero_grad()\n            outputs = model(melspecs)\n            loss = criterion(outputs, labels.to(torch.float32))\n            loss.backward()\n            optimizer.step()\n            running_loss += loss.item()\n            pred_train.append(torch.softmax(outputs,dim=1).detach().cpu().numpy())\n            label_train.append(labels.detach().cpu().numpy())\n    \n        pred_val = []\n        label_val = []\n        running_loss_val = 0.0\n        model.eval()\n        with torch.no_grad():\n            for melspecs, labels in val_loader:\n                melspecs, labels = melspecs.to(device), labels.to(device)\n                outputs = model(melspecs)\n                loss = criterion(outputs, labels.to(torch.float32))\n                running_loss_val += loss.item()\n                pred_val.append(torch.softmax(outputs,dim=1).detach().cpu().numpy())\n                label_val.append(labels.detach().cpu().numpy())\n\n        csv_path = '/kaggle/working/results.csv'\n\n        # Print scores\n        auc_train_val = cal_score(label_train, pred_train)\n        auc_val = cal_score(label_val, pred_val)\n        print(f\"Epoch {epoch+1}/{epochs}, Loss: {running_loss/len(train_loader):.4f}, Loss_val: {running_loss_val/len(train_loader):.4f}\")\n        print(f\"Auc: {auc_train_val:.2f}% Auc_val: {auc_val:.2f}%\")\n    \n        # Log the scores, along with the model name\n        log_metrics_to_csv(\n            csv_path=csv_path,\n            name=f\"cfg_{i}_fft{cfg['n_fft']}_hop{cfg['hop_length']}_mels{cfg['n_mels']}\",\n            epoch=epoch + 1,\n            auc=auc_train_val,\n            auc_val=auc_val,\n            loss=running_loss / len(train_loader),\n            loss_val=running_loss_val / len(val_loader)\n        )\n\n    # Save model\n    torch.save(model.state_dict(), f\"model_cfg_{i}.pth\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-17T22:24:06.801904Z","iopub.execute_input":"2025-05-17T22:24:06.802195Z","iopub.status.idle":"2025-05-17T22:34:03.922082Z","shell.execute_reply.started":"2025-05-17T22:24:06.802172Z","shell.execute_reply":"2025-05-17T22:34:03.920607Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}