{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":70203,"databundleVersionId":8068726,"sourceType":"competition"},{"sourceId":8120971,"sourceType":"datasetVersion","datasetId":4779991},{"sourceId":184195761,"sourceType":"kernelVersion"}],"dockerImageVersionId":30684,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import Packages","metadata":{}},{"cell_type":"code","source":"import re\nimport os\nimport gc\nimport sys\nimport cv2\nimport math\nimport numpy as np\nimport pandas as pd\nfrom glob import glob\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\nimport librosa\nfrom scipy import signal as sci_signal\n\nimport torch\nfrom torch import nn\nfrom torchvision.models import efficientnet\n\nimport albumentations as albu\n\nimport pytorch_lightning as pl\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts","metadata":{"execution":{"iopub.status.busy":"2024-06-19T00:47:26.453463Z","iopub.execute_input":"2024-06-19T00:47:26.453900Z","iopub.status.idle":"2024-06-19T00:47:26.462375Z","shell.execute_reply.started":"2024-06-19T00:47:26.453865Z","shell.execute_reply":"2024-06-19T00:47:26.461074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration","metadata":{}},{"cell_type":"code","source":"#配置参数\nclass config:\n    \n    #通用参数\n    SEED = 2024  #随机种子\n    DEVICE = 'cpu'  #竞赛要求必须用cpu\n    MIXED_PRECISION = False  #不使用混合精度训练，容易影响模型的泛化能力\n    OUTPUT_DIR = '/kaggle/working/'  #输出文件夹\n    \n    #数据参数\n    DATA_ROOT = '/kaggle/input/birdclef-2024'  #原始数据目录\n    PREPROCESSED_DATA_ROOT = '/kaggle/input/birdclef24-spectrograms-via-cupy'\n    LOAD_DATA = True  #使用预训练的数据，提高训练效率\n    FS = 32000  #采样率\n    N_FFT = 1095  #FFT点数\n    WIN_SIZE = 412  #频谱每段样本数量\n    WIN_LAP = 100  #频谱每段重叠样本数\n    MIN_FREQ = 40  #最小频率\n    MAX_FREQ = 15000  #最大频率\n    \n    #模型参数\n    MODEL_TYPE = 'efficientnet_b0'\n    \n    #数据集参数\n    BATCH_SIZE = 64\n    N_WORKERS = 4\n    \n    #预训练权重\n    CKPT_ROOT = '/kaggle/input/train'\n    \n    #其他参数\n    VISUALIZE = True  # whether to visualize data and batch\n    \nprint('fix seed')\npl.seed_everything(config.SEED, workers=True)","metadata":{"execution":{"iopub.status.busy":"2024-06-19T00:50:03.379766Z","iopub.execute_input":"2024-06-19T00:50:03.380190Z","iopub.status.idle":"2024-06-19T00:50:03.394115Z","shell.execute_reply.started":"2024-06-19T00:50:03.380155Z","shell.execute_reply":"2024-06-19T00:50:03.392973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# labels\nlabel_list = sorted(os.listdir(os.path.join(config.DATA_ROOT, 'train_audio')))\nlabel_id_list = list(range(len(label_list)))\nlabel2id = dict(zip(label_list, label_id_list))\nid2label = dict(zip(label_id_list, label_list))","metadata":{"execution":{"iopub.status.busy":"2024-06-19T00:48:00.126259Z","iopub.execute_input":"2024-06-19T00:48:00.126661Z","iopub.status.idle":"2024-06-19T00:48:00.133989Z","shell.execute_reply.started":"2024-06-19T00:48:00.126628Z","shell.execute_reply":"2024-06-19T00:48:00.132650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset & Dataloader","metadata":{}},{"cell_type":"markdown","source":"## Pre-Processing","metadata":{}},{"cell_type":"code","source":"#预处理\ndef oog2spec_via_scipy(audio_data):\n    #处理Nan数据\n    mean_signal = np.nanmean(audio_data)\n    audio_data = np.nan_to_num(audio_data, nan=mean_signal) if np.isnan(audio_data).mean() < 1 else np.zeros_like(audio_data)\n    \n    #频谱转换\n    frequencies, times, spec_data = sci_signal.spectrogram(\n        audio_data, \n        fs=config.FS, \n        nfft=config.N_FFT, \n        nperseg=config.WIN_SIZE, \n        noverlap=config.WIN_LAP, \n        window='hann'\n    )\n    \n    #滤波器频率范围\n    valid_freq = (frequencies >= config.MIN_FREQ) & (frequencies <= config.MAX_FREQ)\n    spec_data = spec_data[valid_freq, :]\n    \n    #对频谱图数据应用对数变换，以增强频率成分的对比度，并添加一个很小的常数1e-20来避免对数运算中的负无穷问题\n    spec_data = np.log10(spec_data + 1e-20)\n    \n    #归一化\n    spec_data = spec_data - spec_data.min()\n    spec_data = spec_data / spec_data.max()\n    \n    return spec_data","metadata":{"execution":{"iopub.status.busy":"2024-06-19T00:48:02.312991Z","iopub.execute_input":"2024-06-19T00:48:02.313392Z","iopub.status.idle":"2024-06-19T00:48:02.323513Z","shell.execute_reply.started":"2024-06-19T00:48:02.313361Z","shell.execute_reply":"2024-06-19T00:48:02.322025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#初始化\nall_bird_data = dict()\n#指定音频文件路径\nif len(glob(f'{config.DATA_ROOT}/test_soundscapes/*.ogg')) > 0:\n    ogg_file_paths = glob(f'{config.DATA_ROOT}/test_soundscapes/*.ogg')\nelse:\n    ogg_file_paths = sorted(glob(f'{config.DATA_ROOT}/unlabeled_soundscapes/*.ogg'))[:10]\n#批量将音频文件转换为频谱图\nfor i, file_path in tqdm(enumerate(ogg_file_paths)):\n    #使用正则表达式从文件路径中提取文件名\n    row_id = re.search(r'/([^/]+)\\.ogg$', file_path).group(1)  # filename\n    #加载音频文件\n    audio_data, _ = librosa.load(file_path, sr=config.FS)\n    \n    #转换为频谱图\n    spec = oog2spec_via_scipy(audio_data)\n    \n    #计算需要填充的列数，以确保频谱图的列数是512的倍数\n    pad = 512 - (spec.shape[1] % 512)\n    if pad > 0:\n        spec = np.pad(spec, ((0,0), (0,pad)))\n    \n    #把频谱图重塑成256x256像素\n    spec = spec.reshape(512,-1,512).transpose([0, 2, 1])\n    spec = cv2.resize(spec, (256, 256), interpolation=cv2.INTER_AREA)\n    #每次迭代都从调整大小的频谱图中提取一个5秒的片段，循环48次，覆盖240s（竞赛提到的4min）\n    for j in range(48):\n        all_bird_data[f'{row_id}_{(j+1)*5}'] = spec[:, :, j]","metadata":{"execution":{"iopub.status.busy":"2024-06-19T00:48:03.989895Z","iopub.execute_input":"2024-06-19T00:48:03.990301Z","iopub.status.idle":"2024-06-19T00:48:29.800960Z","shell.execute_reply.started":"2024-06-19T00:48:03.990268Z","shell.execute_reply":"2024-06-19T00:48:29.799770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"#数据集构建\nclass BirdDataset(torch.utils.data.Dataset):\n    \n    def __init__(\n        self,\n        bird_data,\n        augmentation=None,\n    ):\n        super().__init__()\n        #存储传入的音频数据\n        self.bird_data = bird_data\n        #存储bird_data的键的列表，用于后续索引\n        self.keys_list = list(bird_data.keys())\n        #存储传入的数据增强对象\n        self.augmentation = augmentation\n    \n    def __len__(self):\n        return len(self.bird_data)\n    \n    def __getitem__(self, index):\n        #根据索引从bird_data中获取对应的音频数据\n        _spec = self.bird_data[self.keys_list[index]]\n        #如果提供了数据增强，将其应用于获取的音频数据。这里假设数据增强对象有一个接受image关键字参数的函数，并返回增强后的图像\n        if self.augmentation is not None:\n            _spec = self.augmentation(image=_spec)['image'] \n        \n        return torch.tensor(_spec, dtype=torch.float32)","metadata":{"execution":{"iopub.status.busy":"2024-06-19T00:49:09.857871Z","iopub.execute_input":"2024-06-19T00:49:09.858536Z","iopub.status.idle":"2024-06-19T00:49:09.868035Z","shell.execute_reply.started":"2024-06-19T00:49:09.858499Z","shell.execute_reply":"2024-06-19T00:49:09.866778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Augmentation","metadata":{}},{"cell_type":"code","source":"#数据增强\ndef get_transforms(_type):\n    \n    if _type == 'test':\n        return albu.Compose([])","metadata":{"execution":{"iopub.status.busy":"2024-06-19T00:49:13.681809Z","iopub.execute_input":"2024-06-19T00:49:13.682260Z","iopub.status.idle":"2024-06-19T00:49:13.688030Z","shell.execute_reply.started":"2024-06-19T00:49:13.682219Z","shell.execute_reply":"2024-06-19T00:49:13.686520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Verify","metadata":{}},{"cell_type":"code","source":"#数据可视化函数定义\ndef show_batch(ds, row=2, col=2):\n    fig = plt.figure(figsize=(6, 6))\n    img_index = np.random.randint(0, len(ds)-1, row*col)\n    \n    for i in range(len(img_index)):\n        img = ds[img_index[i]]\n        \n        if isinstance(img, torch.Tensor):\n            img = img.detach().numpy()\n        \n        ax = fig.add_subplot(2, 2, i + 1, xticks=[], yticks=[])\n        ax.imshow(img, cmap='jet')\n        ax.set_title(f'ID: {img_index[i]}')\n    \n    plt.tight_layout()\n    plt.show()\n#数据可视化\ndummy_dataset = BirdDataset(all_bird_data, get_transforms('test'))\n \ntest_input = dummy_dataset[0]\nprint(test_input.detach().numpy().shape)\n \nif config.VISUALIZE:\n    show_batch(dummy_dataset)\n#清理资源\ndel dummy_dataset\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-06-19T00:49:15.547151Z","iopub.execute_input":"2024-06-19T00:49:15.547566Z","iopub.status.idle":"2024-06-19T00:49:16.557217Z","shell.execute_reply.started":"2024-06-19T00:49:15.547532Z","shell.execute_reply":"2024-06-19T00:49:16.555718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"markdown","source":"## Network","metadata":{}},{"cell_type":"code","source":"class EffNet(nn.Module):\n    \n    def __init__(self, model_type, n_classes, pretrained=False):\n        super().__init__()\n        \n        if model_type == 'efficientnet_b0':\n            if pretrained: weights = efficientnet.EfficientNet_B0_Weights.DEFAULT\n            else: weights = None\n            self.base_model = efficientnet.efficientnet_b0(weights=weights)\n        elif model_type == 'efficientnet_b1':\n            if pretrained: weights = efficientnet.EfficientNet_B1_Weights.DEFAULT\n            else: weights = None\n            self.base_model = efficientnet.efficientnet_b1(weights=weights)\n        elif model_type == 'efficientnet_b2':\n            if pretrained: weights = efficientnet.EfficientNet_B2_Weights.DEFAULT\n            else: weights = None\n            self.base_model = efficientnet.efficientnet_b2(weights=weights)\n        elif model_type == 'efficientnet_b3':\n            if pretrained: weights = efficientnet.EfficientNet_B3_Weights.DEFAULT\n            else: weights = None\n            self.base_model = efficientnet.efficientnet_b3(weights=weights)\n        else:\n            raise ValueError('model type not supported')\n        \n        self.base_model.classifier[1] = nn.Linear(self.base_model.classifier[1].in_features, n_classes, dtype=torch.float32)\n    \n    def forward(self, x):\n        x = x.unsqueeze(-1)\n        x = torch.cat([x, x, x], dim=3).permute(0, 3, 1, 2)\n        return self.base_model(x)","metadata":{"execution":{"iopub.status.busy":"2024-06-19T00:49:29.205689Z","iopub.execute_input":"2024-06-19T00:49:29.206648Z","iopub.status.idle":"2024-06-19T00:49:29.222614Z","shell.execute_reply.started":"2024-06-19T00:49:29.206598Z","shell.execute_reply":"2024-06-19T00:49:29.221112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model by PyTorch-Lightning","metadata":{}},{"cell_type":"code","source":"#LightningModule 是 PyTorch Lightning 框架的核心，用于构建易于训练、验证和测试的模型\nclass BirdModel(pl.LightningModule):    \n    def __init__(self):\n        super().__init__()\n        \n        #主干网络\n        self.backbone = EffNet(config.MODEL_TYPE, n_classes=len(label_list))\n        \n        #交叉熵损失函数\n        self.loss_fn = nn.CrossEntropyLoss()\n        \n        #初始化一个列表，用于存储验证步骤的输出\n        self.validation_step_outputs = []\n        \n    def forward(self, images):\n        return self.backbone(images)\n    #定义优化器和学习率调度器\n    def configure_optimizers(self):\n        \n        #定义 Adam 优化器\n        model_optimizer = torch.optim.Adam(\n            filter(lambda p: p.requires_grad, self.parameters()),\n            lr=config.LR,\n            weight_decay=config.WEIGHT_DECAY\n        )\n        \n        #定义余弦退火学习率调度器\n        lr_scheduler = CosineAnnealingWarmRestarts(\n            model_optimizer,\n            T_0=config.EPOCHS,\n            T_mult=1,\n            eta_min=1e-6,\n            last_epoch=-1\n        )\n        \n        return {\n            'optimizer': model_optimizer,\n            'lr_scheduler': {\n                'scheduler': lr_scheduler,\n                'interval': 'epoch',\n                'monitor': 'val_loss',\n                'frequency': 1\n            }\n        }\n    #定义训练步骤\n    def training_step(self, batch, batch_idx):\n        \n        #获取输入\n        image, target = batch\n        image = image.to(self.device)\n        target = target.to(self.device)\n        \n        #前向传播\n        y_pred = self(image)\n        \n        #计算损失\n        train_loss = self.loss_fn(y_pred, target)\n        \n        #记录训练损失\n        self.log('train_loss', train_loss, True)\n        \n        return train_loss\n    #定义验证步骤\n    def validation_step(self, batch, batch_idx):\n        \n        #从验证批次中获取输入图像和目标标签\n        image, target = batch\n        image = image.to(self.device)\n        target = target.to(self.device)\n        \n        #在不计算梯度的情况下进行前向传播\n        with torch.no_grad():\n            y_pred = self(image)\n        #将预测输出和目标存储在validation_step_outputs列表中    \n        self.validation_step_outputs.append({\"logits\": y_pred, \"targets\": target})\n    #返回训练和验证数据的 DataLoader\n    def train_dataloader(self):\n        return self._train_dataloader\n \n    def validation_dataloader(self):\n        return self._validation_dataloader\n   #在每个验证周期结束时调用，用于处理整个验证集的结果\n    def on_validation_epoch_end(self):\n        \n        #合并验证批次数据\n        outputs = self.validation_step_outputs\n        \n        output_val = torch.cat([x['logits'] for x in outputs], dim=0).cpu().detach()\n        target_val = torch.cat([x['targets'] for x in outputs], dim=0).cpu().detach()\n        \n        #计算验证损失，使用存储的目标标签和模型的预测输出\n        val_loss = self.loss_fn(output_val, target_val)\n        \n        #将目标标签转换为独热编码格式\n        target_val = torch.nn.functional.one_hot(target_val, len(label_list))\n        \n        #评估指标\n        gt_df = pd.DataFrame(target_val.numpy().astype(np.float32), columns=label_list)\n        pred_df = pd.DataFrame(output_val.numpy().astype(np.float32), columns=label_list)\n        \n        gt_df['id'] = [f'id_{i}' for i in range(len(gt_df))]\n        pred_df['id'] = [f'id_{i}' for i in range(len(pred_df))]\n        \n        val_score = score(gt_df, pred_df, row_id_column_name='id')\n        #使用self.log记录验证损失和分数，以便在Lightning日志中跟踪\n        self.log(\"val_score\", val_score, True)\n        \n        \n        return {'val_loss': val_loss, 'val_score': val_score}","metadata":{"execution":{"iopub.status.busy":"2024-06-19T00:49:33.142673Z","iopub.execute_input":"2024-06-19T00:49:33.143073Z","iopub.status.idle":"2024-06-19T00:49:33.166970Z","shell.execute_reply.started":"2024-06-19T00:49:33.143025Z","shell.execute_reply":"2024-06-19T00:49:33.165432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(data_loader, model):\n    model.to(config.DEVICE)\n    model.eval()\n    pred = []\n    for batch in tqdm(data_loader):\n        with torch.no_grad():\n            x = batch\n            outputs = model(x)\n            outputs = nn.Softmax(dim=1)(outputs)\n        pred.append(outputs.detach().cpu())\n    \n    pred = torch.cat(pred, dim=0).cpu().detach()\n    \n    return pred.numpy().astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2024-06-19T00:49:39.858709Z","iopub.execute_input":"2024-06-19T00:49:39.859813Z","iopub.status.idle":"2024-06-19T00:49:39.867912Z","shell.execute_reply.started":"2024-06-19T00:49:39.859772Z","shell.execute_reply":"2024-06-19T00:49:39.866312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference & Submision","metadata":{}},{"cell_type":"markdown","source":"## ckpts","metadata":{}},{"cell_type":"code","source":"# ckpt_list = glob(f'{config.CKPT_ROOT}/*.ckpt')\n# print(f'find {len(ckpt_list)} ckpts in {config.CKPT_ROOT}.')\n\nckpt_list = [f'/kaggle/input/train/fold_1.ckpt']","metadata":{"execution":{"iopub.status.busy":"2024-06-19T02:07:32.500345Z","iopub.execute_input":"2024-06-19T02:07:32.500704Z","iopub.status.idle":"2024-06-19T02:07:32.534395Z","shell.execute_reply.started":"2024-06-19T02:07:32.500675Z","shell.execute_reply":"2024-06-19T02:07:32.533284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Main Loop","metadata":{}},{"cell_type":"code","source":"predictions = []\n\nfor ckpt in ckpt_list:\n    \n    bird_model = BirdModel()\n    \n    weights = torch.load(ckpt, map_location=torch.device('cpu'))['state_dict']\n    bird_model.load_state_dict(weights)\n    \n    test_dataset = BirdDataset(all_bird_data, get_transforms('test'))\n    test_loader = torch.utils.data.DataLoader(\n        test_dataset,\n        batch_size=config.BATCH_SIZE,\n        num_workers=config.N_WORKERS,\n        shuffle=False,\n        drop_last=False\n    )\n    \n    predictions.append(predict(test_loader, bird_model))\n    gc.collect()\n\npredictions = np.mean(predictions, axis=0)","metadata":{"execution":{"iopub.status.busy":"2024-06-19T00:50:09.509740Z","iopub.execute_input":"2024-06-19T00:50:09.510178Z","iopub.status.idle":"2024-06-19T00:50:34.683353Z","shell.execute_reply.started":"2024-06-19T00:50:09.510144Z","shell.execute_reply":"2024-06-19T00:50:34.682071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_pred = pd.DataFrame(predictions, columns=label_list)\nsub_id = pd.DataFrame({'row_id': list(all_bird_data.keys())})\n\nsub = pd.concat([sub_id, sub_pred], axis=1)\n\nsub.to_csv('submission.csv',index=False)\nprint(f'Submissionn shape: {sub.shape}')\nsub.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-06-19T00:50:37.512038Z","iopub.execute_input":"2024-06-19T00:50:37.513110Z","iopub.status.idle":"2024-06-19T00:50:37.732122Z","shell.execute_reply.started":"2024-06-19T00:50:37.513061Z","shell.execute_reply":"2024-06-19T00:50:37.730831Z"},"trusted":true},"execution_count":null,"outputs":[]}]}