{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"},{"sourceId":11828260,"sourceType":"datasetVersion","datasetId":7430593},{"sourceId":11870659,"sourceType":"datasetVersion","datasetId":7459867},{"sourceId":11929021,"sourceType":"datasetVersion","datasetId":7478495,"isSourceIdPinned":true},{"sourceId":12288956,"sourceType":"datasetVersion","datasetId":7744941}],"dockerImageVersionId":31040,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🐦 BirdCLEF 2025: Weighted Blend Inference (LB 0.878)\n\n## 🧠 Summary\n\nThis notebook achieves a public leaderboard score of **0.878** using a **Weighted Blend** approach.\n\n## 🧩 Models Used\n\nThe final prediction is based on the ensemble of the following public models:\n\n- 📘 **[Bird2025 | Single SED Model Inference [LB 0.857]](https://www.kaggle.com/code/i2nfinit3y/bird2025-single-sed-model-inference-lb-0-857)**  \n  by [I2nfinit3y](https://www.kaggle.com/i2nfinit3y)  \n  A strong single SED model serving as the base for one of the ensemble components.\n\n- 🧪 **[Post-Processing with Power Adjustment for Low-Rank](https://www.kaggle.com/code/myso1987/post-processing-with-power-adjustment-for-low-rank)**  \n  by [MYSO](https://www.kaggle.com/myso1987)  \n  This notebook introduced a clever post-processing technique to boost low-confidence predictions via power transformation.\n\n- 🔗 **[Bird25 | WeightedBlend | nfnet + convnextv2 | LB.860](https://www.kaggle.com/code/hideyukizushi/bird25-weightedblend-nfnet-convnextv2-lb-860)**  \n  by [yukiZ](https://www.kaggle.com/hideyukizushi)  \n  Provided the core blending logic used to combine model outputs.\n\n## ⚖️ Weighted Blend Strategy\n\nWe use the weighted average of the two model outputs as follows:\n\n- **nfnet**: 25%\n- **seresnext**: 75%\n\nWhile this blend achieves a high LB score, it may be overfitting to the public test set — a pattern often seen in BirdCLEF competitions from 2022 to 2024. Please consider this when evaluating the results.\n\n---","metadata":{}},{"cell_type":"markdown","source":"<h1 style=\"color: #6cb4e4;  text-align: center;  padding: 0.25em;  border-top: solid 2.5px #6cb4e4;  border-bottom: solid 2.5px #6cb4e4;  background: -webkit-repeating-linear-gradient(-45deg, #f0f8ff, #f0f8ff 3px,#e9f4ff 3px, #e9f4ff 7px);  background: repeating-linear-gradient(-45deg, #f0f8ff, #f0f8ff 3px,#e9f4ff 3px, #e9f4ff 7px);height:45px;\">\n<b>\n《《《Submission1(nfnet)》》》\n</b></h1> ","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport time\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport timm\nimport torch.nn.functional as F\nimport torchaudio\nimport torchaudio.transforms as AT\nfrom contextlib import contextmanager\nfrom typing import Union\nimport concurrent.futures","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:03:10.467989Z","iopub.execute_input":"2025-05-28T15:03:10.468365Z","iopub.status.idle":"2025-05-28T15:03:28.904778Z","shell.execute_reply.started":"2025-05-28T15:03:10.468336Z","shell.execute_reply":"2025-05-28T15:03:28.903787Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def apply_power_to_low_ranked_cols(\n    p: np.ndarray,\n    top_k: int = 30,\n    exponent: Union[int, float] = 2,\n    inplace: bool = True\n) -> np.ndarray:\n    if not inplace:\n        p = p.copy()\n\n    # Identify columns whose max value ranks below `top_k`\n    tail_cols = np.argsort(-p.max(axis=0))[top_k:]\n\n    # Apply the power transformation to those columns\n    p[:, tail_cols] = p[:, tail_cols] ** exponent\n    return p","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:03:28.906293Z","iopub.execute_input":"2025-05-28T15:03:28.906816Z","iopub.status.idle":"2025-05-28T15:03:28.913312Z","shell.execute_reply.started":"2025-05-28T15:03:28.90679Z","shell.execute_reply":"2025-05-28T15:03:28.912391Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_audio_dir = '../input/birdclef-2025/test_soundscapes/'\nfile_list = [f for f in sorted(os.listdir(test_audio_dir))]\nfile_list = [file.split('.')[0] for file in file_list if file.endswith('.ogg')]\n\ndebug = False\nif len(file_list) == 0:\n    debug = True\n    debug_st_num = 5\n    debug_num = 8\n    test_audio_dir = '../input/birdclef-2025/train_soundscapes/'\n    file_list = [f for f in sorted(os.listdir(test_audio_dir))]\n    file_list = [file.split('.')[0] for file in file_list if file.endswith('.ogg')]\n    file_list = file_list[debug_st_num:debug_st_num+debug_num]\n\nprint('Debug mode:', debug)\nprint('Number of test soundscapes:', len(file_list))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:03:28.914405Z","iopub.execute_input":"2025-05-28T15:03:28.914916Z","iopub.status.idle":"2025-05-28T15:03:29.067103Z","shell.execute_reply.started":"2025-05-28T15:03:28.914883Z","shell.execute_reply":"2025-05-28T15:03:29.065888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"wav_sec = 5\nsample_rate = 32000\nmin_segment = sample_rate*wav_sec\n\nclass_labels = sorted(os.listdir('../input/birdclef-2025/train_audio/'))\n\nn_fft=1024\nwin_length=1024\nhop_length=512\nf_min=40\nf_max=16000\nn_mels=128\n\nmel_spectrogram = AT.MelSpectrogram(\n    sample_rate=sample_rate,\n    n_fft=n_fft,\n    win_length=win_length,\n    hop_length=hop_length,\n    center=True,\n    f_min=f_min,\n    f_max=f_max,\n    pad_mode=\"reflect\",\n    power=2.0,\n    norm='slaney',\n    n_mels=n_mels,\n    mel_scale=\"htk\",\n    # normalized=True\n)\n\ndef normalize_std(spec, eps=1e-6):\n    mean = torch.mean(spec)\n    std = torch.std(spec)\n    return torch.where(std == 0, spec-mean, (spec - mean) / (std+eps))\n\ndef audio_to_mel(filepath=None):\n    waveform, sample_rate = torchaudio.load(filepath,backend=\"soundfile\")\n    len_wav = waveform.shape[1]\n    waveform = waveform[0,:].reshape(1, len_wav) # stereo->mono mono->mono\n    PREDS = []\n    for i in range(12):\n        waveform2 = waveform[:,i*sample_rate*5:i*sample_rate*5+sample_rate*5]\n        melspec = mel_spectrogram(waveform2)\n        melspec = torch.log(melspec+1e-6)\n        melspec = normalize_std(melspec)\n        melspec = torch.unsqueeze(melspec, dim=0)\n        \n        PREDS.append(melspec)\n    return torch.vstack(PREDS)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:03:29.068184Z","iopub.execute_input":"2025-05-28T15:03:29.068455Z","iopub.status.idle":"2025-05-28T15:03:29.136588Z","shell.execute_reply.started":"2025-05-28T15:03:29.068432Z","shell.execute_reply":"2025-05-28T15:03:29.13548Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n\n\ndef init_weights(model):\n    classname = model.__class__.__name__\n    if classname.find(\"Conv2d\") != -1:\n        nn.init.xavier_uniform_(model.weight, gain=np.sqrt(2))\n        model.bias.data.fill_(0)\n    elif classname.find(\"BatchNorm\") != -1:\n        model.weight.data.normal_(1.0, 0.02)\n        model.bias.data.fill_(0)\n    elif classname.find(\"GRU\") != -1:\n        for weight in model.parameters():\n            if len(weight.size()) > 1:\n                nn.init.orghogonal_(weight.data)\n    elif classname.find(\"Linear\") != -1:\n        model.weight.data.normal_(0, 0.01)\n        model.bias.data.zero_()\n\n\ndef interpolate(x, ratio):\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output, frames_num):\n    output = F.interpolate(\n        framewise_output.unsqueeze(1),\n        size=(frames_num, framewise_output.size(2)),\n        align_corners=True,\n        mode=\"bilinear\").squeeze(1)\n\n    return output\n\n\nclass AttBlockV2(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"linear\"):\n        super().__init__()\n\n        self.activation = activation\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n\n    def forward(self, x):\n        norm_att = torch.softmax(torch.tanh(self.att(x)), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)\n\n\nclass TimmSED(nn.Module):\n    def __init__(self, base_model_name: str, pretrained=False, num_classes=24, in_channels=1, n_mels=24):\n        super().__init__()\n\n        self.bn0 = nn.BatchNorm2d(n_mels)\n\n        base_model = timm.create_model(\n            base_model_name, pretrained=pretrained, in_chans=in_channels)\n        layers = list(base_model.children())[:-2]\n        self.encoder = nn.Sequential(*layers)\n\n        in_features = base_model.num_features\n\n        self.fc1 = nn.Linear(in_features, in_features, bias=True)\n        self.att_block2 = AttBlockV2(\n            in_features, num_classes, activation=\"sigmoid\")\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n        \n\n    def forward(self, input_data):\n        x = input_data.transpose(2,3)\n        x = torch.cat((x,x,x),1)\n\n        x = x.transpose(2, 3)\n\n        x = self.encoder(x)\n        \n        x = torch.mean(x, dim=2)\n\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block2(x)\n        logit = torch.sum(norm_att * self.att_block2.cla(x), dim=2)\n\n        output_dict = {\n            'logit': logit,\n        }\n\n        return output_dict","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:03:29.139322Z","iopub.execute_input":"2025-05-28T15:03:29.139686Z","iopub.status.idle":"2025-05-28T15:03:29.159387Z","shell.execute_reply.started":"2025-05-28T15:03:29.139662Z","shell.execute_reply":"2025-05-28T15:03:29.158467Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"base_model_name='eca_nfnet_l0'\npretrained=False\nin_channels=3\n\nMODELS = [f'/kaggle/input/birdclef-2025-sed-models-p/sed{i}.pth' for i in range(3)]\n\nMODELS","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:03:29.160284Z","iopub.execute_input":"2025-05-28T15:03:29.160543Z","iopub.status.idle":"2025-05-28T15:03:29.190461Z","shell.execute_reply.started":"2025-05-28T15:03:29.160523Z","shell.execute_reply":"2025-05-28T15:03:29.189681Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"models = []\nfor path in MODELS:\n    model = TimmSED(base_model_name=base_model_name,\n               pretrained=pretrained,\n               num_classes=len(class_labels),\n               in_channels=in_channels,\n               n_mels=n_mels);\n    model.load_state_dict(torch.load(path, weights_only=True, map_location=torch.device('cpu')))\n    model.eval();\n    models.append(model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:03:29.191537Z","iopub.execute_input":"2025-05-28T15:03:29.191898Z","iopub.status.idle":"2025-05-28T15:03:35.283753Z","shell.execute_reply.started":"2025-05-28T15:03:29.19187Z","shell.execute_reply":"2025-05-28T15:03:35.282731Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def prediction(afile):    \n    global pred\n    path = test_audio_dir + afile + '.ogg'\n    with torch.inference_mode():\n        sig = audio_to_mel(path)\n        outputs = None\n        for model in models:\n            model.eval()\n            p = model(sig)\n            p = torch.sigmoid(p['logit']).detach().cpu().numpy() \n            p = apply_power_to_low_ranked_cols(p, top_k=30,exponent=2)\n            if outputs is None: outputs = p\n            else: outputs += p\n            \n        outputs /= len(models)\n        chunks = [[] for i in range(12)]\n        for i in range(len(chunks)):        \n            chunk_end_time = (i + 1) * 5\n            row_id = afile + '_' + str(chunk_end_time)\n            pred['row_id'].append(row_id)\n            bird_no = 0\n            for bird in class_labels:         \n                pred[bird].append(outputs[i,bird_no])\n                bird_no += 1\n        gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:03:35.285331Z","iopub.execute_input":"2025-05-28T15:03:35.285755Z","iopub.status.idle":"2025-05-28T15:03:35.293968Z","shell.execute_reply.started":"2025-05-28T15:03:35.28571Z","shell.execute_reply":"2025-05-28T15:03:35.292997Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pred = {'row_id': []}\nfor species_code in class_labels:\n    pred[species_code] = []\n    \nstart = time.time()\nwith concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:\n    _ = list(executor.map(prediction, file_list))\nend_t = time.time()\n\nif debug == True:\n    print(700*(end_t - start)/60/debug_num)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:03:35.295052Z","iopub.execute_input":"2025-05-28T15:03:35.295341Z","iopub.status.idle":"2025-05-28T15:04:01.444834Z","shell.execute_reply.started":"2025-05-28T15:03:35.295307Z","shell.execute_reply":"2025-05-28T15:04:01.443493Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"results = pd.DataFrame(pred, columns = ['row_id'] + class_labels)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:04:01.445969Z","iopub.execute_input":"2025-05-28T15:04:01.446241Z","iopub.status.idle":"2025-05-28T15:04:01.487554Z","shell.execute_reply.started":"2025-05-28T15:04:01.446221Z","shell.execute_reply":"2025-05-28T15:04:01.48651Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"results.to_csv(\"submission1.csv\", index=False)    \n\nsub = pd.read_csv('submission1.csv')\ncols = sub.columns[1:]\ngroups = sub['row_id'].str.rsplit('_', n=1).str[0]\ngroups = groups.values\nfor group in np.unique(groups):\n    sub_group = sub[group == groups]\n    predictions = sub_group[cols].values\n    new_predictions = predictions.copy()\n    for i in range(1, predictions.shape[0]-1):\n        new_predictions[i] = (predictions[i-1] * 0.2) + (predictions[i] * 0.6) + (predictions[i+1] * 0.2)\n    new_predictions[0] = (predictions[0] * 0.8) + (predictions[1] * 0.2)\n    new_predictions[-1] = (predictions[-1] * 0.8) + (predictions[-2] * 0.2)\n    sub_group[cols] = new_predictions\n    sub[group == groups] = sub_group\nsub.to_csv(\"submission1.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:04:01.489014Z","iopub.execute_input":"2025-05-28T15:04:01.489295Z","iopub.status.idle":"2025-05-28T15:04:02.177845Z","shell.execute_reply.started":"2025-05-28T15:04:01.489276Z","shell.execute_reply":"2025-05-28T15:04:02.1767Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<h1 style=\"color: #6cb4e4;  text-align: center;  padding: 0.25em;  border-top: solid 2.5px #6cb4e4;  border-bottom: solid 2.5px #6cb4e4;  background: -webkit-repeating-linear-gradient(-45deg, #f0f8ff, #f0f8ff 3px,#e9f4ff 3px, #e9f4ff 7px);  background: repeating-linear-gradient(-45deg, #f0f8ff, #f0f8ff 3px,#e9f4ff 3px, #e9f4ff 7px);height:45px;\">\n<b>\n《《《Submission2(seresnext)》》》\n</b></h1> ","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport warnings\nimport logging\nimport time\nimport math\nimport cv2\nfrom pathlib import Path\nimport joblib\n\nimport numpy as np\nimport pandas as pd\nimport librosa\nimport soundfile as sf\nfrom soundfile import SoundFile \nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.cuda.amp import autocast, GradScaler\nimport timm\nfrom tqdm.auto import tqdm\nfrom glob import glob\nimport torchaudio\nimport random\nimport itertools\nfrom typing import Union\n\nimport concurrent.futures\n\nwarnings.filterwarnings(\"ignore\")\nlogging.basicConfig(level=logging.ERROR)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:04:02.178899Z","iopub.execute_input":"2025-05-28T15:04:02.179227Z","iopub.status.idle":"2025-05-28T15:04:02.822262Z","shell.execute_reply.started":"2025-05-28T15:04:02.179199Z","shell.execute_reply":"2025-05-28T15:04:02.82116Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CFG:\n    \n    seed = 42\n    print_freq = 100\n    num_workers = 4\n\n    stage = 'train_bce'\n\n    train_datadir = '/kaggle/input/birdclef-2025/train_audio'\n    train_csv = '/kaggle/input/birdclef-2025/train.csv'\n    test_soundscapes = '/kaggle/input/birdclef-2025/test_soundscapes'\n    submission_csv = '/kaggle/input/birdclef-2025/sample_submission.csv'\n    taxonomy_csv = '/kaggle/input/birdclef-2025/taxonomy.csv'\n    model_files = ['/kaggle/input/bird2025-sed-ckpt/sedmodel.pth'\n                  ]\n \n    model_name = 'seresnext26t_32x4d'  \n    pretrained = False\n    in_channels = 1\n\n    \n    SR = 32000\n    target_duration = 5\n    train_duration = 10\n    \n    \n    device = 'cpu'\n\ncfg = CFG()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:04:02.823192Z","iopub.execute_input":"2025-05-28T15:04:02.82354Z","iopub.status.idle":"2025-05-28T15:04:02.829738Z","shell.execute_reply.started":"2025-05-28T15:04:02.823511Z","shell.execute_reply":"2025-05-28T15:04:02.828523Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Using device: {cfg.device}\")\nprint(f\"Loading taxonomy data...\")\ntaxonomy_df = pd.read_csv(cfg.taxonomy_csv)\nspecies_ids = taxonomy_df['primary_label'].tolist()\nnum_classes = len(species_ids)\nprint(f\"Number of classes: {num_classes}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:04:02.832903Z","iopub.execute_input":"2025-05-28T15:04:02.833232Z","iopub.status.idle":"2025-05-28T15:04:02.881781Z","shell.execute_reply.started":"2025-05-28T15:04:02.833211Z","shell.execute_reply":"2025-05-28T15:04:02.880787Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def set_seed(seed=42):\n    \"\"\"\n    Set seed for reproducibility\n    \"\"\"\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nset_seed(cfg.seed)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:04:02.882742Z","iopub.execute_input":"2025-05-28T15:04:02.883052Z","iopub.status.idle":"2025-05-28T15:04:02.909002Z","shell.execute_reply.started":"2025-05-28T15:04:02.883025Z","shell.execute_reply":"2025-05-28T15:04:02.907659Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class AttBlockV2(nn.Module):\n    def __init__(self, in_features: int, out_features: int, activation=\"linear\"):\n        super().__init__()\n\n        self.activation = activation\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True,\n        )\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True,\n        )\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n\n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.tanh(self.att(x)), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == \"linear\":\n            return x\n        elif self.activation == \"sigmoid\":\n            return torch.sigmoid(x)\n\n\ndef init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.0)\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.0)\n    bn.weight.data.fill_(1.0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:04:02.909726Z","iopub.execute_input":"2025-05-28T15:04:02.909993Z","iopub.status.idle":"2025-05-28T15:04:02.921082Z","shell.execute_reply.started":"2025-05-28T15:04:02.909973Z","shell.execute_reply":"2025-05-28T15:04:02.919969Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BirdCLEFModel(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.cfg = cfg\n        \n        taxonomy_df = pd.read_csv('/kaggle/input/birdclef-2025/taxonomy.csv')\n        self.num_classes = len(taxonomy_df)\n\n        self.bn0 = nn.BatchNorm2d(cfg['n_mels'])\n        \n        self.backbone = timm.create_model(\n            cfg['model_name'],\n            pretrained=False,\n            in_chans=cfg['in_channels'],\n            drop_rate=0.2,\n            drop_path_rate=0.2,\n        )\n\n        layers = list(self.backbone.children())[:-2]\n        self.encoder = nn.Sequential(*layers)\n        \n        if \"efficientnet\" in self.cfg['model_name']:\n            backbone_out = self.backbone.classifier.in_features\n        elif \"eca\" in self.cfg['model_name']:\n            backbone_out = self.backbone.head.fc.in_features\n        elif \"res\" in self.cfg['model_name']:\n            backbone_out = self.backbone.fc.in_features\n        else:\n            backbone_out = self.backbone.num_features\n            \n        \n        self.fc1 = nn.Linear(backbone_out, backbone_out, bias=True)\n        self.att_block = AttBlockV2(backbone_out, self.num_classes, activation=\"sigmoid\")\n\n        self.melspec_transform = torchaudio.transforms.MelSpectrogram(\n            sample_rate=self.cfg['SR'],\n            hop_length=self.cfg['hop_length'],\n            n_mels=self.cfg['n_mels'],\n            f_min=self.cfg['f_min'],\n            f_max=self.cfg['f_max'],\n            n_fft=self.cfg['n_fft'],\n            pad_mode=\"constant\",\n            norm=\"slaney\",\n            onesided=True,\n            mel_scale=\"htk\",\n        )\n        if self.cfg['device'] == \"cuda\":\n            self.melspec_transform = self.melspec_transform.cuda()\n        else:\n            self.melspec_transform = self.melspec_transform.cpu()\n\n        self.db_transform = torchaudio.transforms.AmplitudeToDB(\n            stype=\"power\", top_db=80\n        )\n\n\n    def extract_feature(self,x):\n        x = x.permute((0, 1, 3, 2))\n        frames_num = x.shape[2]\n        \n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n        \n        # if self.training:\n        #    x = self.spec_augmenter(x)\n        \n        x = x.transpose(2, 3)\n        # (batch_size, channels, freq, frames)\n        x = self.encoder(x)\n        \n        # (batch_size, channels, frames)\n        x = torch.mean(x, dim=2)\n        \n        # channel smoothing\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n        \n        x = F.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.5, training=self.training)\n        return x, frames_num\n        \n    @torch.cuda.amp.autocast(enabled=False)\n    def transform_to_spec(self, audio):\n\n        audio = audio.float()\n        \n        spec = self.melspec_transform(audio)\n        spec = self.db_transform(spec)\n\n        if self.cfg['normal'] == 80:\n            spec = (spec + 80) / 80\n        elif self.cfg['normal'] == 255:\n            spec = spec / 255\n        else:\n            raise NotImplementedError\n                \n        if self.cfg['in_channels'] == 3:\n            spec = image_delta(spec)\n        \n        return spec\n\n    def forward(self, x):\n\n        with torch.no_grad():\n            x = self.transform_to_spec(x)\n\n        x, frames_num = self.extract_feature(x)\n        \n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n        logit = torch.sum(norm_att * self.att_block.cla(x), dim=2)\n        segmentwise_logit = self.att_block.cla(x).transpose(1, 2)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        return torch.logit(clipwise_output)\n\n    def infer(self, x, tta_delta=2):\n        with torch.no_grad():\n            x = self.transform_to_spec(x)\n        x,_ = self.extract_feature(x)\n        time_att = torch.tanh(self.att_block.att(x))\n        feat_time = x.size(-1)\n        start = (\n            feat_time / 2 - feat_time * (self.cfg['infer_duration'] / self.cfg['duration_train']) / 2\n        )\n        end = start + feat_time * (self.cfg['infer_duration'] / self.cfg['duration_train'])\n        start = int(start)\n        end = int(end)\n        pred = self.attention_infer(start,end,x,time_att)\n\n        start_minus = max(0, start-tta_delta)\n        end_minus=end-tta_delta\n        pred_minus = self.attention_infer(start_minus,end_minus,x,time_att)\n\n        start_plus = start+tta_delta\n        end_plus=min(feat_time, end+tta_delta)\n        pred_plus = self.attention_infer(start_plus,end_plus,x,time_att)\n\n        pred = 0.5*pred + 0.25*pred_minus + 0.25*pred_plus\n        return pred\n        \n    def attention_infer(self,start,end,x,time_att):\n        feat = x[:, :, start:end]\n        # att = torch.softmax(time_att[:, :, start:end], dim=-1)\n        #             print(feat_time, start, end)\n        #             print(att_a.sum(), att.sum(), time_att.shape)\n        framewise_pred = torch.sigmoid(self.att_block.cla(feat))\n        framewise_pred_max = framewise_pred.max(dim=2)[0]\n        # clipwise_output = torch.sum(framewise_pred * att, dim=-1)\n        #logits = torch.sum(\n        #    self.att_block.cla(feat) * att,\n        #    dim=-1,\n        #)\n\n        # return clipwise_output\n        return framewise_pred_max","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:04:02.922041Z","iopub.execute_input":"2025-05-28T15:04:02.922392Z","iopub.status.idle":"2025-05-28T15:04:02.951608Z","shell.execute_reply.started":"2025-05-28T15:04:02.922365Z","shell.execute_reply":"2025-05-28T15:04:02.950173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_sample(path, cfg):\n    audio, orig_sr = sf.read(path, dtype=\"float32\")\n    seconds = []\n    audio_length = cfg.SR * cfg.target_duration\n    step = audio_length\n    for i in range(audio_length, len(audio) + step, step):\n        start = max(0, i - audio_length)\n        end = start + audio_length\n        if end > len(audio):\n            pass\n        else:\n            seconds.append(int(end/cfg.SR))\n\n    audio = np.concatenate([audio,audio,audio])\n    audios = []\n    for i,second in enumerate(seconds):\n        end_seconds = int(second)\n        start_seconds = int(end_seconds - cfg.target_duration)\n    \n        end_index = int(cfg.SR * (end_seconds + (cfg.train_duration - cfg.target_duration) / 2) ) + len(audio) // 3\n        start_index = int(cfg.SR * (start_seconds - (cfg.train_duration - cfg.target_duration) / 2) ) + len(audio) // 3\n        end_pad = int(cfg.SR * (cfg.train_duration - cfg.target_duration) / 2) \n        start_pad = int(cfg.SR * (cfg.train_duration - cfg.target_duration) / 2) \n        y = audio[start_index:end_index].astype(np.float32)\n        if i==0:\n            y[:start_pad] = 0\n        elif i==(len(seconds)-1):\n            y[-end_pad:] = 0\n        audios.append(y)\n\n    return audios\n\ndef sigmoid(x):\n    s = 1 / (1 + np.exp(-x))\n    return s","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:04:02.952782Z","iopub.execute_input":"2025-05-28T15:04:02.953056Z","iopub.status.idle":"2025-05-28T15:04:02.981161Z","shell.execute_reply.started":"2025-05-28T15:04:02.953036Z","shell.execute_reply":"2025-05-28T15:04:02.979907Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def find_model_files(cfg):\n    \"\"\"\n    Find all .pth model files in the specified model directory\n    \"\"\"\n    model_files = []\n    \n    model_dir = Path(cfg.model_path)\n    \n    for path in model_dir.glob('**/*.pth'):\n        model_files.append(str(path))\n    \n    return model_files\n\ndef load_models(cfg, num_classes):\n    \"\"\"\n    Load all found model files and prepare them for ensemble\n    \"\"\"\n    models = []\n    \n    # model_files = find_model_files(cfg)\n    model_files = cfg.model_files\n    \n    if not model_files:\n        print(f\"Warning: No model files found under {cfg.model_path}!\")\n        return models\n    \n    print(f\"Found a total of {len(model_files)} model files.\")\n    \n    for i, model_path in enumerate(model_files):\n        try:\n            print(f\"Loading model: {model_path}\")\n            checkpoint = torch.load(model_path, map_location=torch.device(cfg.device), weights_only=False)\n            cfg_temp = checkpoint['cfg']\n            cfg_temp['device'] = cfg.device\n            \n            model = BirdCLEFModel(cfg_temp)\n            model.load_state_dict(checkpoint['model_state_dict'])\n            model = model.to(cfg.device)\n            model.eval()\n            model.zero_grad()\n            model.half().float()\n            \n            models.append(model)\n        except Exception as e:\n            print(f\"Error loading model {model_path}: {e}\")\n    \n    return models\n\ndef predict_on_spectrogram(audio_path, models, cfg, species_ids):\n    \"\"\"Process a single audio file and predict species presence for each 5-second segment\"\"\"\n    audio_path = str(audio_path)\n    predictions = []\n    row_ids = []\n    soundscape_id = Path(audio_path).stem\n\n    print(f\"Processing {soundscape_id}\")\n    audio_data = load_sample(audio_path, cfg)\n    for segment_idx, audio_input in enumerate(audio_data):\n        \n        end_time_sec = (segment_idx + 1) * cfg.target_duration\n        row_id = f\"{soundscape_id}_{end_time_sec}\"\n        row_ids.append(row_id)\n        \n        mel_spec = torch.tensor(audio_input, dtype=torch.float32).unsqueeze(0).unsqueeze(0)\n        mel_spec = mel_spec.to(cfg.device)\n        \n        if len(models) == 1:\n            with torch.no_grad():\n                outputs = models[0].infer(mel_spec)\n                final_preds = outputs.squeeze()\n                # final_preds = torch.sigmoid(outputs).cpu().numpy().squeeze()\n\n        else:\n            segment_preds = []\n            for model in models:\n                with torch.no_grad():\n                    outputs = model.infer(mel_spec)\n                    probs = outputs.squeeze()\n                    # probs = torch.sigmoid(outputs).cpu().numpy().squeeze()\n                    segment_preds.append(probs)\n\n            \n            final_preds = np.mean(segment_preds, axis=0)\n                \n        predictions.append(final_preds)\n\n    predictions = np.stack(predictions,axis=0)\n    \n    return row_ids, predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:04:02.98208Z","iopub.execute_input":"2025-05-28T15:04:02.982431Z","iopub.status.idle":"2025-05-28T15:04:03.014987Z","shell.execute_reply.started":"2025-05-28T15:04:02.982401Z","shell.execute_reply":"2025-05-28T15:04:03.014002Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def run_inference(cfg, models, species_ids):\n    \"\"\"Run inference on all test soundscapes\"\"\"\n    test_files = list(Path(cfg.test_soundscapes).glob('*.ogg'))\n    if len(test_files) == 0:\n        test_files = sorted(glob(str(Path('/kaggle/input/birdclef-2025/train_soundscapes') / '*.ogg')))[:10]\n    \n    print(f\"Found {len(test_files)} test soundscapes\")\n\n    all_row_ids = []\n    all_predictions = []\n\n    with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:\n        results = list(\n        executor.map(\n            predict_on_spectrogram,\n            test_files,\n            itertools.repeat(models),\n            itertools.repeat(cfg),\n            itertools.repeat(species_ids)\n        )\n    )\n\n    for rids, preds in results:\n        all_row_ids.extend(rids)\n        all_predictions.extend(preds)\n    \n    return all_row_ids, all_predictions\n\ndef create_submission(row_ids, predictions, species_ids, cfg):\n    \"\"\"Create submission dataframe\"\"\"\n    print(\"Creating submission dataframe...\")\n\n    submission_dict = {'row_id': row_ids}\n    \n    for i, species in enumerate(species_ids):\n        submission_dict[species] = [pred[i] for pred in predictions]\n\n    submission_df = pd.DataFrame(submission_dict)\n\n    submission_df.set_index('row_id', inplace=True)\n\n    sample_sub = pd.read_csv(cfg.submission_csv, index_col='row_id')\n\n    missing_cols = set(sample_sub.columns) - set(submission_df.columns)\n    if missing_cols:\n        print(f\"Warning: Missing {len(missing_cols)} species columns in submission\")\n        for col in missing_cols:\n            submission_df[col] = 0.0\n\n    submission_df = submission_df[sample_sub.columns]\n\n    submission_df = submission_df.reset_index()\n    \n    return submission_df\n\n\ndef smooth_submission(submission_path):\n        \"\"\"\n        Post-process the submission CSV by smoothing predictions to enforce temporal consistency.\n        \n        For each soundscape (grouped by the file name part of 'row_id'), each row's predictions\n        are averaged with those of its neighbors using defined weights.\n        \n        :param submission_path: Path to the submission CSV file.\n        \"\"\"\n        print(\"Smoothing submission predictions...\")\n        sub = pd.read_csv(submission_path)\n        cols = sub.columns[1:]\n        # Extract group names by splitting row_id on the last underscore\n        groups = sub['row_id'].str.rsplit('_', n=1).str[0].values\n        unique_groups = np.unique(groups)\n        \n        for group in unique_groups:\n            # Get indices for the current group\n            idx = np.where(groups == group)[0]\n            sub_group = sub.iloc[idx].copy()\n            predictions = sub_group[cols].values\n            new_predictions = predictions.copy()\n            \n            if predictions.shape[0] > 1:\n                # Smooth the predictions using neighboring segments\n                new_predictions[0] = (predictions[0] * 0.8) + (predictions[1] * 0.2)\n                new_predictions[-1] = (predictions[-1] * 0.8) + (predictions[-2] * 0.2)\n                for i in range(1, predictions.shape[0]-1):\n                    new_predictions[i] = (predictions[i-1] * 0.2) + (predictions[i] * 0.6) + (predictions[i+1] * 0.2)\n            # Replace the smoothed values in the submission dataframe\n            sub.iloc[idx, 1:] = new_predictions\n        \n        sub.to_csv(submission_path, index=False)\n        print(f\"Smoothed submission saved to {submission_path}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:04:03.015958Z","iopub.execute_input":"2025-05-28T15:04:03.016345Z","iopub.status.idle":"2025-05-28T15:04:03.037104Z","shell.execute_reply.started":"2025-05-28T15:04:03.016251Z","shell.execute_reply":"2025-05-28T15:04:03.035997Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def main():\n    start_time = time.time()\n    print(\"Starting BirdCLEF-2025 inference...\")\n\n    models = load_models(cfg, num_classes)\n    \n    if not models:\n        print(\"No models found! Please check model paths.\")\n        return\n    \n    print(f\"Model usage: {'Single model' if len(models) == 1 else f'Ensemble of {len(models)} models'}\")\n\n    row_ids, predictions = run_inference(cfg, models, species_ids)\n\n    submission_df = create_submission(row_ids, predictions, species_ids, cfg)\n\n    submission_path = 'submission.csv'\n    submission_df.to_csv(submission_path, index=False)\n    print(f\"Submission saved to {submission_path}\")\n\n    smooth_submission(submission_path)\n    \n    end_time = time.time()\n    print(f\"Inference completed in {(end_time - start_time)/60:.2f} minutes\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:04:03.038125Z","iopub.execute_input":"2025-05-28T15:04:03.038384Z","iopub.status.idle":"2025-05-28T15:04:03.067694Z","shell.execute_reply.started":"2025-05-28T15:04:03.038365Z","shell.execute_reply":"2025-05-28T15:04:03.066369Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:04:03.068851Z","iopub.execute_input":"2025-05-28T15:04:03.069212Z","iopub.status.idle":"2025-05-28T15:04:41.619368Z","shell.execute_reply.started":"2025-05-28T15:04:03.069185Z","shell.execute_reply":"2025-05-28T15:04:41.618431Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport warnings\nimport logging\nimport time\nimport math\nimport cv2\nfrom pathlib import Path\nimport joblib\n\nimport numpy as np\nimport pandas as pd\nimport librosa\nimport soundfile as sf\nfrom soundfile import SoundFile \nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.cuda.amp import autocast, GradScaler\nimport timm\nfrom tqdm.auto import tqdm\nfrom glob import glob\nimport torchaudio\nimport random\nimport itertools\nfrom typing import Union\n\nimport concurrent.futures\n\nwarnings.filterwarnings(\"ignore\")\nlogging.basicConfig(level=logging.ERROR)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CFG:\n    \n    seed = 42\n    print_freq = 100\n    num_workers = 4\n\n    stage = 'train_bce'\n\n    train_datadir = '/kaggle/input/birdclef-2025/train_audio'\n    train_csv = '/kaggle/input/birdclef-2025/train.csv'\n    test_soundscapes = '/kaggle/input/birdclef-2025/test_soundscapes'\n    submission_csv = '/kaggle/input/birdclef-2025/sample_submission.csv'\n    taxonomy_csv = '/kaggle/input/birdclef-2025/taxonomy.csv'\n    model_files = ['/kaggle/input/modelfold0-latesubmission1/model_fold0.pth'\n                  ]\n \n    model_name = 'seresnext26t_32x4d'  \n    pretrained = False\n    in_channels = 1\n\n    \n    SR = 32000\n    target_duration = 5\n    train_duration = 7\n    infer_duration = 5\n\n    N_FFT = 2048\n    HOP_LENGTH = 512  # 417\n    N_MELS = 256\n    FMIN = 20\n    FMAX = 16000\n    normal = 80\n    \n    device = 'cpu'\n\ncfg = CFG()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Using device: {cfg.device}\")\nprint(f\"Loading taxonomy data...\")\ntaxonomy_df = pd.read_csv(cfg.taxonomy_csv)\nspecies_ids = taxonomy_df['primary_label'].tolist()\nnum_classes = len(species_ids)\nprint(f\"Number of classes: {num_classes}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def set_seed(seed=42):\n    \"\"\"\n    Set seed for reproducibility\n    \"\"\"\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nset_seed(cfg.seed)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class AttBlockV2(nn.Module):\n    def __init__(self, in_features: int, out_features: int, activation=\"linear\"):\n        super().__init__()\n\n        self.activation = activation\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True,\n        )\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True,\n        )\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n\n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.tanh(self.att(x)), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == \"linear\":\n            return x\n        elif self.activation == \"sigmoid\":\n            return torch.sigmoid(x)\n\n\ndef init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.0)\n\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.0)\n    bn.weight.data.fill_(1.0)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BirdCLEFModel(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.cfg = cfg\n        \n        taxonomy_df = pd.read_csv('/kaggle/input/birdclef-2025/taxonomy.csv')\n        self.num_classes = len(taxonomy_df)\n\n        self.bn0 = nn.BatchNorm2d(cfg.N_MELS)\n        \n        base_model = timm.create_model(\n            cfg.model_name,\n            pretrained=cfg.pretrained,\n            in_chans=cfg.in_channels,\n            drop_rate=0.2,\n            drop_path_rate=0.2\n        )\n\n        layers = list(base_model.children())[:-2]\n        self.encoder = nn.Sequential(*layers)\n        \n        if \"efficientnet\" in self.cfg.model_name:\n            in_features = base_model.classifier.in_features\n        elif \"eca\" in self.cfg.model_name:\n            in_features = base_model.head.fc.in_features\n        elif \"res\" in self.cfg.model_name:\n            in_features = base_model.fc.in_features\n            \n        \n        self.fc1 = nn.Linear(in_features, in_features, bias=True)\n        self.att_block = AttBlockV2(in_features, self.num_classes, activation=\"sigmoid\")\n\n        self.melspec_transform = torchaudio.transforms.MelSpectrogram(\n            sample_rate=self.cfg.SR,\n            hop_length=self.cfg.HOP_LENGTH,\n            n_mels=self.cfg.N_MELS,\n            f_min=self.cfg.FMIN,\n            f_max=self.cfg.FMAX,\n            n_fft=self.cfg.N_FFT,\n            center=True,\n            pad_mode=\"constant\",\n            norm=\"slaney\",\n            onesided=True,\n            mel_scale=\"htk\",\n        )\n        if self.cfg.device == \"cuda\":\n            self.melspec_transform = self.melspec_transform.cuda()\n        else:\n            self.melspec_transform = self.melspec_transform.cpu()\n\n        self.db_transform = torchaudio.transforms.AmplitudeToDB(\n            stype=\"power\", top_db=80\n        )\n\n\n    def extract_feature(self,x):\n        x = x.permute((0, 1, 3, 2))\n        frames_num = x.shape[2]\n        \n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n        \n        # if self.training:\n        #    x = self.spec_augmenter(x)\n        \n        x = x.transpose(2, 3)\n        # (batch_size, channels, freq, frames)\n        x = self.encoder(x)\n        \n        # (batch_size, channels, frames)\n        x = torch.mean(x, dim=2)\n        \n        # channel smoothing\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n        \n        x = F.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.5, training=self.training)\n        return x, frames_num\n        \n    @torch.cuda.amp.autocast(enabled=False)\n    def transform_to_spec(self, audio):\n\n        audio = audio.float()\n        \n        spec = self.melspec_transform(audio)\n        spec = self.db_transform(spec)\n\n        if self.cfg.normal == 80:\n            spec = (spec + 80) / 80\n        elif self.cfg.normal == 255:\n            spec = spec / 255\n        else:\n            raise NotImplementedError\n                \n        if self.cfg.in_channels == 3:\n            spec = image_delta(spec)\n        \n        return spec\n\n    def forward(self, x):\n\n        with torch.no_grad():\n            x = self.transform_to_spec(x)\n\n        x, frames_num = self.extract_feature(x)\n        \n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n        logit = torch.sum(norm_att * self.att_block.cla(x), dim=2)\n        segmentwise_logit = self.att_block.cla(x).transpose(1, 2)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        return logit\n\n    def infer(self, x, tta_delta=2):\n        with torch.no_grad():\n            x = self.transform_to_spec(x)\n        x,_ = self.extract_feature(x)\n        time_att = torch.tanh(self.att_block.att(x))\n        feat_time = x.size(-1)\n        start = (\n            feat_time / 2 - feat_time * (self.cfg.infer_duration / self.cfg.train_duration) / 2\n        )\n        end = start + feat_time * (self.cfg.infer_duration / self.cfg.train_duration)\n        start = int(start)\n        end = int(end)\n        pred = self.attention_infer(start,end,x,time_att)\n\n        start_minus = max(0, start-tta_delta)\n        end_minus=end-tta_delta\n        pred_minus = self.attention_infer(start_minus,end_minus,x,time_att)\n\n        start_plus = start+tta_delta\n        end_plus=min(feat_time, end+tta_delta)\n        pred_plus = self.attention_infer(start_plus,end_plus,x,time_att)\n\n        pred = 0.5*pred + 0.25*pred_minus + 0.25*pred_plus\n        return pred\n        \n    def attention_infer(self,start,end,x,time_att):\n        feat = x[:, :, start:end]\n        # att = torch.softmax(time_att[:, :, start:end], dim=-1)\n        #             print(feat_time, start, end)\n        #             print(att_a.sum(), att.sum(), time_att.shape)\n        framewise_pred = torch.sigmoid(self.att_block.cla(feat))\n        framewise_pred_max = framewise_pred.max(dim=2)[0]\n        # clipwise_output = torch.sum(framewise_pred * att, dim=-1)\n        #logits = torch.sum(\n        #    self.att_block.cla(feat) * att,\n        #    dim=-1,\n        #)\n\n        # return clipwise_output\n        return framewise_pred_max","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import librosa\ndef load_sample(path, cfg):\n    audio, _ = librosa.load(path, sr=cfg.SR)\n    seconds = []\n    audio_length = cfg.SR * cfg.target_duration\n    step = audio_length\n    for i in range(audio_length, len(audio) + step, step):\n        start = max(0, i - audio_length)\n        end = start + audio_length\n        if end > len(audio):\n            pass\n        else:\n            seconds.append(int(end/cfg.SR))\n\n    audio = np.concatenate([audio,audio,audio])\n    audios = []\n    for i,second in enumerate(seconds):\n        end_seconds = int(second)\n        start_seconds = int(end_seconds - cfg.target_duration)\n    \n        end_index = int(cfg.SR * (end_seconds + (cfg.train_duration - cfg.target_duration) / 2) ) + len(audio) // 3\n        start_index = int(cfg.SR * (start_seconds - (cfg.train_duration - cfg.target_duration) / 2) ) + len(audio) // 3\n        end_pad = int(cfg.SR * (cfg.train_duration - cfg.target_duration) / 2) \n        start_pad = int(cfg.SR * (cfg.train_duration - cfg.target_duration) / 2) \n        y = audio[start_index:end_index].astype(np.float32)\n        if i==0:\n            y[:start_pad] = 0\n        elif i==(len(seconds)-1):\n            y[-end_pad:] = 0\n        audios.append(y)\n\n    return audios\n\ndef sigmoid(x):\n    s = 1 / (1 + np.exp(-x))\n    return s","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def find_model_files(cfg):\n    \"\"\"\n    Find all .pth model files in the specified model directory\n    \"\"\"\n    model_files = []\n    \n    model_dir = Path(cfg.model_path)\n    \n    for path in model_dir.glob('**/*.pth'):\n        model_files.append(str(path))\n    \n    return model_files\n\ndef load_models(cfg, num_classes):\n    \"\"\"\n    Load all found model files and prepare them for ensemble\n    \"\"\"\n    models = []\n    \n    # model_files = find_model_files(cfg)\n    model_files = cfg.model_files\n    \n    if not model_files:\n        print(f\"Warning: No model files found under {cfg.model_path}!\")\n        return models\n    \n    print(f\"Found a total of {len(model_files)} model files.\")\n    \n    for i, model_path in enumerate(model_files):\n        # try:\n        print(f\"Loading model: {model_path}\")\n        checkpoint = torch.load(model_path, map_location=torch.device(cfg.device), weights_only=False)\n\n\n        \n        model = BirdCLEFModel(cfg)\n        model.load_state_dict(checkpoint['model_state_dict'])\n        model = model.to(cfg.device)\n        model.eval()\n        model.zero_grad()\n        model.half().float()\n        \n        models.append(model)\n        # except Exception as e:\n        #     print(f\"Error loading model {model_path}: {e}\")\n    \n    return models\n\ndef predict_on_spectrogram(audio_path, models, cfg, species_ids):\n    \"\"\"Process a single audio file and predict species presence for each 5-second segment\"\"\"\n    audio_path = str(audio_path)\n    predictions = []\n    row_ids = []\n    soundscape_id = Path(audio_path).stem\n\n    print(f\"Processing {soundscape_id}\")\n    audio_data = load_sample(audio_path, cfg)\n    for segment_idx, audio_input in enumerate(audio_data):\n        \n        end_time_sec = (segment_idx + 1) * cfg.target_duration\n        row_id = f\"{soundscape_id}_{end_time_sec}\"\n        row_ids.append(row_id)\n        \n        mel_spec = torch.tensor(audio_input, dtype=torch.float32).unsqueeze(0).unsqueeze(0)\n        mel_spec = mel_spec.to(cfg.device)\n        \n        if len(models) == 1:\n            with torch.no_grad():\n                outputs = models[0].infer(mel_spec)\n                final_preds = outputs.squeeze()\n                # final_preds = torch.sigmoid(outputs).cpu().numpy().squeeze()\n\n        else:\n            segment_preds = []\n            for model in models:\n                with torch.no_grad():\n                    outputs = model.infer(mel_spec)\n                    probs = outputs.squeeze()\n                    # probs = torch.sigmoid(outputs).cpu().numpy().squeeze()\n                    segment_preds.append(probs)\n\n            \n            final_preds = np.mean(segment_preds, axis=0)\n                \n        predictions.append(final_preds)\n\n    predictions = np.stack(predictions,axis=0)\n    \n    return row_ids, predictions","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def run_inference(cfg, models, species_ids):\n    \"\"\"Run inference on all test soundscapes\"\"\"\n    test_files = list(Path(cfg.test_soundscapes).glob('*.ogg'))\n    if len(test_files) == 0:\n        test_files = sorted(glob(str(Path('/kaggle/input/birdclef-2025/train_soundscapes') / '*.ogg')))[:10]\n    \n    print(f\"Found {len(test_files)} test soundscapes\")\n\n    all_row_ids = []\n    all_predictions = []\n\n    with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:\n        results = list(\n        executor.map(\n            predict_on_spectrogram,\n            test_files,\n            itertools.repeat(models),\n            itertools.repeat(cfg),\n            itertools.repeat(species_ids)\n        )\n    )\n\n    for rids, preds in results:\n        all_row_ids.extend(rids)\n        all_predictions.extend(preds)\n    \n    return all_row_ids, all_predictions\n\ndef create_submission(row_ids, predictions, species_ids, cfg):\n    \"\"\"Create submission dataframe\"\"\"\n    print(\"Creating submission dataframe...\")\n\n    submission_dict = {'row_id': row_ids}\n    \n    for i, species in enumerate(species_ids):\n        submission_dict[species] = [pred[i] for pred in predictions]\n\n    submission_df = pd.DataFrame(submission_dict)\n\n    submission_df.set_index('row_id', inplace=True)\n\n    sample_sub = pd.read_csv(cfg.submission_csv, index_col='row_id')\n\n    missing_cols = set(sample_sub.columns) - set(submission_df.columns)\n    if missing_cols:\n        print(f\"Warning: Missing {len(missing_cols)} species columns in submission\")\n        for col in missing_cols:\n            submission_df[col] = 0.0\n\n    submission_df = submission_df[sample_sub.columns]\n\n    submission_df = submission_df.reset_index()\n    \n    return submission_df\n\n\ndef smooth_submission(submission_path):\n        \"\"\"\n        Post-process the submission CSV by smoothing predictions to enforce temporal consistency.\n        \n        For each soundscape (grouped by the file name part of 'row_id'), each row's predictions\n        are averaged with those of its neighbors using defined weights.\n        \n        :param submission_path: Path to the submission CSV file.\n        \"\"\"\n        print(\"Smoothing submission predictions...\")\n        sub = pd.read_csv(submission_path)\n        cols = sub.columns[1:]\n        # Extract group names by splitting row_id on the last underscore\n        groups = sub['row_id'].str.rsplit('_', n=1).str[0].values\n        unique_groups = np.unique(groups)\n        \n        for group in unique_groups:\n            # Get indices for the current group\n            idx = np.where(groups == group)[0]\n            sub_group = sub.iloc[idx].copy()\n            predictions = sub_group[cols].values\n            new_predictions = predictions.copy()\n            \n            if predictions.shape[0] > 1:\n                # Smooth the predictions using neighboring segments\n                new_predictions[0] = (predictions[0] * 0.8) + (predictions[1] * 0.2)\n                new_predictions[-1] = (predictions[-1] * 0.8) + (predictions[-2] * 0.2)\n                for i in range(1, predictions.shape[0]-1):\n                    new_predictions[i] = (predictions[i-1] * 0.2) + (predictions[i] * 0.6) + (predictions[i+1] * 0.2)\n            # Replace the smoothed values in the submission dataframe\n            sub.iloc[idx, 1:] = new_predictions\n        \n        sub.to_csv(submission_path, index=False)\n        print(f\"Smoothed submission saved to {submission_path}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def main():\n    start_time = time.time()\n    print(\"Starting BirdCLEF-2025 inference...\")\n\n    models = load_models(cfg, num_classes)\n    \n    if not models:\n        print(\"No models found! Please check model paths.\")\n        return\n    \n    print(f\"Model usage: {'Single model' if len(models) == 1 else f'Ensemble of {len(models)} models'}\")\n\n    row_ids, predictions = run_inference(cfg, models, species_ids)\n\n    submission_df = create_submission(row_ids, predictions, species_ids, cfg)\n\n    submission_path = 'submission2.csv'\n    submission_df.to_csv(submission_path, index=False)\n    print(f\"Submission saved to {submission_path}\")\n\n    smooth_submission(submission_path)\n    \n    end_time = time.time()\n    print(f\"Inference completed in {(end_time - start_time)/60:.2f} minutes\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    main()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<h1 style=\"color: #6cb4e4;  text-align: center;  padding: 0.25em;  border-top: solid 2.5px #6cb4e4;  border-bottom: solid 2.5px #6cb4e4;  background: -webkit-repeating-linear-gradient(-45deg, #f0f8ff, #f0f8ff 3px,#e9f4ff 3px, #e9f4ff 7px);  background: repeating-linear-gradient(-45deg, #f0f8ff, #f0f8ff 3px,#e9f4ff 3px, #e9f4ff 7px);height:45px;\">\n<b>\n《《《Finaly Blending》》》\n</b></h1> ","metadata":{}},{"cell_type":"code","source":"# ------------------------------------------- #\n# [IMPORTANT]\n# * Blending Weight\n# ------------------------------------------- #\nsub_w=[0.75, 0.25]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:04:41.620259Z","iopub.execute_input":"2025-05-28T15:04:41.62053Z","iopub.status.idle":"2025-05-28T15:04:41.624784Z","shell.execute_reply.started":"2025-05-28T15:04:41.62051Z","shell.execute_reply":"2025-05-28T15:04:41.623759Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"list_TARGETs = sorted(os.listdir('/kaggle/input/birdclef-2025/train_audio/'))\nlist_targets_0 = [f'{TARGET} 0' for TARGET in list_TARGETs]\nlist_targets_1 = [f'{TARGET} 1' for TARGET in list_TARGETs]\nlist_targets_2 = [f'{TARGET} 2' for TARGET in list_TARGETs]\n\ndf0 = pd.read_csv(\"/kaggle/working/submission.csv\")\ndf1 = pd.read_csv(\"/kaggle/working/submission1.csv\")\ndf2 = pd.read_csv(\"/kaggle/working/submission2.csv\")\n\ndf0 = df0.rename(columns={TARGET : f'{TARGET} 0' for TARGET in list_TARGETs})\ndf1 = df1.rename(columns={TARGET : f'{TARGET} 1' for TARGET in list_TARGETs})\ndf2 = df2.rename(columns={TARGET : f'{TARGET} 2' for TARGET in list_TARGETs})\n# 依次合并 df0 和 df1，再与 df2 合并\ndf01 = pd.merge(df0, df1, on='row_id')\ndfs = pd.merge(df01, df2, on='row_id')\n\n\nfor i in range(len(list_TARGETs)):\n    dfs[list_TARGETs[i]] = 0.4*(dfs[list_targets_0[i]]*sub_w[0] + sub_w[1]*dfs[list_targets_1[i]]) + 0.6*dfs[list_targets_2[i]] \n             \nfor col0,col1,col2 in zip(list_targets_0, list_targets_1, list_targets_2):\n    del dfs[col0]\n    del dfs[col1]\n    del dfs[col2]\n    \n    \ndfs.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T15:04:41.625738Z","iopub.execute_input":"2025-05-28T15:04:41.62601Z","iopub.status.idle":"2025-05-28T15:04:42.786962Z","shell.execute_reply.started":"2025-05-28T15:04:41.625987Z","shell.execute_reply":"2025-05-28T15:04:42.78603Z"}},"outputs":[],"execution_count":null}]}