{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"},{"sourceId":11828260,"sourceType":"datasetVersion","datasetId":7430593},{"sourceId":11870659,"sourceType":"datasetVersion","datasetId":7459867}],"dockerImageVersionId":31040,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"credit goes to this notebook and author just changed config here ","metadata":{}},{"cell_type":"markdown","source":"https://www.kaggle.com/code/johnyim1570/bird25-weightedblend-nfnet-seresnext-0-878\n","metadata":{}},{"cell_type":"markdown","source":"# 🐦 BirdCLEF 2025: Weighted Blend Inference (LB 0.878)\n\n## 🧠 Summary\n\nThis notebook achieves a public leaderboard score of **0.878** using a **Weighted Blend** approach.\n\n## 🧩 Models Used\n\nThe final prediction is based on the ensemble of the following public models:\n\n- 📘 **[Bird2025 | Single SED Model Inference [LB 0.857]](https://www.kaggle.com/code/i2nfinit3y/bird2025-single-sed-model-inference-lb-0-857)**  \n  by [I2nfinit3y](https://www.kaggle.com/i2nfinit3y)  \n  A strong single SED model serving as the base for one of the ensemble components.\n\n- 🧪 **[Post-Processing with Power Adjustment for Low-Rank](https://www.kaggle.com/code/myso1987/post-processing-with-power-adjustment-for-low-rank)**  \n  by [MYSO](https://www.kaggle.com/myso1987)  \n  This notebook introduced a clever post-processing technique to boost low-confidence predictions via power transformation.\n\n- 🔗 **[Bird25 | WeightedBlend | nfnet + convnextv2 | LB.860](https://www.kaggle.com/code/hideyukizushi/bird25-weightedblend-nfnet-convnextv2-lb-860)**  \n  by [yukiZ](https://www.kaggle.com/hideyukizushi)  \n  Provided the core blending logic used to combine model outputs.\n\n## ⚖️ Weighted Blend Strategy\n\nWe use the weighted average of the two model outputs as follows:\n\n- **nfnet**: 25%\n- **seresnext**: 75%\n\nWhile this blend achieves a high LB score, it may be overfitting to the public test set — a pattern often seen in BirdCLEF competitions from 2022 to 2024. Please consider this when evaluating the results.\n\n---","metadata":{}},{"cell_type":"markdown","source":"# Nfnet","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport time\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport timm\nfrom tqdm import tqdm\nimport torch.nn.functional as F\nimport torchaudio\nimport torchaudio.transforms as AT\nfrom torch.utils.data import Dataset, DataLoader\nfrom contextlib import contextmanager\nfrom typing import Union\nimport concurrent.futures\nimport pickle\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T16:41:24.175336Z","iopub.execute_input":"2025-06-04T16:41:24.175512Z","iopub.status.idle":"2025-06-04T16:41:37.903280Z","shell.execute_reply.started":"2025-06-04T16:41:24.175496Z","shell.execute_reply":"2025-06-04T16:41:37.902471Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def apply_power_to_low_ranked_cols(\n    p: np.ndarray,\n    top_k: int = 30,\n    exponent: Union[int, float] = 2,\n    inplace: bool = True\n) -> np.ndarray:\n    if not inplace:\n        p = p.copy()\n\n    # Identify columns whose max value ranks below `top_k`\n    tail_cols = np.argsort(-p.max(axis=0))[top_k:]\n\n    # Apply the power transformation to those columns\n    p[:, tail_cols] = p[:, tail_cols] ** exponent\n    return p","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T16:41:37.904179Z","iopub.execute_input":"2025-06-04T16:41:37.904581Z","iopub.status.idle":"2025-06-04T16:41:37.909485Z","shell.execute_reply.started":"2025-06-04T16:41:37.904563Z","shell.execute_reply":"2025-06-04T16:41:37.908580Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_audio_dir = '../input/birdclef-2025/test_soundscapes/'\nfile_list = [f for f in sorted(os.listdir(test_audio_dir))]\nfile_list = [file.split('.')[0] for file in file_list if file.endswith('.ogg')]\n\ndebug = False\nif len(file_list) == 0:\n    debug = True\n    debug_st_num = 5\n    debug_num = 8\n    test_audio_dir = '../input/birdclef-2025/train_soundscapes/'\n    file_list = [f for f in sorted(os.listdir(test_audio_dir))]\n    file_list = [file.split('.')[0] for file in file_list if file.endswith('.ogg')]\n    # file_list = file_list[debug_st_num:debug_st_num+debug_num]\n\nprint('Debug mode:', debug)\nprint('Number of test soundscapes:', len(file_list))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T16:41:37.911357Z","iopub.execute_input":"2025-06-04T16:41:37.911853Z","iopub.status.idle":"2025-06-04T16:41:38.049366Z","shell.execute_reply.started":"2025-06-04T16:41:37.911828Z","shell.execute_reply":"2025-06-04T16:41:38.048765Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"wav_sec = 5\nsample_rate = 32000\nmin_segment = sample_rate*wav_sec\n\nclass_labels = sorted(os.listdir('../input/birdclef-2025/train_audio/'))\n\nn_fft=1024\nwin_length=1024\nhop_length=512\nf_min=40\nf_max=15000\nn_mels=128\n\nmel_spectrogram = AT.MelSpectrogram(\n    sample_rate=sample_rate,\n    n_fft=n_fft,\n    win_length=win_length,\n    hop_length=hop_length,\n    center=True,\n    f_min=f_min,\n    f_max=f_max,\n    pad_mode=\"reflect\",\n    power=2.0,\n    norm='slaney',\n    n_mels=n_mels,\n    mel_scale=\"htk\",\n    # normalized=True\n)\n\ndef normalize_std(spec, eps=1e-6):\n    mean = torch.mean(spec)\n    std = torch.std(spec)\n    return torch.where(std == 0, spec-mean, (spec - mean) / (std+eps))\n\ndef audio_to_mel(filepath=None):\n    waveform, sample_rate = torchaudio.load(filepath,backend=\"soundfile\")\n    len_wav = waveform.shape[1]\n    waveform = waveform[0,:].reshape(1, len_wav) # stereo->mono mono->mono\n    PREDS = []\n    for i in range(12):\n        waveform2 = waveform[:,i*sample_rate*5:i*sample_rate*5+sample_rate*5]\n        melspec = mel_spectrogram(waveform2)\n        melspec = torch.log(melspec+1e-6)\n        melspec = normalize_std(melspec)\n        melspec = torch.unsqueeze(melspec, dim=0)\n        \n        PREDS.append(melspec)\n    return torch.vstack(PREDS)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T16:41:38.049975Z","iopub.execute_input":"2025-06-04T16:41:38.050241Z","iopub.status.idle":"2025-06-04T16:41:38.091673Z","shell.execute_reply.started":"2025-06-04T16:41:38.050222Z","shell.execute_reply":"2025-06-04T16:41:38.090954Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n\n\ndef init_weights(model):\n    classname = model.__class__.__name__\n    if classname.find(\"Conv2d\") != -1:\n        nn.init.xavier_uniform_(model.weight, gain=np.sqrt(2))\n        model.bias.data.fill_(0)\n    elif classname.find(\"BatchNorm\") != -1:\n        model.weight.data.normal_(1.0, 0.02)\n        model.bias.data.fill_(0)\n    elif classname.find(\"GRU\") != -1:\n        for weight in model.parameters():\n            if len(weight.size()) > 1:\n                nn.init.orghogonal_(weight.data)\n    elif classname.find(\"Linear\") != -1:\n        model.weight.data.normal_(0, 0.01)\n        model.bias.data.zero_()\n\n\ndef interpolate(x, ratio):\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output, frames_num):\n    output = F.interpolate(\n        framewise_output.unsqueeze(1),\n        size=(frames_num, framewise_output.size(2)),\n        align_corners=True,\n        mode=\"bilinear\").squeeze(1)\n\n    return output\n\n\nclass AttBlockV2(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"linear\"):\n        super().__init__()\n\n        self.activation = activation\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n\n    def forward(self, x):\n        norm_att = torch.softmax(torch.tanh(self.att(x)), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)\n\n\nclass TimmSED(nn.Module):\n    def __init__(self, base_model_name: str, pretrained=False, num_classes=24, in_channels=1, n_mels=24):\n        super().__init__()\n\n        self.bn0 = nn.BatchNorm2d(n_mels)\n\n        base_model = timm.create_model(\n            base_model_name, pretrained=pretrained, in_chans=in_channels)\n        layers = list(base_model.children())[:-2]\n        self.encoder = nn.Sequential(*layers)\n\n        in_features = base_model.num_features\n\n        self.fc1 = nn.Linear(in_features, in_features, bias=True)\n        self.att_block2 = AttBlockV2(\n            in_features, num_classes, activation=\"sigmoid\")\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n        \n\n    def forward(self, input_data):\n        x = input_data.transpose(2,3)\n        x = torch.cat((x,x,x),1)\n\n        x = x.transpose(2, 3)\n\n        x = self.encoder(x)\n        \n        x = torch.mean(x, dim=2)\n\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block2(x)\n        logit = torch.sum(norm_att * self.att_block2.cla(x), dim=2)\n\n        output_dict = {\n            'logit': logit,\n        }\n\n        return output_dict","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T16:41:38.092468Z","iopub.execute_input":"2025-06-04T16:41:38.092706Z","iopub.status.idle":"2025-06-04T16:41:38.106858Z","shell.execute_reply.started":"2025-06-04T16:41:38.092682Z","shell.execute_reply":"2025-06-04T16:41:38.106287Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"base_model_name='eca_nfnet_l0'\npretrained=False\nin_channels=3\n\nMODELS = [f'/kaggle/input/birdclef-2025-sed-models-p/sed{i}.pth' for i in range(3)]\n\nMODELS","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T16:41:38.107452Z","iopub.execute_input":"2025-06-04T16:41:38.107644Z","iopub.status.idle":"2025-06-04T16:41:38.123377Z","shell.execute_reply.started":"2025-06-04T16:41:38.107630Z","shell.execute_reply":"2025-06-04T16:41:38.122653Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"models = []\nfor path in MODELS:\n    model = TimmSED(base_model_name=base_model_name,\n               pretrained=pretrained,\n               num_classes=len(class_labels),\n               in_channels=in_channels,\n               n_mels=n_mels);\n    model.load_state_dict(torch.load(path, weights_only=True, map_location=torch.device('cuda')))\n    model.eval();\n    models.append(model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T16:41:38.123948Z","iopub.execute_input":"2025-06-04T16:41:38.124180Z","iopub.status.idle":"2025-06-04T16:41:42.247675Z","shell.execute_reply.started":"2025-06-04T16:41:38.124164Z","shell.execute_reply":"2025-06-04T16:41:42.247102Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def prediction(afile):    \n    global pred\n    path = test_audio_dir + afile + '.ogg'\n    with torch.inference_mode():\n        sig = audio_to_mel(path)\n        outputs = None\n        for model in models:\n            model.eval()\n            p = model(sig)\n            p = torch.sigmoid(p['logit']).detach().cpu().numpy() \n            p = apply_power_to_low_ranked_cols(p, top_k=30,exponent=2)\n            if outputs is None: outputs = p\n            else: outputs += p\n            \n        outputs /= len(models)\n        chunks = [[] for i in range(12)]\n        for i in range(len(chunks)):        \n            chunk_end_time = (i + 1) * 5\n            row_id = afile + '_' + str(chunk_end_time)\n            pred['row_id'].append(row_id)\n            bird_no = 0\n            for bird in class_labels:         \n                pred[bird].append(outputs[i,bird_no])\n                bird_no += 1\n        gc.collect()\n\ndef custom_collate_fn(batch):\n    # batch is a list of (mel_spectrograms_for_file, file_name_base) tuples\n    all_segments_in_batch = []\n    segment_file_map = [] # To store (file_name_base, start_index_of_its_segments_in_all_segments_list, num_segments)\n\n    current_segment_idx = 0\n    for mels_for_file, file_name_base in batch:\n        num_segments = mels_for_file.shape[0]\n        if num_segments > 0:\n            all_segments_in_batch.append(mels_for_file)\n            segment_file_map.append({\n                'file_name': file_name_base,\n                'start_idx': current_segment_idx,\n                'num_segments': num_segments\n            })\n            current_segment_idx += num_segments\n\n    if not all_segments_in_batch:\n        return torch.empty(0, 1, N_MELS_EXPECTED, TIME_STEPS_PER_SEGMENT_EXPECTED), [] # Return empty tensors if no valid segments in batch\n\n    batched_mels_for_model = torch.cat(all_segments_in_batch, dim=0)\n    return batched_mels_for_model, segment_file_map\n\nclass SoundscapeDataset(Dataset):\n    def __init__(self, files, dir, transform=None):\n        self.files = files\n        self.dir = dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.files)\n    \n    def __getitem__(self, idx):\n        file = self.files[idx]\n        audio_path = os.path.join(self.dir, file + '.ogg')\n        if self.transform:\n            return self.transform(audio_path), file\n        else:\n            return torchaudio.load(filepath,backend=\"soundfile\"), file\n\ntrain_dataset = SoundscapeDataset(file_list, test_audio_dir, audio_to_mel)\ntrain_dataloader = DataLoader(\n    train_dataset,\n    batch_size=16,\n    shuffle=False,\n    num_workers=2,\n    collate_fn=custom_collate_fn\n)\nglobal pred\npred = {'row_id': []}\nfor species_code in class_labels:\n    pred[species_code] = []\n[model.to(\"cuda\") for model in models]\n[model.eval() for model in models]\nfor i, (sig, batch_file_maps) in enumerate(tqdm(train_dataloader)):\n    with torch.inference_mode():\n        sig = sig.to(\"cuda\")\n        outputs = None\n        for model in models:\n            p = model(sig)\n            p = torch.sigmoid(p['logit'])\n            if outputs is None: outputs = p\n            else: outputs += p\n\n        outputs /= len(models)\n        outputs = outputs.detach().cpu().numpy() \n        outputs = apply_power_to_low_ranked_cols(outputs, top_k=30,exponent=2)\n\n        if outputs is not None:\n            for file_map_entry in batch_file_maps:\n                file_name = file_map_entry['file_name']\n                start_idx = file_map_entry['start_idx']\n                num_segments = file_map_entry['num_segments']\n\n                file_segment_predictions = outputs[start_idx : start_idx + num_segments]\n\n                for i in range(num_segments):\n                    chunk_end_time = (i + 1) * 5\n                    row_id = f\"{file_name}_{chunk_end_time}\"\n\n                    pred['row_id'].append(row_id)\n\n                    bird_no = 0\n                    for bird in class_labels:\n                        pred[bird].append(file_segment_predictions[i, bird_no])\n                        bird_no += 1\n\n        if i % 10 == 0:\n            gc.collect()\n            torch.cuda.empty_cache()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T16:44:43.550543Z","iopub.execute_input":"2025-06-04T16:44:43.551291Z","iopub.status.idle":"2025-06-04T17:07:06.748539Z","shell.execute_reply.started":"2025-06-04T16:44:43.551261Z","shell.execute_reply":"2025-06-04T17:07:06.747765Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Results dump","metadata":{}},{"cell_type":"code","source":"results = pd.DataFrame(pred, columns = ['row_id'] + class_labels)\ndisplay(results)\n\npred_pickle_path = '/kaggle/working/nfnet_pred.pkl'\nprint(f\"Saving to: {pred_pickle_path}\")\nwith open(pred_pickle_path, 'wb') as f:\n    pickle.dump(pred, f)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T17:17:01.719580Z","iopub.execute_input":"2025-06-04T17:17:01.719843Z","iopub.status.idle":"2025-06-04T17:17:01.907682Z","shell.execute_reply.started":"2025-06-04T17:17:01.719825Z","shell.execute_reply":"2025-06-04T17:17:01.906961Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Seresnext","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport warnings\nimport logging\nimport time\nimport math\nimport cv2\nfrom pathlib import Path\nimport joblib\n\nimport numpy as np\nimport pandas as pd\nimport librosa\nimport soundfile as sf\nfrom soundfile import SoundFile \nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.cuda.amp import autocast, GradScaler\nimport timm\nfrom tqdm.auto import tqdm\nfrom glob import glob\nimport torchaudio\nimport random\nimport itertools\nfrom typing import Union\n\nimport concurrent.futures\n\nwarnings.filterwarnings(\"ignore\")\nlogging.basicConfig(level=logging.ERROR)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T02:25:13.679106Z","iopub.execute_input":"2025-06-03T02:25:13.679784Z","iopub.status.idle":"2025-06-03T02:25:14.271597Z","shell.execute_reply.started":"2025-06-03T02:25:13.679760Z","shell.execute_reply":"2025-06-03T02:25:14.271040Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CFG:\n    \n    seed = 42\n    print_freq = 100\n    num_workers = 4\n\n    stage = 'train_bce'\n\n    train_datadir = '/kaggle/input/birdclef-2025/train_audio'\n    train_csv = '/kaggle/input/birdclef-2025/train.csv'\n    test_soundscapes = '/kaggle/input/birdclef-2025/test_soundscapes'\n    submission_csv = '/kaggle/input/birdclef-2025/sample_submission.csv'\n    taxonomy_csv = '/kaggle/input/birdclef-2025/taxonomy.csv'\n    model_files = ['/kaggle/input/bird2025-sed-ckpt/sedmodel.pth'\n                  ]\n \n    model_name = 'seresnext26t_32x4d'  \n    pretrained = False\n    in_channels = 1\n\n    \n    SR = 32000\n    target_duration = 5\n    train_duration = 10\n    \n    \n    device = 'cuda'\n\ncfg = CFG()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T02:25:14.272933Z","iopub.execute_input":"2025-06-03T02:25:14.273202Z","iopub.status.idle":"2025-06-03T02:25:14.277509Z","shell.execute_reply.started":"2025-06-03T02:25:14.273175Z","shell.execute_reply":"2025-06-03T02:25:14.276878Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Using device: {cfg.device}\")\nprint(f\"Loading taxonomy data...\")\ntaxonomy_df = pd.read_csv(cfg.taxonomy_csv)\nspecies_ids = taxonomy_df['primary_label'].tolist()\nnum_classes = len(species_ids)\nprint(f\"Number of classes: {num_classes}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T02:25:14.278356Z","iopub.execute_input":"2025-06-03T02:25:14.278605Z","iopub.status.idle":"2025-06-03T02:25:14.311871Z","shell.execute_reply.started":"2025-06-03T02:25:14.278582Z","shell.execute_reply":"2025-06-03T02:25:14.311343Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def set_seed(seed=42):\n    \"\"\"\n    Set seed for reproducibility\n    \"\"\"\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nset_seed(cfg.seed)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T02:25:14.312470Z","iopub.execute_input":"2025-06-03T02:25:14.312643Z","iopub.status.idle":"2025-06-03T02:25:14.319434Z","shell.execute_reply.started":"2025-06-03T02:25:14.312622Z","shell.execute_reply":"2025-06-03T02:25:14.318655Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class AttBlockV2(nn.Module):\n    def __init__(self, in_features: int, out_features: int, activation=\"linear\"):\n        super().__init__()\n\n        self.activation = activation\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True,\n        )\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True,\n        )\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n\n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.tanh(self.att(x)), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == \"linear\":\n            return x\n        elif self.activation == \"sigmoid\":\n            return torch.sigmoid(x)\n\n\ndef init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.0)\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.0)\n    bn.weight.data.fill_(1.0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T02:25:14.321958Z","iopub.execute_input":"2025-06-03T02:25:14.322257Z","iopub.status.idle":"2025-06-03T02:25:14.334505Z","shell.execute_reply.started":"2025-06-03T02:25:14.322241Z","shell.execute_reply":"2025-06-03T02:25:14.333832Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BirdCLEFModel(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.cfg = cfg\n        \n        taxonomy_df = pd.read_csv('/kaggle/input/birdclef-2025/taxonomy.csv')\n        self.num_classes = len(taxonomy_df)\n\n        self.bn0 = nn.BatchNorm2d(cfg['n_mels'])\n        \n        self.backbone = timm.create_model(\n            cfg['model_name'],\n            pretrained=False,\n            in_chans=cfg['in_channels'],\n            drop_rate=0.2,\n            drop_path_rate=0.2,\n        )\n\n        layers = list(self.backbone.children())[:-2]\n        self.encoder = nn.Sequential(*layers)\n        \n        if \"efficientnet\" in self.cfg['model_name']:\n            backbone_out = self.backbone.classifier.in_features\n        elif \"eca\" in self.cfg['model_name']:\n            backbone_out = self.backbone.head.fc.in_features\n        elif \"res\" in self.cfg['model_name']:\n            backbone_out = self.backbone.fc.in_features\n        else:\n            backbone_out = self.backbone.num_features\n            \n        \n        self.fc1 = nn.Linear(backbone_out, backbone_out, bias=True)\n        self.att_block = AttBlockV2(backbone_out, self.num_classes, activation=\"sigmoid\")\n\n        self.melspec_transform = torchaudio.transforms.MelSpectrogram(\n            sample_rate=self.cfg['SR'],\n            hop_length=self.cfg['hop_length'],\n            n_mels=self.cfg['n_mels'],\n            f_min=self.cfg['f_min'],\n            f_max=self.cfg['f_max'],\n            n_fft=self.cfg['n_fft'],\n            pad_mode=\"constant\",\n            norm=\"slaney\",\n            onesided=True,\n            mel_scale=\"htk\",\n        )\n        if self.cfg['device'] == \"cuda\":\n            self.melspec_transform = self.melspec_transform.cuda()\n        else:\n            self.melspec_transform = self.melspec_transform.cpu()\n\n        self.db_transform = torchaudio.transforms.AmplitudeToDB(\n            stype=\"power\", top_db=80\n        )\n\n\n    def extract_feature(self,x):\n        x = x.permute((0, 1, 3, 2))\n        frames_num = x.shape[2]\n        \n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n        \n        # if self.training:\n        #    x = self.spec_augmenter(x)\n        \n        x = x.transpose(2, 3)\n        # (batch_size, channels, freq, frames)\n        x = self.encoder(x)\n        \n        # (batch_size, channels, frames)\n        x = torch.mean(x, dim=2)\n        \n        # channel smoothing\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n        \n        x = F.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.5, training=self.training)\n        return x, frames_num\n        \n    @torch.cuda.amp.autocast(enabled=False)\n    def transform_to_spec(self, audio):\n\n        audio = audio.float()\n        \n        spec = self.melspec_transform(audio)\n        spec = self.db_transform(spec)\n\n        if self.cfg['normal'] == 80:\n            spec = (spec + 80) / 80\n        elif self.cfg['normal'] == 255:\n            spec = spec / 255\n        else:\n            raise NotImplementedError\n                \n        if self.cfg['in_channels'] == 3:\n            spec = image_delta(spec)\n        \n        return spec\n\n    def forward(self, x):\n\n        with torch.no_grad():\n            x = self.transform_to_spec(x)\n\n        x, frames_num = self.extract_feature(x)\n        \n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n        logit = torch.sum(norm_att * self.att_block.cla(x), dim=2)\n        segmentwise_logit = self.att_block.cla(x).transpose(1, 2)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        return torch.logit(clipwise_output)\n\n    def infer(self, x, tta_delta=2):\n        with torch.no_grad():\n            x = self.transform_to_spec(x)\n        x,_ = self.extract_feature(x)\n        time_att = torch.tanh(self.att_block.att(x))\n        feat_time = x.size(-1)\n        start = (\n            feat_time / 2 - feat_time * (self.cfg['infer_duration'] / self.cfg['duration_train']) / 2\n        )\n        end = start + feat_time * (self.cfg['infer_duration'] / self.cfg['duration_train'])\n        start = int(start)\n        end = int(end)\n        pred = self.attention_infer(start,end,x,time_att)\n\n        start_minus = max(0, start-tta_delta)\n        end_minus=end-tta_delta\n        pred_minus = self.attention_infer(start_minus,end_minus,x,time_att)\n\n        start_plus = start+tta_delta\n        end_plus=min(feat_time, end+tta_delta)\n        pred_plus = self.attention_infer(start_plus,end_plus,x,time_att)\n\n        pred = 0.5*pred + 0.25*pred_minus + 0.25*pred_plus\n        return pred\n        \n    def attention_infer(self,start,end,x,time_att):\n        feat = x[:, :, start:end]\n        # att = torch.softmax(time_att[:, :, start:end], dim=-1)\n        #             print(feat_time, start, end)\n        #             print(att_a.sum(), att.sum(), time_att.shape)\n        framewise_pred = torch.sigmoid(self.att_block.cla(feat))\n        framewise_pred_max = framewise_pred.max(dim=2)[0]\n        # clipwise_output = torch.sum(framewise_pred * att, dim=-1)\n        #logits = torch.sum(\n        #    self.att_block.cla(feat) * att,\n        #    dim=-1,\n        #)\n\n        # return clipwise_output\n        return framewise_pred_max","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T02:25:14.335034Z","iopub.execute_input":"2025-06-03T02:25:14.335226Z","iopub.status.idle":"2025-06-03T02:25:14.354252Z","shell.execute_reply.started":"2025-06-03T02:25:14.335212Z","shell.execute_reply":"2025-06-03T02:25:14.353503Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_sample(path, cfg):\n    audio, orig_sr = sf.read(path, dtype=\"float32\")\n    seconds = []\n    audio_length = cfg.SR * cfg.target_duration\n    step = audio_length\n    for i in range(audio_length, len(audio) + step, step):\n        start = max(0, i - audio_length)\n        end = start + audio_length\n        if end > len(audio):\n            pass\n        else:\n            seconds.append(int(end/cfg.SR))\n\n    audio = np.concatenate([audio,audio,audio])\n    audios = []\n    for i,second in enumerate(seconds):\n        end_seconds = int(second)\n        start_seconds = int(end_seconds - cfg.target_duration)\n    \n        end_index = int(cfg.SR * (end_seconds + (cfg.train_duration - cfg.target_duration) / 2) ) + len(audio) // 3\n        start_index = int(cfg.SR * (start_seconds - (cfg.train_duration - cfg.target_duration) / 2) ) + len(audio) // 3\n        end_pad = int(cfg.SR * (cfg.train_duration - cfg.target_duration) / 2) \n        start_pad = int(cfg.SR * (cfg.train_duration - cfg.target_duration) / 2) \n        y = audio[start_index:end_index].astype(np.float32)\n        if i==0:\n            y[:start_pad] = 0\n        elif i==(len(seconds)-1):\n            y[-end_pad:] = 0\n        audios.append(y)\n\n    return audios\n\ndef sigmoid(x):\n    s = 1 / (1 + np.exp(-x))\n    return s","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T02:25:14.355136Z","iopub.execute_input":"2025-06-03T02:25:14.355405Z","iopub.status.idle":"2025-06-03T02:25:14.377955Z","shell.execute_reply.started":"2025-06-03T02:25:14.355380Z","shell.execute_reply":"2025-06-03T02:25:14.377305Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def find_model_files(cfg):\n    \"\"\"\n    Find all .pth model files in the specified model directory\n    \"\"\"\n    model_files = []\n    \n    model_dir = Path(cfg.model_path)\n    \n    for path in model_dir.glob('**/*.pth'):\n        model_files.append(str(path))\n    \n    return model_files\n\ndef load_models(cfg, num_classes):\n    \"\"\"\n    Load all found model files and prepare them for ensemble\n    \"\"\"\n    models = []\n    \n    # model_files = find_model_files(cfg)\n    model_files = cfg.model_files\n    \n    if not model_files:\n        print(f\"Warning: No model files found under {cfg.model_path}!\")\n        return models\n    \n    print(f\"Found a total of {len(model_files)} model files.\")\n    \n    for i, model_path in enumerate(model_files):\n        try:\n            print(f\"Loading model: {model_path}\")\n            checkpoint = torch.load(model_path, map_location=torch.device(cfg.device), weights_only=False)\n            cfg_temp = checkpoint['cfg']\n            cfg_temp['device'] = cfg.device\n            \n            model = BirdCLEFModel(cfg_temp)\n            model.load_state_dict(checkpoint['model_state_dict'])\n            model = model.to(cfg.device)\n            model.eval()\n            model.zero_grad()\n            model.half().float()\n            \n            models.append(model)\n        except Exception as e:\n            print(f\"Error loading model {model_path}: {e}\")\n    \n    return models\n\ndef predict_on_spectrogram(audio_path, models, cfg, species_ids):\n    \"\"\"Process a single audio file and predict species presence for each 5-second segment\"\"\"\n    audio_path = str(audio_path)\n    predictions = []\n    row_ids = []\n    soundscape_id = Path(audio_path).stem\n\n    print(f\"Processing {soundscape_id}\")\n    audio_data = load_sample(audio_path, cfg)\n    for segment_idx, audio_input in enumerate(audio_data):\n        \n        end_time_sec = (segment_idx + 1) * cfg.target_duration\n        row_id = f\"{soundscape_id}_{end_time_sec}\"\n        row_ids.append(row_id)\n        \n        mel_spec = torch.tensor(audio_input, dtype=torch.float32).unsqueeze(0).unsqueeze(0)\n        mel_spec = mel_spec.to(cfg.device)\n        \n        if len(models) == 1:\n            with torch.no_grad():\n                outputs = models[0].infer(mel_spec)\n                final_preds = outputs.squeeze()\n                # final_preds = torch.sigmoid(outputs).cpu().numpy().squeeze()\n\n        else:\n            segment_preds = []\n            for model in models:\n                with torch.no_grad():\n                    outputs = model.infer(mel_spec)\n                    probs = outputs.squeeze()\n                    # probs = torch.sigmoid(outputs).cpu().numpy().squeeze()\n                    segment_preds.append(probs)\n\n            \n            final_preds = np.mean(segment_preds, axis=0)\n                \n        predictions.append(final_preds)\n\n    predictions = np.stack(predictions,axis=0)\n    \n    return row_ids, predictions\n\ndef prediction(afile):    \n    global pred\n    path = test_audio_dir + afile + '.ogg'\n    with torch.inference_mode():\n        sig = audio_to_mel(path)\n        outputs = None\n        for model in models:\n            model.eval()\n            p = model(sig)\n            p = torch.sigmoid(p['logit']).detach().cpu().numpy() \n            p = apply_power_to_low_ranked_cols(p, top_k=30,exponent=2)\n            if outputs is None: outputs = p\n            else: outputs += p\n            \n        outputs /= len(models)\n        chunks = [[] for i in range(12)]\n        for i in range(len(chunks)):        \n            chunk_end_time = (i + 1) * 5\n            row_id = afile + '_' + str(chunk_end_time)\n            pred['row_id'].append(row_id)\n            bird_no = 0\n            for bird in class_labels:         \n                pred[bird].append(outputs[i,bird_no])\n                bird_no += 1\n        gc.collect()\n        \n# class SoundscapeDataset(Dataset):\n#     def __init__(self, files, dir, transform=None):\n#         self.files = files\n#         self.dir = dir\n#         self.transform = transform\n\n#     def __len__(self):\n#         return len(self.files)\n    \n#     def __getitem__(self, idx):\n#         file = self.files[idx]\n#         audio_path = os.path.join(self.dir, file + '.ogg')\n#         if self.transform:\n#             return self.transform(audio_path), file\n#         else:\n#             return torchaudio.load(filepath,backend=\"soundfile\"), file\n\nclass SoundscapeDataset(Dataset):\n    def __init__(self, files, dir, transform=None):\n        self.files = files\n        self.dir = dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.files)\n    \n    def __getitem__(self, idx):\n        file = self.files[idx]\n        audio_path = os.path.join(self.dir, file + '.ogg')\n        \n        # Load raw audio and split into 5-second chunks\n        waveform, sample_rate = torchaudio.load(audio_path, backend=\"soundfile\")\n        waveform = waveform[0,:].reshape(1, -1)  # stereo->mono\n        \n        chunks = []\n        for i in range(12):\n            start = i * sample_rate * 5\n            end = start + sample_rate * 5\n            chunk = waveform[:, start:end]\n            \n            # Pad if needed\n            if chunk.shape[1] < sample_rate * 5:\n                padding = torch.zeros(1, sample_rate * 5 - chunk.shape[1])\n                chunk = torch.cat([chunk, padding], dim=1)\n            \n            chunks.append(chunk)\n        \n        return torch.stack(chunks), file\n\n# Remove the transform parameter since we're not using audio_to_mel anymore\ntrain_dataset = SoundscapeDataset(file_list, test_audio_dir)\ntrain_dataloader = DataLoader(\n    train_dataset,\n    batch_size=32,\n    shuffle=False,\n    num_workers=2,\n    collate_fn=custom_collate_fn\n)\nmodels = load_models(cfg, num_classes)\n\nglobal pred\npred = {'row_id': []}\nfor species_code in class_labels:\n    pred[species_code] = []\n[model.eval() for model in models]\nfor i, (sig, batch_file_maps) in enumerate(tqdm(train_dataloader)):\n    with torch.inference_mode():\n        sig = sig.to(\"cuda\")\n        outputs = None\n        for model in models:\n            p = model(sig)\n            p = torch.sigmoid(p)\n            if outputs is None: outputs = p\n            else: outputs += p\n\n        outputs /= len(models)\n        outputs = outputs.detach().cpu().numpy() \n        outputs = apply_power_to_low_ranked_cols(outputs, top_k=30,exponent=2)\n\n        if outputs is not None:\n            for file_map_entry in batch_file_maps:\n                file_name = file_map_entry['file_name']\n                start_idx = file_map_entry['start_idx']\n                num_segments = file_map_entry['num_segments']\n\n                file_segment_predictions = outputs[start_idx : start_idx + num_segments]\n\n                for i in range(num_segments):\n                    chunk_end_time = (i + 1) * 5\n                    row_id = f\"{file_name}_{chunk_end_time}\"\n\n                    pred['row_id'].append(row_id)\n\n                    bird_no = 0\n                    for bird in class_labels:\n                        pred[bird].append(file_segment_predictions[i, bird_no])\n                        bird_no += 1\n\n        if i % 10 == 0:\n            gc.collect()\n            torch.cuda.empty_cache()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T03:33:34.350113Z","iopub.execute_input":"2025-06-03T03:33:34.350459Z","iopub.status.idle":"2025-06-03T03:47:28.334944Z","shell.execute_reply.started":"2025-06-03T03:33:34.350424Z","shell.execute_reply":"2025-06-03T03:47:28.334065Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Results  dump","metadata":{}},{"cell_type":"code","source":"results = pd.DataFrame(pred, columns = ['row_id'] + class_labels)\ndisplay(results)\n\npred_pickle_path = '/kaggle/working/seresnext_pred.pkl'\nprint(f\"Saving to: {pred_pickle_path}\")\nwith open(pred_pickle_path, 'wb') as f:\n    pickle.dump(pred, f)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T03:21:52.599520Z","iopub.execute_input":"2025-06-03T03:21:52.599908Z","iopub.status.idle":"2025-06-03T03:22:19.175389Z","shell.execute_reply.started":"2025-06-03T03:21:52.599869Z","shell.execute_reply":"2025-06-03T03:22:19.174602Z"}},"outputs":[],"execution_count":null}]}