{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"},{"sourceId":12056512,"sourceType":"datasetVersion","datasetId":7127464}],"dockerImageVersionId":31012,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Inference Notebook with 8 Models Ensembling","metadata":{}},{"cell_type":"markdown","source":"- Model Summary\n\n| Model No | Publ LB Score | Arch | Segment Backbone | Global Backbone | Inputs | MH-Attention Feature Fusing | Ensemble Weight | Submission Time | Pseudo Teacher |\n| ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |\n| 1 | 0.873 | [Type 1] Backbone + AttnHead | regnety_008 | - | t-1/t/t+1 (15sec) | q : t, k=v : cat(t-1, t+1) | 0.175 | ~10min | public model LB-0.807 -> own repeative -> public model LB-0.857 |\n| 2 | 0.865 | [Type 1] Backbone + AttnHead | regnety_008 | - | t-1/t/t+1 (15sec) | q : t, k=v : cat(t-1, t+1) | 0.029 | ~10min | public model LB-0.807 -> own repeative |\n| 3 | 0.868 | [Type 1] Backbone + AttnHead | regnety_008 | - | t-1/t/t+1 (15sec) | q : t, k=v : cat(t-1, t+1) | 0.117 | ~10min | public model LB-0.807 -> own repeative -> public model LB-0.850 |\n| 4 | 0.864 | [Type 2] Backbone + freq-temporal multi-directional SED & AttnHead | regnety_008 | - | t-1/t/t+1 (15sec) | q : t, k=v : cat(t-1, t, t+1) | 0.018 | ~10min | public model LB-0.850 |\n| 5 | 0.863 | [Type 2] Backbone + freq-temporal multi-directional SED & AttnHead | tf_efficientnetv2_b0 | - | t-1/t/t+1 (15sec) | q : t, k=v : cat(t-1, t, t+1) | 0.018 | ~10min | public model LB-0.850 |\n| 6 | 0.878 | [Type 2] Backbone + freq-temporal multi-directional SED & AttnHead | eca_nfnet_l0 | - | t-1/t/t+1 (15sec) | q : t, k=v : cat(t-1, t, t+1) | 0.263 | ~26min | public model LB-0.850 |\n| 7 | 0.868 | [Type 3] Dual-Backbone + freq-temporal multi-directional SED & AttnHead | tf_efficientnetv2_b0 | tf_efficientnetv2_b0 | t-1/t/t+1 (15sec) & global (60sec) | Step1: q ← t, kv ← cat(t-1, t, t+1)<br>Step2: q ← Step1 output, kv ← global | 0.088 | ~10min | public model LB-0.850 |\n| 8 | 0.880 | [Type 3] Dual-Backbone + freq-temporal multi-directional SED & AttnHead | eca_nfnet_l0 | tf_efficientnetv2_b0 | t-1/t/t+1 (15sec) & global (60sec) | Step1: q ← t, kv ← cat(t-1, t, t+1)<br>Step2: q ← Step1 output, kv ← global | 0.292 | ~26min | public model LB-0.850 -> public model LB-0857 |\n| ensemble | 0.902 | - | - | - | - | - |- | ~75min | - |  \n\nSegment-wise backbone takes 5sec melspectrogram input with shape (1, 256, 256), Global-wise backbone takes whole 60sec melspectrogram input with shape (1, 256, 512).  \n\n---  \n\n- Postprocess\n\n| category | description |\n| ---- | ---- |\n| Class&Model-Wise Logits Scale Adjustment | Since I used BCE Loss, the scales of predicted logits is independent between the classes and the models. To align the scales of logits between models for fair ensembling, I applied class-wise standard scaling to each model's predicted logits before taking sigmoid and weighted model ensembling. This improved public LB score a bit (+0.003). Other scale insensitive methods like converting to rank was +0.002. |\n| Temporal Smoothing | Not used. For my case, it seemed temporal smoothing is only effective for the models that takes single t segment input, but have (almost) no effect for the models that takes t-1/t/t+1 multi segments inputs like those listed above. Maybe the temporal context are already well incorporated by the adjuscent inputs and model itself. |\n| Segment(Row)-Wise Adjustment | TODO such as softmax with temperature. |\n| Class&File-Wise Adjustment | The methods like Low Rank Power(LRP) Adjustment proposed in the public NB is not applied, since no improvement in score seen in my cases. <br>class-wise standard scaling -> LRP -> ensemble : +0.000<br>LRP -> class-wise standard scaling -> ensemble : -0.001<br>ensemble -> LRP : -0.004|\n","metadata":{}},{"cell_type":"code","source":"!pip install -q --no-deps /kaggle/input/birdclef-2025-private-dataset/audiomentations-0.40.0-py3-none-any.whl\n!pip install -q --no-deps /kaggle/input/birdclef-2025-private-dataset/numpy_minmax-0.3.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n!pip install -q --no-deps /kaggle/input/birdclef-2025-private-dataset/numpy_rms-0.4.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n!pip install -q --no-deps /kaggle/input/birdclef-2025-private-dataset/python_stretch-0.3.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T03:48:54.043375Z","iopub.execute_input":"2025-06-05T03:48:54.043739Z","iopub.status.idle":"2025-06-05T03:49:03.366214Z","shell.execute_reply.started":"2025-06-05T03:48:54.043703Z","shell.execute_reply":"2025-06-05T03:49:03.364437Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from glob import glob\nimport pickle\nimport os\nfrom datetime import datetime\nimport math\nimport warnings\nfrom typing import Union\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchaudio\nfrom tqdm import tqdm\nfrom fastprogress import progress_bar as pb\nimport timm\nimport audiomentations as adm\nimport albumentations as alb\nimport matplotlib.pyplot as plt\nimport einops\nfrom scipy.special import expit, logit\n\ntqdm.pandas()\nwarnings.simplefilter('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-05T03:49:08.692417Z","iopub.execute_input":"2025-06-05T03:49:08.693106Z","iopub.status.idle":"2025-06-05T03:49:58.272300Z","shell.execute_reply.started":"2025-06-05T03:49:08.693074Z","shell.execute_reply":"2025-06-05T03:49:58.271057Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 0. Configs","metadata":{}},{"cell_type":"code","source":"SPECIES = [\n    '1139490', '1192948', '1194042', '126247', '1346504', '134933',\n    '135045', '1462711', '1462737', '1564122', '21038', '21116',\n    '21211', '22333', '22973', '22976', '24272', '24292', '24322',\n    '41663', '41778', '41970', '42007', '42087', '42113', '46010',\n    '47067', '476537', '476538', '48124', '50186', '517119', '523060',\n    '528041', '52884', '548639', '555086', '555142', '566513', '64862',\n    '65336', '65344', '65349', '65373', '65419', '65448', '65547',\n    '65962', '66016', '66531', '66578', '66893', '67082', '67252',\n    '714022', '715170', '787625', '81930', '868458', '963335',\n    'amakin1', 'amekes', 'ampkin1', 'anhing', 'babwar', 'bafibi1',\n    'banana', 'baymac', 'bbwduc', 'bicwre1', 'bkcdon', 'bkmtou1',\n    'blbgra1', 'blbwre1', 'blcant4', 'blchaw1', 'blcjay1', 'blctit1',\n    'blhpar1', 'blkvul', 'bobfly1', 'bobher1', 'brtpar1', 'bubcur1',\n    'bubwre1', 'bucmot3', 'bugtan', 'butsal1', 'cargra1', 'cattyr',\n    'chbant1', 'chfmac1', 'cinbec1', 'cocher1', 'cocwoo1', 'colara1',\n    'colcha1', 'compau', 'compot1', 'cotfly1', 'crbtan1', 'crcwoo1',\n    'crebob1', 'cregua1', 'creoro1', 'eardov1', 'fotfly', 'gohman1',\n    'grasal4', 'grbhaw1', 'greani1', 'greegr', 'greibi1', 'grekis',\n    'grepot1', 'gretin1', 'grnkin', 'grysee1', 'gybmar', 'gycwor1',\n    'labter1', 'laufal1', 'leagre', 'linwoo1', 'littin1', 'mastit1',\n    'neocor', 'norscr1', 'olipic1', 'orcpar', 'palhor2', 'paltan1',\n    'pavpig2', 'piepuf1', 'pirfly1', 'piwtyr1', 'plbwoo1', 'plctan1',\n    'plukit1', 'purgal2', 'ragmac1', 'rebbla1', 'recwoo1', 'rinkin1',\n    'roahaw', 'rosspo1', 'royfly1', 'rtlhum', 'rubsee1', 'rufmot1',\n    'rugdov', 'rumfly1', 'ruther1', 'rutjac1', 'rutpuf1', 'saffin',\n    'sahpar1', 'savhaw1', 'secfly1', 'shghum1', 'shtfly1', 'smbani',\n    'snoegr', 'sobtyr1', 'socfly1', 'solsan', 'soulap1', 'spbwoo1',\n    'speowl1', 'spepar1', 'srwswa1', 'stbwoo2', 'strcuc1', 'strfly1',\n    'strher', 'strowl1', 'tbsfin1', 'thbeup1', 'thlsch3', 'trokin',\n    'tropar', 'trsowl', 'turvul', 'verfly', 'watjac1', 'wbwwre1',\n    'whbant1', 'whbman1', 'whfant1', 'whmtyr1', 'whtdov', 'whttro1',\n    'whwswa1', 'woosto', 'y00678', 'yebela1', 'yebfly1', 'yebsee1',\n    'yecspi2', 'yectyr1', 'yehbla2', 'yehcar1', 'yelori1', 'yeofly1',\n    'yercac1', 'ywcpar'\n]\nN_CLASSES = 206\nassert len(SPECIES) == N_CLASSES\n\nTEST_AUDIO_DIR = '/kaggle/input/birdclef-2025/test_soundscapes'\nUNLABELED_AUDIO_DIR = '/kaggle/input/birdclef-2025/train_soundscapes'\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T03:49:58.274150Z","iopub.execute_input":"2025-06-05T03:49:58.274814Z","iopub.status.idle":"2025-06-05T03:49:58.288373Z","shell.execute_reply.started":"2025-06-05T03:49:58.274785Z","shell.execute_reply":"2025-06-05T03:49:58.287431Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Mel Spectrogram Config","metadata":{}},{"cell_type":"code","source":"# melspec config for model 1\nMELSPEC_CONFIG_V1 = {\n    'n_fft': 2*1024,\n    'hop_length': 2*512,\n    'n_mels': 128,\n    'f_min': 20,\n    'f_max': 16000,\n    'target_shape': (256, 256),\n    'melspec_extra_kwargs': {\n        'pad_mode': 'reflect',\n        'power': 2.0,\n        'norm': 'slaney',\n        'mel_scale': \"htk\"\n    }\n}\n# melspec config for model 2&3\nMELSPEC_CONFIG_V23 = {\n    'n_fft': 1024,\n    'hop_length': 512,\n    'n_mels': 128,\n    'f_min': 50,\n    'f_max': 14000,\n    'target_shape': (256, 256),\n    'melspec_extra_kwargs': {},\n}\n# melspec config for model 4&5&6\nMELSPEC_CONFIG_V456 = {\n    'n_fft': 1024,\n    'hop_length': 512,\n    'n_mels': 128,\n    'f_min': 50,\n    'f_max': 16000,\n    'target_shape': (256, 256),\n    'melspec_extra_kwargs': {\n        'pad_mode': 'reflect',\n        'power': 2.0,\n        'norm': 'slaney',\n        'mel_scale': \"htk\"\n    }\n}\n# melspec config for model 7\nMELSPEC_CONFIG_V7 = {\n    'n_fft': 1024,\n    'hop_length': 512,\n    'n_mels': 128,\n    'f_min': 50,\n    'f_max': 16000,\n    'target_shape': (256, 256),\n    'melspec_extra_kwargs': {\n        'pad_mode': 'reflect',\n        'power': 2.0,\n        'norm': 'slaney',\n        'mel_scale': \"htk\"\n    },\n    'min_max_scale': False,\n}\n# melspec config for model 8\nMELSPEC_CONFIG_V8 = {\n    'n_fft': 2*1024,\n    'hop_length': 2*512,\n    'n_mels': 128,\n    'f_min': 20,\n    'f_max': 16000,\n    'target_shape': (256, 256),\n    'melspec_extra_kwargs': {\n        'pad_mode': 'reflect',\n        'power': 2.0,\n        'norm': 'slaney',\n        'mel_scale': \"htk\"\n    },\n    'min_max_scale': False,\n}\nGLOBAL_SPEC_RESIZE_SIZE = (256, 2*256)\nMELSPEC_CONFIG_LIST = [\n    MELSPEC_CONFIG_V1, MELSPEC_CONFIG_V23, MELSPEC_CONFIG_V23,\n    MELSPEC_CONFIG_V456, MELSPEC_CONFIG_V456, MELSPEC_CONFIG_V456,\n    MELSPEC_CONFIG_V7, MELSPEC_CONFIG_V8,\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T03:49:58.289435Z","iopub.execute_input":"2025-06-05T03:49:58.289811Z","iopub.status.idle":"2025-06-05T03:49:58.356121Z","shell.execute_reply.started":"2025-06-05T03:49:58.289780Z","shell.execute_reply":"2025-06-05T03:49:58.355169Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Model Config","metadata":{}},{"cell_type":"code","source":"# model filepath list\nMODEL_FILEPATH_LIST = [\n    '/kaggle/input/birdclef-2025-private-dataset/specimagebase_exp25_1_bce_sigmoid_27epoch.pth',  # LB 0.873 | regnety_008\n    '/kaggle/input/birdclef-2025-private-dataset/specimagebase_exp8_5-2_bce_sigmoid_3epoch.pth',  # LB 0.865 | regnety_008\n    '/kaggle/input/birdclef-2025-private-dataset/specimagebase_exp18_5-3_bce_sigmoid_30epoch.pth',  # LB 0.868 | regnety_008\n    '/kaggle/input/birdclef-2025-private-dataset/specimagebase_exp19_3_bce_sigmoid_49epoch.pth',  # LB 0.864 | regnety_008 SED\n    '/kaggle/input/birdclef-2025-private-dataset/specimagebase_exp19_2_bce_sigmoid_32epoch.pth',  # LB 0.863 | tf_efficientnetv2_b0 SED\n    '/kaggle/input/birdclef-2025-private-dataset/specimagebase_exp19_1-2_bce_sigmoid_55epoch.pth',  # LB 0.878 | eca_nfnet_l0 SED\n    '/kaggle/input/birdclef-2025-private-dataset/specimagebase_exp23_2-2_bce_sigmoid_6epoch.pth',  # LB 0.868 | tf_efficientnetv2_b0 SED | global\n    '/kaggle/input/birdclef-2025-private-dataset/specimagebase_exp24_4-4_bce_sigmoid_21epoch.pth',  # LB 0.880 | eca_nfnet_l0 x tf_efficientnetv2_b0 SED | global\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T03:49:58.358389Z","iopub.execute_input":"2025-06-05T03:49:58.358724Z","iopub.status.idle":"2025-06-05T03:49:58.380959Z","shell.execute_reply.started":"2025-06-05T03:49:58.358699Z","shell.execute_reply":"2025-06-05T03:49:58.379782Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Postprocess Config","metadata":{}},{"cell_type":"code","source":"############### Low Rank Power Adjustment #################\nAPPLY_LOW_RANK_POWER_SINGLE_MODEL = False  # whether to apply LRP to each model's prediction\nAPPLY_LOW_RANK_POWER_ENSEMBLE = False  # whether to apply LRP to ensamble result\nLRP_TOP_K = 30\nLRP_EXPONENT = 2\n\n############## Ensembling ##################\n# For my case, prob_weighted_sum > rank_weighted_sum > logits_weighted_sum (the LB score differences were slight)\n# ENSEMBLE_METHOD = 'logits_weighted_sum'\nENSEMBLE_METHOD = 'prob_weighted_sum'\n# ENSEMBLE_METHOD = 'rank_weighted_sum'\n\n# ensemble weight list\nENS_WEIGHT_LIST = np.array([0.3, 0.05, 0.2, 0.03, 0.03, 0.45, 0.15, 0.5])\nENS_WEIGHT_LIST = ENS_WEIGHT_LIST / ENS_WEIGHT_LIST.sum()\nprint(ENS_WEIGHT_LIST)\nassert np.round(sum(ENS_WEIGHT_LIST), 6) == 1, sum(ENS_WEIGHT_LIST)\nassert len(ENS_WEIGHT_LIST) == len(MELSPEC_CONFIG_LIST)\n\n############## Temporal Smoothing ##############\n# For my case, it seemed temporal smoothing is only effective for the models that takes single t segment feature(spectrogram) as input, \n# but have no effect for the models that takes t-1/t/t+1 multi segments inputs like those used above.\nAPPLY_TEMPORAL_SMOOTHING = False\nADJ_MIX_RATIO = 0.1\nCENTER_MIX_RATIO = 1.0 - 2 * ADJ_MIX_RATIO","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T03:49:58.381926Z","iopub.execute_input":"2025-06-05T03:49:58.382240Z","iopub.status.idle":"2025-06-05T03:49:58.412233Z","shell.execute_reply.started":"2025-06-05T03:49:58.382211Z","shell.execute_reply":"2025-06-05T03:49:58.411171Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1. Prepare Data","metadata":{}},{"cell_type":"code","source":"test_files = glob(os.path.join(TEST_AUDIO_DIR, '**/*.ogg'), recursive=True)\nif len(test_files) == 0:\n    print(f'loading from {UNLABELED_AUDIO_DIR}')\n    test_files = glob(os.path.join(UNLABELED_AUDIO_DIR, '**/*.ogg'), recursive=True)[:50]\n\ndf_test_files = pd.DataFrame(test_files, columns=['file_path'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T03:49:58.413412Z","iopub.execute_input":"2025-06-05T03:49:58.414185Z","iopub.status.idle":"2025-06-05T03:50:08.717903Z","shell.execute_reply.started":"2025-06-05T03:49:58.414157Z","shell.execute_reply":"2025-06-05T03:50:08.716503Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test_files","metadata":{"trusted":true,"scrolled":true,"execution":{"iopub.status.busy":"2025-06-05T03:50:08.719961Z","iopub.execute_input":"2025-06-05T03:50:08.720416Z","iopub.status.idle":"2025-06-05T03:50:08.750937Z","shell.execute_reply.started":"2025-06-05T03:50:08.720352Z","shell.execute_reply":"2025-06-05T03:50:08.750023Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Model","metadata":{}},{"cell_type":"markdown","source":"### 2.1. Attention Pooling Head","metadata":{}},{"cell_type":"code","source":"class AttnPooling(nn.Module):\n    \"\"\"Attention-based temporal pooling layer for aggregating time-series features.\"\"\"\n\n    def __init__(self, in_features, n_classes, intermediate_dim: int = 128):\n        super().__init__()\n        self.attn = nn.Sequential(\n            nn.Linear(in_features, intermediate_dim),\n            nn.Tanh(),\n            nn.Linear(intermediate_dim, 1)\n        )\n        self.classifier = nn.Linear(in_features, n_classes)\n\n    def forward(self, x):\n        \"\"\"\n        Forward pass for attention pooling.\n\n        Args:\n            x (torch.Tensor): Input tensor of shape (B, T, F), where\n                B is the batch size,\n                T is the number of time steps (sequence length),\n                F is the feature dimension.\n\n        Returns:\n            Tuple[torch.Tensor, torch.Tensor]:\n                - clip_logits (torch.Tensor): Pooled logits of shape (B, n_classes).\n                - attn_weights (torch.Tensor): Attention weights of shape (B, T).\n        \"\"\"\n        attn_weights = self.attn(x)  # (B, T, 1)\n        attn_weights = F.softmax(attn_weights, dim=1)  # (B, T, 1)\n\n        x_weighted = (x * attn_weights).sum(dim=1)  # (B, F)\n        clip_logits = self.classifier(x_weighted)  # (B, C)\n\n        return clip_logits, attn_weights.squeeze(-1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T03:50:08.751944Z","iopub.execute_input":"2025-06-05T03:50:08.752208Z","iopub.status.idle":"2025-06-05T03:50:08.761280Z","shell.execute_reply.started":"2025-06-05T03:50:08.752187Z","shell.execute_reply":"2025-06-05T03:50:08.760354Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2.2. Main Classifier","metadata":{}},{"cell_type":"code","source":"class AudioClassifier(nn.Module):\n    \"\"\"Audio classification model using a timm backbone and attention-based pooling.\n    For Model 1/2/3.\n    \"\"\"\n\n    def __init__(self, num_classes=N_CLASSES, model_name='efficientnet_b0', in_channs=1, pretrained=False, drop_rate=0.0, drop_path_rate=0.0, num_heads: int = 8):\n        super().__init__()\n        self.backbone = timm.create_model(\n            model_name,\n            pretrained=pretrained,\n            in_chans=in_channs,\n            features_only=True,\n            drop_rate=drop_rate,\n            drop_path_rate=drop_path_rate\n        )\n        self.in_channs = in_channs\n        self.feat_dim = self.backbone.feature_info.channels()[-1]\n        self.norm = nn.LayerNorm(self.feat_dim)\n        self.q_proj = nn.Linear(self.feat_dim, self.feat_dim)\n        self.kv_proj = nn.Linear(self.feat_dim, self.feat_dim)\n        self.norm_post_attn = nn.LayerNorm(self.feat_dim // num_heads)\n        self.drop_rate = drop_rate\n        self.num_heads = num_heads\n        assert self.feat_dim % self.num_heads == 0\n        self.pooling = AttnPooling(self.feat_dim, num_classes)\n\n    def forward(self, spec=None, pre_calc_feat=None, return_attn_weight=False, **kwargs):\n        \"\"\"\n        Forward pass of the classifier.\n\n        Either `spec` or `pre_calc_feat` must be provided. Features from \n        three temporal segments (t-1, t, t+1) are fused using multi-head attention.\n\n        Args:\n            spec (torch.Tensor, optional): Input spectrogram tensor of shape \n                (B, 3, H, W), where 3 represents t-1, t, t+1 segments.\n            pre_calc_feat (torch.Tensor, optional): Precomputed features of shape \n                (B, 3, C_feat, H_feat, W_feat).\n            return_attn_weight (bool): If True, returns attention weights along with logits.\n            **kwargs: Additional arguments for compatibility.\n\n        Returns:\n            torch.Tensor: Logits of shape (B, num_classes) if `return_attn_weight` is False.\n            Tuple[torch.Tensor, torch.Tensor]: Logits and attention weights if `return_attn_weight` is True.\n        \"\"\"\n        # 1. calculate features with backbone\n        if pre_calc_feat is not None:\n            bs, n_f, f_ch, f_h, f_w = pre_calc_feat.shape\n            assert n_f == 3\n            assert f_ch == self.feat_dim\n            feat = einops.rearrange(pre_calc_feat, 'bs ns c2 h w -> bs ns (h w) c2', bs=bs, ns=3)  # (bs, n_segments(3), seq_len, dim)\n        elif spec is not None:\n            bs, n_f, h, w = spec.shape\n            assert n_f == 3\n            spec = einops.rearrange(spec, 'bs ns h w -> (bs ns) 1 h w').repeat((1, self.in_channs, 1, 1))\n            feat = self.backbone(spec)[-1]  # (bs x n_segments(3), C', H', W')\n            feat = einops.rearrange(feat, '(bs ns) c2 h w -> bs ns (h w) c2', bs=bs, nf=3)  # (bs, n_segments(3), seq_len, dim)        \n        # 2. fusing center t chunk feature with adjuscent t-1/t+1 chunk features\n        q = feat[:, 1, :, :]  # (bs, seq_len, dim)\n        kv = torch.cat([feat[:, 0, :, :], feat[:, 2, :, :]], dim=1)  # (bs, 2xseq_len, dim)\n        q = F.dropout(F.gelu(self.q_proj(self.norm(q))), p=self.drop_rate)\n        kv = F.dropout(F.gelu(self.kv_proj(self.norm(kv))), p=self.drop_rate)\n        q_mh = einops.rearrange(q, 'b sl (nh d) -> b nh sl d', nh=self.num_heads, d=self.feat_dim // self.num_heads)  # (bs, num_heads, seq_len, dim_head)\n        kv_mh = einops.rearrange(kv, 'b sl (nh d) -> b nh sl d', nh=self.num_heads, d=self.feat_dim // self.num_heads)  # (bs, num_heads, 2xseq_len, dim_head)\n        feat_fused = q_mh + F.scaled_dot_product_attention(query=q_mh, key=kv_mh, value=kv_mh, dropout_p=self.drop_rate, is_causal=False)  # (bs, num_heads, seq_len, dim_head)\n        feat_fused = self.norm_post_attn(feat_fused)\n        # 3. forward attention pooling heads\n        feat_fused = einops.rearrange(feat_fused, 'b nh sl d -> b sl (nh d)')\n        logits, attn_weights = self.pooling(feat_fused)\n        if return_attn_weight:\n            return logits, attn_weights\n        else:\n            return logits\n\n    def forward_features(self, spec):\n        \"\"\"\n        Extracts features from a single spectrogram image.\n\n        Args:\n            spec (torch.Tensor): Spectrogram input tensor of shape (B, 1, H, W).\n\n        Returns:\n            torch.Tensor: Feature map of shape (B, C_feat, H_feat, W_feat) extracted by the backbone.\n        \"\"\"\n        bs, in_ch, h, w = spec.shape\n        assert in_ch == 1\n        spec = spec.repeat((1, self.in_channs, 1, 1))\n        feat = self.backbone(spec)[-1]  # (bs, C', H', W')\n        return feat\n\n\nclass AudioClassifierSED(nn.Module):\n    \"\"\"Sound Event Detection (SED) model with timm backbone and attention-based fusion and pooling.\n    For Model 4/5/6.\n    \"\"\"\n\n    def __init__(self, num_classes=N_CLASSES, model_name='eca_nfnet_l0', in_channs=1, pretrained=False, drop_rate=0.0, drop_path_rate=0.0, num_heads: int = 8, post_backbone_dim: int = 256):\n        super().__init__()\n        self.backbone = timm.create_model(\n            model_name,\n            pretrained=pretrained,\n            in_chans=in_channs,\n            features_only=True,\n            drop_rate=drop_rate,\n            drop_path_rate=drop_path_rate\n        )\n        self.in_channs = in_channs\n        self.feat_dim = self.backbone.feature_info.channels()[-1]\n        self.post_backbone_dim = post_backbone_dim\n        self.norm = nn.LayerNorm(self.feat_dim)\n        self.q_proj = nn.Linear(self.feat_dim, post_backbone_dim)\n        self.kv_proj = nn.Linear(self.feat_dim, post_backbone_dim)\n        self.norm_post_attn = nn.LayerNorm(post_backbone_dim // num_heads)\n        self.drop_rate = drop_rate\n        self.num_heads = num_heads\n        assert self.feat_dim % self.num_heads == 0\n        self.pooling = AttnPooling(post_backbone_dim, num_classes)\n        self.temporal_freq_fc = nn.Linear(post_backbone_dim, num_classes)\n\n    def forward(self, spec=None, pre_calc_feat=None, only_return_clip_logits=True, **kwargs):\n        \"\"\"\n        Forward pass for SED. Performs multi-head attention fusion over temporal segments \n        and computes both clip-level and frame-level predictions.\n\n        Args:\n            spec (torch.Tensor, optional): Input spectrogram tensor of shape \n                (B, 3, H, W), where the 3 segments represent (t-1, t, t+1).\n            pre_calc_feat (torch.Tensor, optional): Precomputed backbone features of shape \n                (B, 3, C_feat, H_feat, W_feat).\n            only_return_clip_logits (bool): If True, only returns clip-level logits; \n                if False, returns a dictionary including frame-level and attention outputs.\n            **kwargs: Additional keyword arguments (unused but accepted for flexibility).\n\n        Returns:\n            torch.Tensor or Dict[str, torch.Tensor]: \n                - If `only_return_clip_logits` is True: Returns clip-level logits of shape (B, num_classes).\n                - Else: Returns a dictionary with:\n                    - \"clip_logits\" (torch.Tensor): (B, num_classes)\n                    - \"frame_logits\" (torch.Tensor): (B, H'+W', num_classes)\n                    - \"attn_weights\" (torch.Tensor): Attention weights from pooling.\n        \"\"\"\n        # 1. calculate features with backbone\n        if pre_calc_feat is not None:\n            bs, n_f, f_ch, f_h, f_w = pre_calc_feat.shape\n            assert n_f == 3\n            assert f_ch == self.feat_dim\n            feat = einops.rearrange(pre_calc_feat, 'bs ns c2 h w -> bs ns (h w) c2', bs=bs, ns=3)  # (bs, n_segments(3), seq_len, dim)\n        elif spec is not None:\n            bs, n_f, h, w = spec.shape\n            assert n_f == 3\n            spec = einops.rearrange(spec, 'bs ns h w -> (bs ns) 1 h w').repeat((1, self.in_channs, 1, 1))\n            feat = self.backbone(spec)[-1]  # (bs x n_segments(3), C', H', W')\n            _, _, f_h, f_w = feat.shape\n            feat = einops.rearrange(feat, '(bs ns) c2 h w -> bs ns (h w) c2', bs=bs, nf=3)  # (bs, n_segments(3), seq_len, dim)\n        # 2. fusing center t chunk feature with adjuscent t-1/t/t+1 chunk features\n        q = feat[:, 1, :, :]  # (bs, seq_len, dim)\n        kv = torch.cat([feat[:, 0, :, :], feat[:, 1, :, :], feat[:, 2, :, :]], dim=1)  # (bs, 3xseq_len, dim)\n        q = F.dropout(F.gelu(self.q_proj(self.norm(q))), p=self.drop_rate)\n        kv = F.dropout(F.gelu(self.kv_proj(self.norm(kv))), p=self.drop_rate)\n        q_mh = einops.rearrange(q, 'b sl (nh d) -> b nh sl d', nh=self.num_heads, d=self.post_backbone_dim // self.num_heads)  # (bs, num_heads, seq_len, dim_head)\n        kv_mh = einops.rearrange(kv, 'b sl (nh d) -> b nh sl d', nh=self.num_heads, d=self.post_backbone_dim // self.num_heads)  # (bs, num_heads, 3xseq_len, dim_head)\n        feat_fused = q_mh + F.scaled_dot_product_attention(query=q_mh, key=kv_mh, value=kv_mh, dropout_p=self.drop_rate, is_causal=False)  # (bs, num_heads, seq_len, dim_head)\n        feat_fused = self.norm_post_attn(feat_fused)\n        # 3. forward SED & attention pooling heads\n        feat_fused = einops.rearrange(feat_fused, 'b nh sl d -> b sl (nh d)')  # (bs, seq_len, dim)\n        feat_fused = einops.rearrange(feat_fused, 'b (h w) c -> b h w c', h=f_h, w=f_w)\n        feat_fused = torch.cat([feat_fused.mean(dim=1), feat_fused.mean(dim=2)], dim=1)  # (bs, (H' + W'), dim)\n        frame_logits = self.temporal_freq_fc(feat_fused)  # (B,  (H' + W'), dim) -> (B, (H' + W'), num_classes)\n        clip_logits, attn_weights = self.pooling(feat_fused)  # (bs, num_classes), (bs,  (H' + W')(T))\n        if only_return_clip_logits:\n            return clip_logits\n        else:\n            return {\n                \"clip_logits\": clip_logits,\n                \"frame_logits\": frame_logits,\n                \"attn_weights\": attn_weights,\n            }\n\n    def forward_features(self, spec):\n        \"\"\"\n        Extracts CNN backbone features from a spectrogram.\n\n        Args:\n            spec (torch.Tensor): Spectrogram input of shape (B, 1, H, W).\n\n        Returns:\n            torch.Tensor: Feature tensor of shape (B, C_feat, H_feat, W_feat) from the backbone.\n        \"\"\"\n        bs, in_ch, h, w = spec.shape\n        assert in_ch == 1\n        spec = spec.repeat((1, self.in_channs, 1, 1))\n        feat = self.backbone(spec)[-1]  # (bs, C', H', W')\n        return feat\n\n\nclass AudioClassifierSEDGlobal(nn.Module):\n    \"\"\"Sound Event Detection (SED) model with local and global context fusion via attention.\n    For Model 7/8.\n    \"\"\"\n\n    def __init__(self, num_classes=N_CLASSES, model_name='eca_nfnet_l0', global_model_name=None, in_channs=1, pretrained=False, drop_rate=0.0, drop_path_rate=0.0, num_heads: int = 8, post_backbone_dim: int = 256, use_separate_norm: bool = False):\n        super().__init__()\n        if global_model_name is None:\n            global_model_name = model_name\n        self.backbone = timm.create_model(\n            model_name,\n            pretrained=pretrained,\n            in_chans=in_channs,\n            features_only=True,\n            drop_rate=drop_rate,\n            drop_path_rate=drop_path_rate\n        )\n        self.backbone_global = timm.create_model(\n            global_model_name,\n            pretrained=pretrained,\n            in_chans=in_channs,\n            features_only=True,\n            drop_rate=drop_rate,\n            drop_path_rate=drop_path_rate\n        )\n        self.in_channs = in_channs\n        self.feat_dim = self.backbone.feature_info.channels()[-1]\n        self.feat_dim_global = self.backbone_global.feature_info.channels()[-1]\n        self.post_backbone_dim = post_backbone_dim\n        self.use_separate_norm = use_separate_norm or (self.feat_dim != self.feat_dim_global)\n        self.norm = nn.LayerNorm(self.feat_dim)\n        if self.use_separate_norm:\n            self.norm_kv = nn.LayerNorm(self.feat_dim)\n            self.norm_kv_g = nn.LayerNorm(self.feat_dim_global)\n        else:\n            self.norm_kv = None\n            self.norm_kv_g = None\n        self.q_proj = nn.Linear(self.feat_dim, post_backbone_dim)\n        self.kv_proj = nn.Linear(self.feat_dim, post_backbone_dim)\n        self.kv_proj_g = nn.Linear(self.feat_dim_global, post_backbone_dim)\n        self.norm_post_attn = nn.LayerNorm(post_backbone_dim // num_heads)\n        if self.use_separate_norm:\n            self.norm_post_attn_g = nn.LayerNorm(post_backbone_dim // num_heads)\n        else:\n            self.norm_post_attn_g = None\n        self.drop_rate = drop_rate\n        self.num_heads = num_heads\n        assert self.feat_dim % self.num_heads == 0\n        self.pooling = AttnPooling(post_backbone_dim, num_classes)\n        self.temporal_freq_fc = nn.Linear(post_backbone_dim, num_classes)\n\n    def forward(self, spec=None, spec_global=None, pre_calc_feat=None, pre_calc_g_feat=None, only_return_clip_logits=True, **kwargs):\n        \"\"\"\n        Forward pass of the model with both local and global feature attention fusion.\n\n        Args:\n            spec (torch.Tensor, optional): Local spectrogram input of shape \n                (B, 3, H, W) representing t-1, t, t+1 segments.\n            spec_global (torch.Tensor, optional): Global spectrogram input of shape \n                (B, 1, H_g, W_g), representing the entire clip.\n            pre_calc_feat (torch.Tensor, optional): Precomputed local features of shape \n                (B, 3, C_F, H_F, W_F).\n            pre_calc_g_feat (torch.Tensor, optional): Precomputed global features of shape \n                (B, C_F_G, H_F_G, W_F_G).\n            only_return_clip_logits (bool): If True, only returns clip-level logits.\n                If False, returns full dictionary with frame-level outputs and attention weights.\n            **kwargs: Additional unused keyword arguments.\n\n        Returns:\n            torch.Tensor or Dict[str, torch.Tensor]: \n                - If `only_return_clip_logits` is True: Returns clip-level logits (B, num_classes).\n                - Else: Returns dictionary with:\n                    - \"clip_logits\" (torch.Tensor): (B, num_classes)\n                    - \"frame_logits\" (torch.Tensor): (B, H'+W', num_classes)\n                    - \"attn_weights\" (torch.Tensor): Attention weights after pooling.\n        \"\"\"\n        # 1. calculate features with backbone\n        if pre_calc_feat is not None:\n            bs, n_f, f_ch, f_h, f_w = pre_calc_feat.shape\n            assert n_f == 3\n            assert f_ch == self.feat_dim\n            feat = einops.rearrange(pre_calc_feat, 'bs nf c2 h w -> bs nf (h w) c2', bs=bs, nf=3)  # (bs, n_frames(3), seq_len, dim)\n        elif spec is not None:\n            bs, n_f, h, w = spec.shape\n            assert n_f == 3\n            spec = einops.rearrange(spec, 'bs c h w -> (bs c) 1 h w').repeat((1, self.in_channs, 1, 1))\n            feat = self.backbone(spec)[-1]  # (bs x n_frames(3), C', H', W')\n            _, _, f_h, f_w = feat.shape\n            feat = einops.rearrange(feat, '(bs nf) c2 h w -> bs nf (h w) c2', bs=bs, nf=3)  # (bs, n_frames(3), seq_len, dim)\n        if pre_calc_g_feat is not None:\n            g_feat = pre_calc_g_feat\n        else:\n            g_feat = self.forward_global_features(spec_global)  # (bs, C', H', G_W')\n        norm_q = self.norm\n        norm_post_attn = self.norm_post_attn\n        if self.use_separate_norm:\n            norm_kv, norm_kv_g = self.norm_kv, self.norm_kv_g\n            norm_post_attn_g = self.norm_post_attn_g\n        else:\n            norm_kv = norm_kv_g = self.norm\n            norm_post_attn_g = self.norm_post_attn\n        # 2. fusing center t chunk feature with adjuscent t-1/t/t+1 chunk features\n        q = feat[:, 1, :, :]  # (bs, seq_len, dim)\n        kv = torch.cat([feat[:, 0, :, :], feat[:, 1, :, :], feat[:, 2, :, :]], dim=1)  # (bs, 3xseq_len, dim)\n        q = F.dropout(F.gelu(self.q_proj(norm_q(q))), p=self.drop_rate)\n        kv = F.dropout(F.gelu(self.kv_proj(norm_kv(kv))), p=self.drop_rate)\n        q_mh = einops.rearrange(q, 'b sl (nh d) -> b nh sl d', nh=self.num_heads, d=self.post_backbone_dim // self.num_heads)  # (bs, num_heads, seq_len, dim_head)\n        kv_mh = einops.rearrange(kv, 'b sl (nh d) -> b nh sl d', nh=self.num_heads, d=self.post_backbone_dim // self.num_heads)  # (bs, num_heads, 3xseq_len, dim_head)\n        feat_fused = q_mh + F.scaled_dot_product_attention(query=q_mh, key=kv_mh, value=kv_mh, dropout_p=self.drop_rate, is_causal=False)\n        feat_fused = norm_post_attn(feat_fused)\n        # 3. fusing with global feature\n        q_mh = feat_fused  # (bs, num_heads, seq_len, dim_head)\n        g_feat = einops.rearrange(g_feat, 'b c h w -> b h w c')\n        kv = F.dropout(F.gelu(self.kv_proj_g(norm_kv_g(\n            torch.cat([g_feat.mean(dim=1), g_feat.mean(dim=2)], dim=1)  # (bs, (G_H' + G_W'), dim)\n        ))), p=self.drop_rate)  # \n        kv_mh = einops.rearrange(kv, 'b sl (nh d) -> b nh sl d', nh=self.num_heads, d=self.post_backbone_dim // self.num_heads)  # (bs, num_heads, 3xseq_len, dim_head)\n        feat_fused = q_mh + F.scaled_dot_product_attention(query=q_mh, key=kv_mh, value=kv_mh, dropout_p=self.drop_rate, is_causal=False)\n        feat_fused = norm_post_attn_g(feat_fused)\n        # 4. forward SED & attention pooling heads\n        feat_fused = einops.rearrange(feat_fused, 'b nh sl d -> b sl (nh d)')  # (bs, seq_len, dim)\n        feat_fused = einops.rearrange(feat_fused, 'b (h w) c -> b h w c', h=f_h, w=f_w)\n        feat_fused = torch.cat([feat_fused.mean(dim=1), feat_fused.mean(dim=2)], dim=1)  # (bs, (H' + W'), dim)\n        frame_logits = self.temporal_freq_fc(feat_fused)  # (B,  (H' + W'), dim) -> (B, (H' + W'), num_classes)\n        clip_logits, attn_weights = self.pooling(feat_fused)  # (bs, num_classes), (bs,  (H' + W')(T))\n        if only_return_clip_logits:\n            return clip_logits\n        else:\n            return {\n                \"clip_logits\": clip_logits,\n                \"frame_logits\": frame_logits,\n                \"attn_weights\": attn_weights,\n            }\n\n    def forward_features(self, spec):\n        \"\"\"\n        Extract features using the local CNN backbone.\n\n        Args:\n            spec (torch.Tensor): Input spectrogram of shape (B, 1, H, W).\n\n        Returns:\n            torch.Tensor: Feature map of shape (B, C_feat, H_feat, W_feat).\n        \"\"\"\n        bs, in_ch, h, w = spec.shape\n        assert in_ch == 1\n        spec = spec.repeat((1, self.in_channs, 1, 1))\n        feat = self.backbone(spec)[-1]  # (bs, C', H', W')\n        return feat\n\n    def forward_global_features(self, spec):\n        \"\"\"\n        Extract features using the global CNN backbone.\n\n        Args:\n            spec (torch.Tensor): Global input spectrogram of shape (B, 1, H, W).\n\n        Returns:\n            torch.Tensor: Global feature map of shape (B, C_feat, H_feat, W_feat).\n        \"\"\"\n        bs, in_ch, h, w = spec.shape\n        assert in_ch == 1\n        spec = spec.repeat((1, self.in_channs, 1, 1))\n        feat = self.backbone_global(spec)[-1]  # (bs, C', H', W')\n        return feat","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T03:50:08.762663Z","iopub.execute_input":"2025-06-05T03:50:08.762946Z","iopub.status.idle":"2025-06-05T03:50:08.811257Z","shell.execute_reply.started":"2025-06-05T03:50:08.762923Z","shell.execute_reply":"2025-06-05T03:50:08.810206Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model1 = AudioClassifier(model_name='regnety_008', drop_rate=0.05, drop_path_rate=0.05).to(device).eval().requires_grad_(False)\nmodel1.load_state_dict(torch.load(MODEL_FILEPATH_LIST[0], map_location=torch.device('cpu')))\n\nmodel2 = AudioClassifier(model_name='regnety_008', drop_rate=0.05, drop_path_rate=0.05).to(device).eval().requires_grad_(False)\nmodel2.load_state_dict(torch.load(MODEL_FILEPATH_LIST[1], map_location=torch.device('cpu')))\n\nmodel3 = AudioClassifier(model_name='regnety_008', drop_rate=0.08, drop_path_rate=0.08).to(device).eval().requires_grad_(False)\nmodel3.load_state_dict(torch.load(MODEL_FILEPATH_LIST[2], map_location=torch.device('cpu')))\n\nmodel4 = AudioClassifierSED(model_name='regnety_008', drop_rate=0.06, drop_path_rate=0.06).to(device).eval().requires_grad_(False)\nmodel4.load_state_dict(torch.load(MODEL_FILEPATH_LIST[3], map_location=torch.device('cpu')))\n\nmodel5 = AudioClassifierSED(model_name='tf_efficientnetv2_b0', drop_rate=0.08, drop_path_rate=0.08).to(device).eval().requires_grad_(False)\nmodel5.load_state_dict(torch.load(MODEL_FILEPATH_LIST[4], map_location=torch.device('cpu')))\n\nmodel6 = AudioClassifierSED(model_name='eca_nfnet_l0', drop_rate=0.08, drop_path_rate=0.08).to(device).eval().requires_grad_(False)\nmodel6.load_state_dict(torch.load(MODEL_FILEPATH_LIST[5], map_location=torch.device('cpu')))\n\nmodel7 = AudioClassifierSEDGlobal(model_name='tf_efficientnetv2_b0', drop_rate=0.08, drop_path_rate=0.08, use_separate_norm=True).to(device).eval().requires_grad_(False)\nmodel7.load_state_dict(torch.load(MODEL_FILEPATH_LIST[6], map_location=torch.device('cpu')))\n\nmodel8 = AudioClassifierSEDGlobal(model_name='eca_nfnet_l0', global_model_name='tf_efficientnetv2_b0', drop_rate=0.08, drop_path_rate=0.08, use_separate_norm=True).to(device).eval().requires_grad_(False)\nmodel8.load_state_dict(torch.load(MODEL_FILEPATH_LIST[7], map_location=torch.device('cpu')))\n\nmodel_list = [model1, model2, model3, model4, model5, model6, model7, model8]\nassert len(ENS_WEIGHT_LIST) == len(MELSPEC_CONFIG_LIST) == len(model_list)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T03:50:08.813495Z","iopub.execute_input":"2025-06-05T03:50:08.813811Z","iopub.status.idle":"2025-06-05T03:50:15.558718Z","shell.execute_reply.started":"2025-06-05T03:50:08.813787Z","shell.execute_reply":"2025-06-05T03:50:15.557366Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. Preprocessor","metadata":{}},{"cell_type":"code","source":"class MelSpectrogramTransform:\n    def __init__(\n        self,\n        n_fft,\n        hop_length,\n        n_mels,\n        f_min,\n        f_max,\n        target_shape,\n        sample_rate=32000,\n        mode='resize',  # 'pad' or 'resize'\n        to_db=True,\n        melspec_extra_kwargs={},\n        min_max_scale=True,\n    ):\n        print(\n            'Initializing MelSpectrogramTransform',\n            n_fft,\n            hop_length,\n            n_mels,\n            f_min,\n            f_max,\n            target_shape,\n            sample_rate,\n            mode,  # 'pad' or 'resize'\n            to_db,\n            melspec_extra_kwargs,\n            min_max_scale,\n        )\n        assert mode in ['pad', 'resize'], \"mode must be 'pad' or 'resize'\"\n        self.mode = mode\n        self.target_shape = target_shape\n        self.to_db = to_db\n\n        self.mel_transform = torchaudio.transforms.MelSpectrogram(\n            sample_rate=sample_rate,\n            n_fft=n_fft,\n            hop_length=hop_length,\n            n_mels=n_mels,\n            f_min=f_min,\n            f_max=f_max,\n            **melspec_extra_kwargs,\n        )\n        self.db_transform = torchaudio.transforms.AmplitudeToDB(top_db=80)\n        self.min_max_scale = min_max_scale\n\n    def __call__(self, waveform: torch.Tensor) -> torch.Tensor:\n        if waveform.dim() == 1:\n            waveform = waveform.unsqueeze(0)\n\n        mel = self.mel_transform(waveform)\n\n        if self.to_db:\n            mel = self.db_transform(mel)\n\n        # mel = (mel - mel.mean()) / (mel.std() + 1e-6)\n        if self.min_max_scale:\n            mel = (mel - mel.min()) / (mel.max() - mel.min() + 1e-6)\n\n        if self.target_shape is not None:\n            if self.mode == 'pad':\n                mel = self._pad_to_shape(mel, self.target_shape)\n            elif self.mode == 'resize':\n                mel = self._resize_to_shape(mel, self.target_shape)\n\n        return mel\n\n    def _pad_to_shape(self, mel, target_shape):\n        _, h, w = mel.shape\n        pad_h = max(0, target_shape[0] - h)\n        pad_w = max(0, target_shape[1] - w)\n        pad = (0, pad_w, 0, pad_h)\n        mel = F.pad(mel, pad, mode='constant', value=0)\n        mel = mel[:, :target_shape[0], :target_shape[1]]\n        return mel\n\n    def _resize_to_shape(self, mel, target_shape):\n        mel = F.interpolate(mel.unsqueeze(0), size=target_shape, mode='bilinear', align_corners=False)\n        return mel.squeeze(0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T03:50:15.560623Z","iopub.execute_input":"2025-06-05T03:50:15.560984Z","iopub.status.idle":"2025-06-05T03:50:15.576069Z","shell.execute_reply.started":"2025-06-05T03:50:15.560950Z","shell.execute_reply":"2025-06-05T03:50:15.574975Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"wf_transform = adm.Compose([\n    adm.Normalize(p=1.0),\n])  # waveform audio transform\nspec_transform = alb.Compose([\n    alb.Resize(256, 256),\n    alb.Normalize(mean=0.5, std=0.5, max_pixel_value=1.0),\n])  # melspectrogram image transform\n\npreprocessor_list = [\n    (MelSpectrogramTransform(**spec_config), wf_transform, spec_transform) for spec_config in MELSPEC_CONFIG_LIST\n]  # (to_spec, wf_transform, spec_transform) for each models\nassert len(preprocessor_list) == len(model_list)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T03:50:15.577389Z","iopub.execute_input":"2025-06-05T03:50:15.578333Z","iopub.status.idle":"2025-06-05T03:50:15.674263Z","shell.execute_reply.started":"2025-06-05T03:50:15.578300Z","shell.execute_reply":"2025-06-05T03:50:15.673124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. Inference","metadata":{}},{"cell_type":"code","source":"@torch.inference_mode()\ndef classify_audio_chunks(waveform_chunk_list, model, to_spec, wf_transform, spec_transform, sr=32000, debug=False):\n    n_org_chunks = len(waveform_chunk_list)\n    spec_chunk_list = [\n        torch.Tensor(spec_transform(image=to_spec(wf).permute(1, 2, 0).numpy())['image']).permute(2, 0, 1)\n        for wf in waveform_chunk_list\n    ]\n    assert all([sp_chunk.shape == (1, 256, 256) for sp_chunk in spec_chunk_list])\n    spec = torch.stack(spec_chunk_list, dim=0)\n    assert spec.shape == (n_org_chunks, 1, 256, 256), spec.shape\n    s_dt = datetime.now()\n    # pre-calculate features of all chunk spectrogram for better computation efficiency\n    feats = model.forward_features(spec=spec)  # (n_chunks, f_ch, f_h, f_w)\n    feats = torch.cat([feats[[0]], feats, feats[[-1]]], dim=0)\n    feats = torch.stack([\n        torch.stack([feats[i-1], feats[i], feats[i+1]], dim=0)  # (3, f_ch, f_h, f_w)\n        for i in range(1, len(feats)-1)\n    ], dim=0)  # (n_chunks, 3, f_ch, f_h, f_w)\n    assert len(feats) == n_org_chunks\n    pred_logits = model(pre_calc_feat=feats)\n    e_dt = datetime.now()\n    if debug:\n        print(f'model took {(e_dt-s_dt).total_seconds()} {pred_logits.isnan().any()} {pred_logits.min()}-{pred_logits.max()}')\n    # return pred_probs.cpu().numpy()\n    return pred_logits.cpu().numpy()  # return logits instead of probs\n\n\n@torch.inference_mode()\ndef classify_audio_chunks_global(waveform_chunk_list, model, to_spec, wf_transform, spec_transform, sr=32000, debug=False):\n    n_org_chunks = len(waveform_chunk_list)\n    raw_spec_chunk_list = [\n        to_spec(wf)\n        for wf in waveform_chunk_list\n    ]\n    def min_max_scale(spec):\n        return (spec - spec.min()) / (spec.max() - spec.min() + 1e-6)\n    scaled_spec_chunk_list = [\n        torch.Tensor(spec_transform(image=min_max_scale(spec).permute(1, 2, 0).numpy())['image']).permute(2, 0, 1)\n        for spec in raw_spec_chunk_list\n    ]\n    assert all([sp_chunk.shape == (1, 256, 256) for sp_chunk in scaled_spec_chunk_list])\n    spec = torch.stack(scaled_spec_chunk_list, dim=0)\n    assert spec.shape == (n_org_chunks, 1, 256, 256), spec.shape\n    s_dt = datetime.now()\n    feats = model.forward_features(spec=spec)  # (n_chunks, f_ch, f_h, f_w)\n    feats = torch.cat([feats[[0]], feats, feats[[-1]]], dim=0)\n    feats = torch.stack([\n        torch.stack([feats[i-1], feats[i], feats[i+1]], dim=0)  # (3, f_ch, f_h, f_w)\n        for i in range(1, len(feats)-1)\n    ], dim=0)  # (n_chunks, 3, f_ch, f_h, f_w)\n    assert len(feats) == n_org_chunks\n\n    spec_global = torch.cat(raw_spec_chunk_list, dim=-1)  # (1, 256, n_org_chunks x 256)\n    spec_global = min_max_scale(spec_global)\n    spec_global = (spec_global - 0.5) / 0.5  # alb.Normalize(mean=0.5, std=0.5, max_pixel_value=1.0)\n    assert spec_global.shape == (1, 256, n_org_chunks * 256), spec.shape\n    spec_global = F.interpolate(spec_global.unsqueeze(0), size=GLOBAL_SPEC_RESIZE_SIZE, mode='bilinear')  # (1, 1, 256, 2 x 256)\n    # pre-calculate global feature of whole spectrogram for better computation efficiency\n    global_feats = model.forward_global_features(spec=spec_global)  # (1, f_ch, f_h, f_w_g)\n    global_feats = global_feats.repeat((n_org_chunks, 1, 1, 1))  # (n_org_chunks, f_ch, f_h, f_w_g)\n\n    pred_logits = model(pre_calc_feat=feats, pre_calc_g_feat=global_feats)\n    e_dt = datetime.now()\n    if debug:\n        print(f'model took {(e_dt-s_dt).total_seconds()} {pred_logits.isnan().any()} {pred_logits.min()}-{pred_logits.max()}')\n    # return pred_probs.cpu().numpy()\n    return pred_logits.cpu().numpy()  # return logits instead of probs\n\n\n@torch.inference_mode()\ndef inference_single_file(filepath: str, model, to_spec, wf_transform, spec_transform, classify_audio_chunks_fn, resample_sr=32000, debug=False) -> pd.DataFrame:\n    waveform, sr = torchaudio.load(filepath, normalize=True)  # (1, sr x duration_sec)\n    if sr != resample_sr:\n        waveform = torchaudio.functional.resample(waveform, sr, resample_sr)  # (1, resample_sr x duration_sec)\n    waveform = torch.Tensor(wf_transform(waveform.squeeze(0).numpy(), sample_rate=resample_sr)).unsqueeze(0)\n    duration_sec = len(waveform.squeeze()) / resample_sr\n    end_secs = list(range(5, int(duration_sec) + 1, 5))\n    filename_prefix_id = filepath.split('/')[-1].replace('.ogg', '')\n    row_ids = [f'{filename_prefix_id}_{end_time}' for end_time in end_secs]\n    waveform_chunk_list = [waveform[:, int((end_sec-5)*resample_sr):int(end_sec*resample_sr)] for end_sec in end_secs]\n    preds = classify_audio_chunks_fn(waveform_chunk_list, model, to_spec, wf_transform, spec_transform, sr=resample_sr, debug=debug)\n    assert preds.shape == (len(row_ids), N_CLASSES)\n    df_preds = pd.DataFrame(index=row_ids, data=preds, columns=SPECIES)\n    df_preds.index.name = 'row_id'\n    return df_preds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T03:50:16.184677Z","iopub.execute_input":"2025-06-05T03:50:16.184990Z","iopub.status.idle":"2025-06-05T03:50:16.206445Z","shell.execute_reply.started":"2025-06-05T03:50:16.184968Z","shell.execute_reply":"2025-06-05T03:50:16.205159Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def apply_power_to_low_ranked_cols(\n    df_pred_prob: pd.DataFrame,\n    top_k: int = 30,\n    exponent: Union[int, float] = 2,\n) -> pd.DataFrame:\n\n    def apply_power_to_low_ranked_cols_file(\n        p: np.ndarray,\n        top_k: int = 30,\n        exponent: Union[int, float] = 2,\n        inplace: bool = True\n    ) -> np.ndarray:\n        if not inplace:\n            p = p.copy()\n        # Identify columns whose max value ranks below `top_k`\n        tail_cols = np.argsort(-p.max(axis=0))[top_k:]\n        # Apply the power transformation to those columns\n        p[:, tail_cols] = p[:, tail_cols] ** exponent\n        return p\n\n    df_pred_prob_cp = df_pred_prob.copy()\n    df_pred_prob_cp['file'] = df_pred_prob_cp.index.str.rsplit('_', n=1).str[0]\n    for file, df_pred_prob_file in df_pred_prob_cp.groupby('file'):\n        assert len(df_pred_prob_file) == 12\n        probs = df_pred_prob_file[SPECIES].values\n        preds_adjusted = apply_power_to_low_ranked_cols_file(probs, top_k, exponent, inplace=False)\n        df_pred_prob_cp.loc[df_pred_prob_file.index, SPECIES] = preds_adjusted\n    df_pred_prob_cp = df_pred_prob_cp.drop(columns=['file'])\n\n    return df_pred_prob_cp","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T03:50:19.304686Z","iopub.execute_input":"2025-06-05T03:50:19.304996Z","iopub.status.idle":"2025-06-05T03:50:19.312964Z","shell.execute_reply.started":"2025-06-05T03:50:19.304973Z","shell.execute_reply":"2025-06-05T03:50:19.311933Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_pred_logits_model_list = []\n\nfor model, (to_spec, wf_transform, spec_transform) in zip(pb(model_list), preprocessor_list):\n    if isinstance(model, AudioClassifierSEDGlobal):\n        classify_audio_chunks_fn = classify_audio_chunks_global\n        assert to_spec.min_max_scale is False\n    else:\n        classify_audio_chunks_fn = classify_audio_chunks\n        assert to_spec.min_max_scale is True\n    df_pred_logits_list = []\n    for filepath in pb(df_test_files['file_path']):\n        df_pred_logits = inference_single_file(filepath, model, to_spec, wf_transform, spec_transform, classify_audio_chunks_fn, debug=False)\n        df_pred_logits_list.append(df_pred_logits)\n\n    df_pred_logits = pd.concat(df_pred_logits_list, axis=0)  # index : row_id, columns : SPECIES\n    # Since the ROC AUC is a class-wise, rank-based metric, align the scales of the predicted values between models by normalizing the logits class-by-class for later weighted ensemble blending.\n    df_pred_logits = (df_pred_logits - df_pred_logits.mean(axis=0)) / (df_pred_logits.std(axis=0) + 1e-8)\n    if APPLY_LOW_RANK_POWER_SINGLE_MODEL:\n        df_pred_probs = df_pred_logits.apply(expit)  # convert to probs for low rank power adjustment\n        df_pred_probs = apply_power_to_low_ranked_cols(df_pred_probs, top_k=LRP_TOP_K, exponent=LRP_EXPONENT)\n        df_pred_logits = df_pred_probs.apply(lambda x: logit(np.clip(x, 1e-8, 1 - 1e-8)))  # convert back to logits for later postporcess & ensembling\n    df_pred_logits_model_list.append(df_pred_logits)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T03:50:19.439401Z","iopub.execute_input":"2025-06-05T03:50:19.440177Z","iopub.status.idle":"2025-06-05T03:57:33.008974Z","shell.execute_reply.started":"2025-06-05T03:50:19.440148Z","shell.execute_reply":"2025-06-05T03:57:33.007848Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. Ensemble","metadata":{}},{"cell_type":"code","source":"# make sure index(row_id) are all aligned\nassert all([(df.index == df_pred_logits_model_list[0].index).all() for df in df_pred_logits_model_list])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T03:57:47.274080Z","iopub.execute_input":"2025-06-05T03:57:47.274375Z","iopub.status.idle":"2025-06-05T03:57:47.280627Z","shell.execute_reply.started":"2025-06-05T03:57:47.274354Z","shell.execute_reply":"2025-06-05T03:57:47.279663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if ENSEMBLE_METHOD == 'logits_weighted_sum':\n    # logits weighted sum ensemble -> convert to probs by sigmoid\n    df_pred_logits = sum([w * df[SPECIES] for df, w in zip(df_pred_logits_model_list, ENS_WEIGHT_LIST)])\n    df_pred = df_pred_logits.apply(expit)\nelif ENSEMBLE_METHOD == 'prob_weighted_sum':\n    # convert to probs by sigmoid -> probs weighted sum ensemble\n    df_pred = sum([w * df[SPECIES].apply(expit) for df, w in zip(df_pred_logits_model_list, ENS_WEIGHT_LIST)])\nelif ENSEMBLE_METHOD == 'rank_weighted_sum':\n    # percent rank -> rank weighted sum ensemble\n    ranked_dfs = [\n        df.rank(axis=0, method=\"average\", pct=True)\n        for df in df_pred_logits_model_list\n    ]\n    df_pred = sum([w * df[SPECIES] for df, w in zip(ranked_dfs, ENS_WEIGHT_LIST)])\nelse:\n    raise Exception(ENSEMBLE_METHOD)\n\nif APPLY_LOW_RANK_POWER_ENSEMBLE:\n    df_pred = apply_power_to_low_ranked_cols(df_pred, top_k=LRP_TOP_K, exponent=LRP_EXPONENT)\n\ndf_pred = df_pred.reset_index()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T03:57:47.440781Z","iopub.execute_input":"2025-06-05T03:57:47.441094Z","iopub.status.idle":"2025-06-05T03:57:47.478506Z","shell.execute_reply.started":"2025-06-05T03:57:47.441071Z","shell.execute_reply":"2025-06-05T03:57:47.477391Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_pred","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T03:57:50.583350Z","iopub.execute_input":"2025-06-05T03:57:50.583748Z","iopub.status.idle":"2025-06-05T03:57:50.611276Z","shell.execute_reply.started":"2025-06-05T03:57:50.583721Z","shell.execute_reply":"2025-06-05T03:57:50.610221Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6. Submit","metadata":{}},{"cell_type":"code","source":"df_pred.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T03:57:51.126764Z","iopub.execute_input":"2025-06-05T03:57:51.127055Z","iopub.status.idle":"2025-06-05T03:57:51.292874Z","shell.execute_reply.started":"2025-06-05T03:57:51.127034Z","shell.execute_reply":"2025-06-05T03:57:51.291666Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7. Temporal Smoothing","metadata":{}},{"cell_type":"code","source":"if APPLY_TEMPORAL_SMOOTHING:\n    # sub = pd.read_csv('submission.csv')\n    sub = df_pred\n\n    sub['group'] = sub['row_id'].str.rsplit('_', n=1).str[0]\n    sub['end_sec'] = sub['row_id'].str.rsplit('_', n=1).str[1].astype(int)\n\n    weights = np.array([ADJ_MIX_RATIO, CENTER_MIX_RATIO, ADJ_MIX_RATIO])\n    w_center = weights\n    w_start = np.array([CENTER_MIX_RATIO, ADJ_MIX_RATIO]) / (ADJ_MIX_RATIO + CENTER_MIX_RATIO)\n    w_end = np.array([ADJ_MIX_RATIO, CENTER_MIX_RATIO]) / (ADJ_MIX_RATIO + CENTER_MIX_RATIO)\n    \n    smoothed_sub = sub.copy()\n\n    for group, group_df in pb(sub.groupby('group')):\n        group_df = group_df.sort_values('end_sec')\n        preds = group_df[SPECIES].values\n        new_preds = preds.copy()\n        \n        # rolling for 5sec~55sec\n        rolled = pd.DataFrame(preds).rolling(3, center=True).apply(\n            lambda x: np.sum(x * w_center), raw=True\n        ).values\n        \n        # rolling for 0-5sec & 55-60sec\n        new_preds[1:-1] = rolled[1:-1]\n        new_preds[0] = preds[0] * w_start[0] + preds[1] * w_start[1]\n        new_preds[-1] = preds[-2] * w_end[0] + preds[-1] * w_end[1]\n        \n        smoothed_sub.loc[group_df.index, SPECIES] = new_preds\n    \n    smoothed_sub = smoothed_sub.drop(columns='group')\n    smoothed_sub = smoothed_sub.drop(columns='end_sec')\n    smoothed_sub[['row_id'] + SPECIES].to_csv(\"submission.csv\", index=False)\n\n    display(smoothed_sub)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T03:58:14.005080Z","iopub.execute_input":"2025-06-05T03:58:14.005506Z","iopub.status.idle":"2025-06-05T03:58:14.015424Z","shell.execute_reply.started":"2025-06-05T03:58:14.005477Z","shell.execute_reply":"2025-06-05T03:58:14.014371Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}