{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":690481,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":519622,"modelId":534001},{"sourceId":690482,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":519622,"modelId":534001},{"sourceId":690633,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":519622,"modelId":534001},{"sourceId":692242,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":519622,"modelId":534001}],"dockerImageVersionId":31234,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os \nimport pandas as pd\nimport numpy as np\nimport pickle","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T11:34:20.078320Z","iopub.execute_input":"2025-12-19T11:34:20.078649Z","iopub.status.idle":"2025-12-19T11:34:20.084490Z","shell.execute_reply.started":"2025-12-19T11:34:20.078620Z","shell.execute_reply":"2025-12-19T11:34:20.083209Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#submission notebook doesn't have internet access, so I simply copy pasted code: https://huggingface.co/bird-of-paradise/deepseek-mla\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom typing import Tuple\n\nimport math\n\n\n\ndef precompute_freqs_cis(dim: int, end: int, theta: float = 10000.0):\n    freqs = 1.0 / (theta ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim))\n    t = torch.arange(end, device=freqs.device)  # type: ignore\n    freqs = torch.outer(t, freqs).float()  # type: ignore\n    freqs_cis = torch.polar(torch.ones_like(freqs), freqs)  # complex64\n    return freqs_cis\n\ndef reshape_for_broadcast(freqs_cis: torch.Tensor, x: torch.Tensor):\n    ndim = x.ndim\n    assert 0 <= 1 < ndim\n    assert freqs_cis.shape == (x.shape[1], x.shape[-1])\n    shape = [d if i == 1 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)]\n    return freqs_cis.view(*shape)\n\n\ndef apply_rotary_emb(\n    xq: torch.Tensor,\n    xk: torch.Tensor,\n    freqs_cis: torch.Tensor,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n    # Validate input dimensions\n    assert xq.shape[-1] == xk.shape[-1], \"Query and Key must have same embedding dimension\"\n    assert xq.shape[-1] % 2 == 0, \"Embedding dimension must be even\"\n\n    # Get sequence lengths\n    q_len = xq.shape[1]\n    k_len = xk.shape[1]\n    \n    # Use appropriate part of freqs_cis for each sequence\n    q_freqs = freqs_cis[:q_len]\n    k_freqs = freqs_cis[:k_len]\n    \n    # Apply rotary embeddings separately\n    # split last dimention to [xq.shape[:-1]/2, 2]\n    xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2))\n    xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2))\n    \n \n    # Reshape freqs for each\n    q_freqs = reshape_for_broadcast(q_freqs, xq_)\n    k_freqs = reshape_for_broadcast(k_freqs, xk_)\n    \n    # Works for both [bsz, seqlen, n_heads*head_dim] and [bsz, seqlen, n_heads, head_dim]\n    xq_out = torch.view_as_real(xq_ * q_freqs).flatten(xq.ndim-1) \n    xk_out = torch.view_as_real(xk_ * k_freqs).flatten(xk.ndim-1)\n\n    return xq_out.type_as(xq), xk_out.type_as(xk)\n\n\n\n\nclass MultiHeadLatentAttention(nn.Module):\n    \"\"\"\n        Multi-Head Latent Attention(MLA) Module As in DeepSeek_V2 pape\n        Key innovation from standard MHA:\n             1. Low-Rank Key-Value Joint Compression \n             2. Decoupled Rotary Position Embedding\n             \n    Args:\n        d_model:  Total dimension of the model.\n        num_head: Number of attention heads.\n        d_embed:  Embedding dimension\n        d_c:      K/V compression dimension\n        d_c1:     Q compression dimension\n        d_rotate: Dimension for Rotary Position Embedding\n        dropout:  Dropout rate for attention scores.\n        bias:     Whether to include bias in linear projections.\n        d_head:   Inferred from d_model//num_head\n    Inputs:\n        sequence: input sequence for self-attention and the query for cross-attention\n        key_value_state: input for the key, values for cross-attention\n    \"\"\"\n    def __init__(\n        self, \n        d_model,             # Infer d_head from d_model\n        num_head, \n        d_embed, \n        d_c, \n        d_c1, \n        d_rotate, \n        dropout=0.1, \n        bias=True,\n        max_batch_size=32,   # For KV cache sizing\n        max_seq_len=2048     # For KV cache sizing \n        ):\n        super().__init__()\n        \n        assert d_model % num_head == 0, \"d_model must be divisible by num_head\"\n        assert d_c < d_embed, \"Compression dim should be smaller than embedding dim\"\n        assert d_c1 < d_embed, \"Query compression dim should be smaller than embedding dim\"\n        \n        self.d_model = d_model\n        self.num_head = num_head\n        # Verify dimensions match up\n        assert d_model % num_head == 0, f\"d_model ({d_model}) must be divisible by num_head ({num_head})\"\n        self.d_head=d_model//num_head\n        self.d_embed = d_embed\n        self.d_c = d_c\n        self.d_c1 = d_c1\n        self.d_rotate = d_rotate\n        self.dropout_rate = dropout  # Store dropout rate separately\n\n        # Linear down-projection(compression) transformations\n        self.DKV_proj = nn.Linear(d_embed, d_c, bias=bias)\n        self.DQ_proj = nn.Linear(d_embed, d_c1, bias=bias)\n        \n        # linear up-projection transformations\n        self.UQ_proj = nn.Linear(d_c1, d_model, bias=bias)\n        self.UK_proj = nn.Linear(d_c, d_model, bias=bias)\n        self.UV_proj = nn.Linear(d_c, d_model, bias=bias)\n\n        # Linear RoPE-projection\n        self.RQ_proj = nn.Linear(d_c1, num_head*d_rotate, bias=bias)\n        self.RK_proj = nn.Linear(d_embed, d_rotate, bias=bias)\n        \n        # linear output transformations\n        self.output_proj = nn.Linear( d_model, d_model, bias=bias)\n\n        # Dropout layer\n        self.dropout = nn.Dropout(p=dropout)\n\n        # Initiialize scaler\n        self.scaler = float(1.0 / math.sqrt(self.d_head + d_rotate)) # Store as float in initialization\n\n        # Initialize C_KV and R_K cache for inference\n        self.cache_kv = torch.zeros(\n            (max_batch_size, max_seq_len, d_c)\n        )\n        self.cache_rk = torch.zeros(\n            (max_batch_size, max_seq_len, d_rotate)\n        )\n\n        # Initialize freqs_cis for RoPE\n        self.freqs_cis = precompute_freqs_cis(\n            d_rotate, max_seq_len * 2\n        )\n    \n\n    def forward(\n        self, \n        sequence, \n        key_value_states = None, \n        att_mask=None,\n        use_cache=False,\n        start_pos: int = 0\n    ):\n\n        \"\"\"\n        Forward pass supporting both standard attention and cached inference\n        Input shape: [batch_size, seq_len, d_model=num_head * d_head]\n        Args:\n            sequence: Input sequence [batch_size, seq_len, d_model]\n            key_value_states: Optional states for cross-attention\n            att_mask: Optional attention mask\n            use_cache: Whether to use KV caching (for inference)\n            start_pos: Position in sequence when using KV cache\n        \"\"\"\n        batch_size, seq_len, model_dim = sequence.size()\n        # prepare for RoPE\n        self.freqs_cis = self.freqs_cis.to(sequence.device)\n        freqs_cis = self.freqs_cis[start_pos : ]\n\n        # Check only critical input dimensions\n        assert model_dim == self.d_model, f\"Input dimension {model_dim} doesn't match model dimension {self.d_model}\"\n        if key_value_states is not None:\n            assert key_value_states.size(-1) == self.d_model, \\\n            f\"Cross attention key/value dimension {key_value_states.size(-1)} doesn't match model dimension {self.d_model}\"\n\n        # if key_value_states are provided this layer is used as a cross-attention layer\n        # for the decoder\n        is_cross_attention = key_value_states is not None\n\n        # Determine kv_seq_len early\n        kv_seq_len = key_value_states.size(1) if is_cross_attention else seq_len\n        \n        # Linear projections and reshape for multi-head, in the order of Q, K/V\n        # Down and up projection for query\n        C_Q = self.DQ_proj(sequence)     #[batch_size, seq_len, d_c1]\n        Q_state = self.UQ_proj(C_Q)      #[batch_size, seq_len, d_model]\n        # Linear projection for query RoPE pathway\n        Q_rotate = self.RQ_proj(C_Q)      #[batch_size, seq_len, num_head*d_rotate]\n\n\n        if use_cache:\n            #Equation (41) in DeepSeek-v2 paper: cache c^{KV}_t\n            self.cache_kv = self.cache_kv.to(sequence.device)\n\n            # Get current compressed KV states\n            current_kv = self.DKV_proj(key_value_states if is_cross_attention else sequence) #[batch_size, kv_seq_len, d_c]\n            # Update cache using kv_seq_len instead of seq_len\n            self.cache_kv[:batch_size, start_pos:start_pos + kv_seq_len] = current_kv\n            # Use cached compressed KV up to current position\n            C_KV = self.cache_kv[:batch_size, :start_pos + kv_seq_len]\n\n            #Equation (43) in DeepSeek-v2 paper: cache the RoPE pathwway for shared key k^R_t\n            assert self.cache_rk.size(-1) == self.d_rotate, \"RoPE cache dimension mismatch\"\n            self.cache_rk = self.cache_rk.to(sequence.device)\n            # Get current RoPE key\n            current_K_rotate = self.RK_proj(key_value_states if is_cross_attention else sequence) #[batch_size, kv_seq_len, d_rotate]\n            # Update cache using kv_seq_len instead of seq_len\n            self.cache_rk[:batch_size, start_pos:start_pos + kv_seq_len] = current_K_rotate\n            # Use cached RoPE key up to current position\n            K_rotate = self.cache_rk[:batch_size, :start_pos + kv_seq_len] #[batch_size, cached_len, d_rotate]\n            \n            \n            \"\"\"handling attention mask\"\"\"\n            if att_mask is not None:\n                # Get the original mask shape\n                mask_size = att_mask.size(-1)\n                cached_len = start_pos + kv_seq_len        # cached key_len, including previous key\n                assert C_KV.size(1) == cached_len, \\\n            f\"Cached key/value length {C_KV.size(1)} doesn't match theoretical length {cached_len}\"\n                \n                # Create new mask matching attention matrix shape\n                extended_mask = torch.zeros(\n                    (batch_size, 1, seq_len, cached_len),  # [batch, head, query_len, key_len]\n                    device=att_mask.device,\n                    dtype=att_mask.dtype\n                )\n                \n                # Fill in the mask appropriately - we need to be careful about the causality here\n                # For each query position, it should only attend to cached positions up to that point\n                for i in range(seq_len):\n                    extended_mask[:, :, i, :(start_pos + i + 1)] = 0  # Can attend\n                    extended_mask[:, :, i, (start_pos + i + 1):] = float('-inf')  # Cannot attend\n                    \n                att_mask = extended_mask\n        else:\n            # Compression projection for C_KV\n            C_KV = self.DKV_proj(key_value_states if is_cross_attention else sequence) #[batch_size, kv_seq_len, d_c]\\\n            # RoPE pathway for *shared* key\n            K_rotate = self.RK_proj(key_value_states if is_cross_attention else sequence)\n            \n\n        # Up projection for key and value\n        K_state = self.UK_proj(C_KV)               #[batch_size, kv_seq_len/cached_len, d_model]\n        V_state = self.UV_proj(C_KV)               #[batch_size, kv_seq_len/cached_len, d_model]\n\n        \n        Q_state = Q_state.view(batch_size, seq_len, self.num_head, self.d_head)\n\n        # After getting K_state from projection, get its actual sequence length\n        actual_kv_len = K_state.size(1)    # kv_seq_len or start_pos + kv_seq_len\n        # in cross-attention, key/value sequence length might be different from query sequence length\n        # Use actual_kv_len instead of kv_seq_len for reshaping\n        K_state = K_state.view(batch_size, actual_kv_len, self.num_head, self.d_head) \n        V_state = V_state.view(batch_size, actual_kv_len, self.num_head, self.d_head)\n\n\n        #Apply RoPE to query and shared key\n        Q_rotate = Q_rotate.view(batch_size, seq_len, self.num_head, self.d_rotate)\n        K_rotate = K_rotate.unsqueeze(2).expand(-1, -1, self.num_head, -1)  # [batch, cached_len, num_head, d_rotate]\n        Q_rotate, K_rotate = apply_rotary_emb(Q_rotate, K_rotate, freqs_cis=freqs_cis)\n\n\n        # Concatenate along head dimension\n        Q_state = torch.cat([Q_state, Q_rotate], dim=-1)  # [batch_size, seq_len, num_head, d_head + d_rotate]\n        K_state = torch.cat([K_state, K_rotate], dim=-1)  # [batch_size, actual_kv_len, num_head, d_head + d_rotate]\n\n\n        # Scale Q by 1/sqrt(d_k)\n        Q_state = Q_state * self.scaler\n        Q_state = Q_state.transpose(1, 2)  # [batch_size, num_head, seq_len, head_dim]\n        K_state = K_state.transpose(1, 2)  # [batch_size, num_head, actual_kv_len, head_dim]\n        V_state = V_state.transpose(1, 2)  # [batch_size, num_head, actual_kv_len, head_dim]\n\n    \n        # Compute attention matrix: QK^T\n        self.att_matrix = torch.matmul(Q_state, K_state.transpose(-1,-2)) \n    \n        # apply attention mask to attention matrix\n        if att_mask is not None and not isinstance(att_mask, torch.Tensor):\n            raise TypeError(\"att_mask must be a torch.Tensor\")\n\n        if att_mask is not None:\n            self.att_matrix = self.att_matrix + att_mask\n        \n        # apply softmax to the last dimension to get the attention score: softmax(QK^T)\n        att_score = F.softmax(self.att_matrix, dim = -1)\n    \n        # apply drop out to attention score\n        att_score = self.dropout(att_score)\n    \n        # get final output: softmax(QK^T)V\n        att_output = torch.matmul(att_score, V_state)\n        assert att_output.size(0) == batch_size, \"Batch size mismatch\"\n        assert att_output.size(2) == seq_len, \"Output sequence length should match query sequence length\"\n        \n            \n        # concatinate all attention heads\n        att_output = att_output.transpose(1, 2).contiguous().view(batch_size, seq_len, self.num_head*self.d_head) \n\n\n        # final linear transformation to the concatenated output\n        att_output = self.output_proj(att_output)\n\n        assert att_output.size() == (batch_size, seq_len, self.d_model), \\\n        f\"Final output shape {att_output.size()} incorrect\"\n\n        return att_output","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T11:34:20.284380Z","iopub.execute_input":"2025-12-19T11:34:20.284707Z","iopub.status.idle":"2025-12-19T11:34:20.320692Z","shell.execute_reply.started":"2025-12-19T11:34:20.284679Z","shell.execute_reply":"2025-12-19T11:34:20.319713Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Well, I think it's a good practice nowadays, to explicitly write, where LLMs were used\nfrom dataclasses import dataclass, field\nfrom typing import Callable, Optional, Any\n\n@dataclass\nclass AudioConfig:\n    \"\"\"\n    Central configuration for Audio Processing.\n    \"\"\"\n    sr: int = 32000\n    n_fft: int = 2048\n    win_length: int = n_fft\n    hop_length: int = win_length // 2\n    n_mels: int = 256\n    top_db: int = 80\n    \n    # Slicing/Striping params\n    stripe_width: int = sr // (n_fft // 4) #~1 secnod\n    stripe_overlap: int = stripe_width // 5 #~0.2 seconds\n    \n    # Helper to inject external slicing logic\n    slicing_func: Any = None\n\n    prior_cut_sec: int = 5","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T11:34:20.322625Z","iopub.execute_input":"2025-12-19T11:34:20.322925Z","iopub.status.idle":"2025-12-19T11:34:20.347220Z","shell.execute_reply.started":"2025-12-19T11:34:20.322897Z","shell.execute_reply":"2025-12-19T11:34:20.346235Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torchaudio.transforms as transforms\nimport torchaudio\n#import librosa\nfrom joblib import Parallel, delayed\nimport ast\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nclass BirdDataset(torch.utils.data.Dataset):\n    def __init__(\n        self,\n        meta_df,\n        label_encoder,\n        config: AudioConfig,\n        group_mode:bool = True,\n        return_id: bool = True,\n        prediction_as_target: bool = False,\n        root_dir:str = '/kaggle/input/birdclef-2025/train_audio',\n        multitarget:bool = True,\n    ):\n        super().__init__()\n        self.cfg = config\n        #sklearn labelencoder\n        self.label_encoder = label_encoder\n        #files root dir\n        self.root_dir = root_dir\n        #flag to retrieve whole sequnces instead of just tokens\n        self.group_mode = group_mode\n        #flag to retrieve filenames to sequence of tensors\n        self.return_id = return_id\n        #includes the secondary labels as primary\n        self.multitarget = multitarget\n        #flag to retrive pseudo labels\n        self.prediction_as_target = prediction_as_target\n        #will serve as pseudo labels storage later\n        self.predictions = None\n        \n        #creating spectograms and preparing labels\n        results_with_index = self._parallel_prepare(meta_df)\n        \n        #extract spectograms and labels\n        self.spectrs, self.labels, self.idx = zip(*results_with_index)\n\n        if not group_mode:\n            #few row down we do item assignment\n            self.labels = list(self.labels)\n            \n            #expand labels with respect to token_num in each sample\n            for i, tokenized_tensor in enumerate(self.spectrs):\n                expand_size = tokenized_tensor.shape[-1]\n                self.labels[i] = np.tile(self.labels[i], (expand_size,1))\n                \n            #concatenate all tensor into 1 huge brick, because we doesn't care about their relations\n            self.spectrs = torch.cat(self.spectrs, dim = -1)\n            self.labels = [arr for sublist in self.labels for arr in sublist]\n\n        #label tensor init with n_classes\n        label_tensor = torch.zeros(len(self.labels), len(self.label_encoder.classes_)) #B, L\n        #ohe assigning \n        #todo it seems very slow, but I don't know better approach for multi label ohe assigning\n        #because once again, self.labels isn't a size=1 array\n        for i, label in enumerate(self.labels):\n            label_tensor[i, label] = 1\n\n        self.labels = label_tensor\n        self.idx = np.array(self.idx, dtype = object)\n        \n        self.token_h, self.token_w =  config.n_mels, config.stripe_width\n    \n    def preprocess_label_file(\n        self, \n        file_path, \n        label, \n        secondary_labels, \n        root_dir = None,\n    ):\n        \"\"\"\n        Reads audio -> STFT -> Mel/Mel**2/Linear Features -> Stacks them.\n        Reads id(str) -> id from label_encoder\n        \"\"\"\n        if root_dir is None:\n            root_dir = self.root_dir\n        path = os.path.join(root_dir, file_path)\n        waveform, sr = torchaudio.load(path, num_frames = self.cfg.sr * self.cfg.prior_cut_sec)\n\n        if sr != self.cfg.sr:\n            waveform = transforms.Resample(sr, self.cfg.sr)(waveform)\n            \n        waveform = waveform[0]\n        \n        label_id = self.label_encoder.transform([label])\n\n        #secondary label processing\n        #str -> list\n        secondary_labels = ast.literal_eval(secondary_labels)\n        #if secondary labels actually exist:\n        if (secondary_labels != ['']) and self.multitarget:\n            secondary_labels = self.label_encoder.transform(secondary_labels)\n            label_id = np.concatenate((label_id, secondary_labels))\n        \n        slices = self.cfg.slicing_func( #[C, H, W] -> [C, H, W, T]\n                    ar = waveform.unsqueeze(0).unsqueeze(0),\n                    stripe = self.cfg.stripe_width * self.cfg.hop_length, \n                    overlap = self.cfg.stripe_overlap * self.cfg.hop_length,\n                    pad_value = 0,\n                )\n      \n        return slices, label_id, file_path\n\n    def _parallel_prepare(self, df):\n        \"\"\"\n        parallel file processing using joblib\n        \"\"\"\n\n        columns_of_interest = ['filename', 'primary_label', 'secondary_labels']        \n        \n        results_with_index = Parallel(n_jobs=-1)(\n                    delayed(self.preprocess_label_file)(\n                        file_path = row.filename, \n                        label = row.primary_label,\n                        secondary_labels = row.secondary_labels,\n                    ) \n                    for _, row in df[columns_of_interest].iterrows()\n                )\n\n        return results_with_index\n\n    def update_pseudo_labels(self, obj:pd.Series):\n        \"\"\"\n        appends new pseudolabels into existing pseudolabels storage\n        \"\"\"\n        if self.predictions is None:\n            self.predictions = pd.Series()\n\n        self.predictions = pd.concat([self.predictions, obj])\n        \n    def __len__(self):\n        return len(self.labels) \n\n    def __getitem__(self, index):\n        \"\"\"\n        returns batch with ohe target\n        \"\"\"\n        if self.group_mode:\n            #it is the only option to fix dataloader select random samples via list problem\n            #we can't select tuple[list], so instead we have to itterate\n            #yes, it's the ONLY PLACE, we can't even use collate_fn\n            if isinstance(index, list) or isinstance(index, np.ndarray):\n                batch_tensor = [self.spectrs[i] for i in index]\n            else:\n                batch_tensor = self.spectrs[index]\n\n        else:\n            #because in not group mode we iterate through tokens and they are in the last dim\n            batch_tensor = self.spectrs[..., index]\n\n        str_id = self.idx[index]\n        #pseudo labels\n        if self.prediction_as_target:\n            label_tensor = self.predictions[str_id].values\n        else:\n            label_tensor = self.labels[index]\n\n        #in predict mode, I also want to retrieve a file name\n        if self.return_id:\n            return batch_tensor, label_tensor, str_id\n        else:\n            return batch_tensor, label_tensor","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T11:34:20.348695Z","iopub.execute_input":"2025-12-19T11:34:20.349249Z","iopub.status.idle":"2025-12-19T11:34:20.372466Z","shell.execute_reply.started":"2025-12-19T11:34:20.349217Z","shell.execute_reply":"2025-12-19T11:34:20.371330Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class OnlineDataset(torch.utils.data.Dataset):\n    def __init__(        \n        self,\n        meta_df,\n        label_encoder,\n        config: AudioConfig,\n        group_mode:bool = True,\n        return_id: bool = True,\n        prediction_as_target: bool = False,\n        root_dir:str = '/kaggle/input/birdclef-2025/train_audio/',\n        multitarget:bool = True,\n    ):\n        super().__init__()\n        self.cfg = config\n        #sklearn labelencoder\n        self.label_encoder = label_encoder\n        #files root dir\n        self.root_dir = root_dir\n        #flag to retrieve whole sequnces instead of just tokens\n        self.group_mode = group_mode\n        #flag to retrieve filenames to sequence of tensors\n        self.return_id = return_id\n        #includes the secondary labels as primary\n        self.multitarget = multitarget\n        #flag to retrive pseudo labels\n        self.prediction_as_target = prediction_as_target\n        \n        #df that contains info about each file\n        self.meta_df = meta_df\n        self.predictions = None \n\n    def update_pseudo_labels(self, obj:pd.Series):\n        \"\"\"\n        appends new pseudolabels into existing pseudolabels storage\n        \"\"\"\n        if self.predictions is None:\n            self.predictions = pd.Series()\n\n        self.predictions = pd.concat([self.predictions, obj])\n\n    def __len__(self):\n        return len(self.meta_df)\n\n    def __getitem__(self, index):\n        temp_ds = BirdDataset(\n            meta_df = self.meta_df.iloc[index],\n            label_encoder = self.label_encoder,\n            config = self.cfg,\n            group_mode = self.group_mode,\n            return_id = self.return_id,\n            prediction_as_target = self.prediction_as_target,\n            root_dir = self.root_dir,\n            multitarget = self.multitarget,\n        )\n        if self.prediction_as_target:\n            temp_ds.predictions = self.predictions[index]\n            \n        return temp_ds[:]\n        ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T11:34:20.373525Z","iopub.execute_input":"2025-12-19T11:34:20.373811Z","iopub.status.idle":"2025-12-19T11:34:20.393016Z","shell.execute_reply.started":"2025-12-19T11:34:20.373781Z","shell.execute_reply":"2025-12-19T11:34:20.391753Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#AI generated (not slope)\nfrom torch.utils.data import Sampler\n\nclass IndexBatchSampler(Sampler):\n    def __init__(self, data_source, batch_size: int, shuffle: bool = True, drop_last: bool = False):\n        \"\"\"\n        Yields a list of indices at each iteration instead of a single index.\n        \n        Args:\n            data_source: The dataset (used to determine length).\n            batch_size: Number of indices to yield per iteration.\n            shuffle: Whether to shuffle indices before batching.\n            drop_last: Whether to drop the last incomplete batch.\n        \"\"\"\n        self.data_source = data_source\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.drop_last = drop_last\n\n    def __iter__(self):\n        # Create a list of all indices\n        indices = list(range(len(self.data_source)))\n        \n        if self.shuffle:\n            np.random.shuffle(indices)\n            \n        # Yield chunks (batches) of indices\n        batch = []\n        for idx in indices:\n            batch.append(idx)\n            if len(batch) == self.batch_size:\n                yield batch\n                batch = []\n        \n        # Handle the remaining items\n        if len(batch) > 0 and not self.drop_last:\n            yield batch\n\n    def __len__(self):\n        # Calculate how many batches this sampler will produce\n        if self.drop_last:\n            return len(self.data_source) // self.batch_size\n        else:\n            return (len(self.data_source) + self.batch_size - 1) // self.batch_size","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T11:34:20.395294Z","iopub.execute_input":"2025-12-19T11:34:20.395737Z","iopub.status.idle":"2025-12-19T11:34:20.415798Z","shell.execute_reply.started":"2025-12-19T11:34:20.395702Z","shell.execute_reply":"2025-12-19T11:34:20.414720Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torch import nn\nimport timm \nclass Encoder(nn.Module):\n    def __init__(self, backbone_name:str, original_weights: bool = True):\n        super().__init__()\n        self.backbone = timm.create_model(\n            backbone_name, \n            pretrained=original_weights, \n            num_classes=0,\n        )\n        #init data-transform functions\n        self.stft_transform = None\n        self.db_transform = None\n        self.mel_transform = None\n        self.pooler = None\n\n    def set_audiopreprocessing(self, cfg):\n        self.stft_transform = transforms.Spectrogram(\n            n_fft=cfg.n_fft, \n            hop_length=cfg.hop_length,\n            power=1.0,\n        )\n        self.db_transform = transforms.AmplitudeToDB(\n            stype=\"magnitude\", \n            top_db=cfg.top_db\n        )\n        self.mel_transform = transforms.MelScale(\n            sample_rate=cfg.sr,\n            n_stft=cfg.n_fft // 2 + 1,\n            n_mels=cfg.n_mels,\n        )\n        self.pooler = nn.AdaptiveMaxPool1d(cfg.n_mels)\n        \n    def stripe_w_overlap(\n        self, \n        ar:torch.Tensor, \n        stripe:int, \n        overlap:int,\n        pad_value:float,\n    ):\n        \"\"\"\n        splits tensor into overlaping chunks along time dim\n        the last token is dropped\n        if the time < token_width => pad with constant value of pad_value\n        returns (B(depends on input), C, H, W, tokens)\n        \"\"\"\n        step = stripe - overlap\n        time_len = ar.shape[-1] #time\n\n        #if time < stripe => pad with zeros, because we can't stack None values\n        if time_len < stripe:\n            ar = torch.nn.functional.pad(\n                input = ar,\n                pad = (0, stripe - time_len), \n                mode = 'constant', \n                value = pad_value\n                \n            )\n            num_steps = 1\n        #if time < stripe+step means that there is possible only 1 step \n        elif time_len < stripe+step:\n            num_steps = 1\n        #number of full chunks\n        else:\n            num_steps = (time_len - stripe) // step\n\n        try:\n            striped_tensor = torch.stack(\n                [\n                    ar[:,..., i * step : i * step + stripe] \n                     for i in range(num_steps)\n                ]\n            ) #(T, ...)\n        except: \n            raise RuntimeError(f\"error in stripes. here is the tensor shape: {ar.shape} ; the stripe: {stripe} ; and overlap: {overlap} ; and steps: {num_steps} \")\n\n        striped_tensor = torch.moveaxis(striped_tensor, 0, -1).contiguous() # (T, ...) -> (..., T)\n        \n        return striped_tensor\n        \n    def forward(self, x, **kwargs):\n        #(B*T, C, H, W) -> (B*T, C_backbone)\n        \n        #==prerpcoessing raw waves into images\n        #(B*T, C, H, W) -> (B, C_spects, C_mel, W_stft)\n        \n        stft = self.stft_transform(x)\n        stft = stft[..., :320] #todo idk why, but I started to receive 321 windows after moving stiping on raw waves\n        #conver to dB spectrs\n        mel_out_stft = self.db_transform(self.mel_transform(stft))\n        mel_out_stft_2 = self.db_transform(self.mel_transform(stft**2))\n        stft = self.db_transform(stft)\n        \n        #normalize stft to mel size\n        mega_B, H, W, C_stft, W_stft = stft.shape\n        stft = torch.transpose(stft, -1, -2) # mega_B, H, W, C_stft, W_stft -> mega_B, H, W, W_stft, C_stft\n\n        #flatten B, H, W, T into a single *B* dimension \n        stft = stft.reshape(-1, 1, C_stft) #(B,H,W,W_s,C) -> (*B*, 1, C)\n        pooled_lin = self.pooler(stft) # (*B*, 1, C_stft) -> (*B*, 1, C_mel) \n        pooled_lin = pooled_lin.reshape(mega_B, H, W, W_stft, -1) #(mega_B, H, W, W_stft, C_mel)\n        \n        x = torch.stack((mel_out_stft, mel_out_stft_2, pooled_lin), dim = 1) #B, C_spects, H, W, C_mel, W_stft \n        x = x.squeeze(2).squeeze(2) #B, C_spects, C_mel, W_stft \n        #==prerpcoessing raw waves into images\n        \n        return self.backbone(x) #(B*T, C, H, W) -> (B*T, C)\n\nclass Classifier(nn.Module):\n    def __init__(\n        self, \n        seq_mode = True, #to process instance wise or sequence wise\n        single_head:nn.Module = None,\n        multi_head:nn.Module = None,\n        single_activation:nn.Module = None,\n        multi_activation:nn.Module = None,\n\n    ):\n        super().__init__()\n        self.seq_mode = seq_mode\n        self.single_target_model = single_head\n        self.multi_target_model = multi_head\n        #activation fn in this class instead of token encoder, if I want to thrain only 1 SED head\n        self.single_activation = single_activation\n        self.multi_activation = multi_activation\n    \n    def forward(\n            self, \n            x, \n            multitarget_mask = None,\n            attention_mask:torch.Tensor = None,\n            return_NoF:bool = False,\n            pool:bool = True,\n        ):\n        \"\"\"\n        Pass padded vision embeddings in classifier with implementing additional logic:\n        * multi target model split(multitarget_mask)\n        * removes NoF token (return_NoF = False)\n        * returns prediction token wise (pool = False)\n\n        the classifier object's attribute seq_mode determines, whether to return list of token-wise predictions\n        or padded tensor of token-wise predictions\n        Args:\n            x: features from vision encoder.\n            multitarget_mask: list of booleans, where True means to use multitarget model.\n            return_NoF: whether to return NoF token.\n            pool: wheter to return output sequence wise or token wise.\n        \"\"\"\n\n        #B, T_max, Channels -> B, empty/T_max, Class_digit+NoF\n        #we have rotary embeddings, so we can't pass empty batches anymore\n        if torch.all(multitarget_mask) == True:\n            output = self.multi_target_model(x, return_NoF, attention_mask, pool)\n            output = self.multi_activation(output)\n        elif torch.any(multitarget_mask) == True:\n            multi_output = self.multi_target_model(x[multitarget_mask], return_NoF, attention_mask[multitarget_mask], pool)\n            multi_output = self.multi_activation(multi_output)\n            single_output = self.single_target_model(x[~multitarget_mask], return_NoF, attention_mask[~multitarget_mask], pool)\n            single_output = self.single_activation(single_output)\n            output = torch.cat([multi_output, single_output], dim = 0)\n        else:\n            output = self.single_target_model(x, return_NoF, attention_mask, pool)\n            output = self.single_activation(output)\n    \n        if not self.seq_mode:\n\n            masked_output = []\n            last_indx = torch.sum(attention_mask, dim = 1).int()\n            \n            #list of token predictions for each instance\n            for i, out in enumerate(output):\n                masked_output.append(out[:last_indx[i]].cpu().detach())\n\n            output = masked_output\n            \n        return output\n\n\nclass CLEFModel(nn.Module):\n    def __init__(\n        self, \n        encoder = None, \n        classifier = None,\n        padding_value = 0,\n        return_NoF = False,\n    ):\n        super().__init__()\n        self.encoder = encoder\n        self.classifier = classifier\n        self.padding_value = padding_value\n        self.return_NoF = return_NoF\n\n    def pad_list_to_tensor(self, tensor_list:list, padding_value:float = None):\n        \"\"\"\n        takes a list of embeddings [(T_i, C), ...] and creates out of them 1 padded tensor\n        \"\"\"\n        if padding_value is None:\n            padding_value = self.padding_value\n        #in batch maximum for padding (in tokens)\n        max_len = np.max([s.shape[0] for s in tensor_list])\n        \n        batch_list = []\n        attention_mask_list = []\n        for tensor in tensor_list:\n            #create attention mask, to prevent usage of padded values in the future\n            attention_mask = torch.ones(tensor.shape[0])\n            attention_mask = torch.nn.functional.pad(\n                attention_mask, \n                (0, max_len - tensor.shape[0]), #token wise padding\n                mode ='constant',\n                value = 0\n            )\n            \n            attention_mask_list.append(attention_mask)\n            \n            tensor = torch.nn.functional.pad(\n                            tensor, \n                            (0,0,0, max_len - tensor.shape[0]), #token wise padding\n                            mode ='constant', \n                            value = padding_value #equivavelnt of zero\n                        )\n        \n            batch_list.append(tensor)\n        \n        batch_tensor = torch.stack(batch_list, dim=0) # B, max(T_b), C\n        attention_mask = torch.stack(attention_mask_list, dim=0) # B, max(T_b)\n        \n        return batch_tensor, attention_mask\n\n    def forward(\n        self, \n        x, \n        multitarget_mask,\n        pool,\n    ):\n        \"\"\"\n        Pass list of sequences into vision encoder and classifier models.\n        \n        seq-wise:\n        Unioun list of tensor into huge one, to pass it through encoder,\n        The received features are padded, to create once again 1 huge tensor for classifier\n        \"\"\"\n        #list -> predict seq-wise\n        if not isinstance(x, torch.Tensor):\n            \n            #huge tensor for feature encoding \n            x_feature_tensor = torch.cat(x, dim = -1).permute(-1, 0, 1, 2).to(device) # (T*B, C, H, W)\n            try:\n                x_feature_tensor = self.encoder(x_feature_tensor) #(T*B, C)\n            except:\n                raise ValueError(f\"Most likely cuda memmory allocation, tensor shape: {x_feature_tensor.shape}\")\n            \n            #split back into seqs \n            prev_len = 0\n            embed = [] # (B, T_b, C)\n            for sub_x in x:\n                cur_len = sub_x.shape[-1]\n                embed.append(x_feature_tensor[prev_len : prev_len+cur_len])# (T_b, C)\n                prev_len = prev_len+cur_len\n                \n            #pad sequnces\n            embed, attention_mask = self.pad_list_to_tensor(embed)\n        else:        \n            embed = self.encoder(x.to(device))\n            attention_mask = torch.ones((embed.shape(0), embed.shape(1)))\n\n        probs = self.classifier(\n            x = embed, \n            attention_mask = attention_mask,\n            multitarget_mask = multitarget_mask,\n            return_NoF = self.return_NoF,\n            pool = pool\n        )\n        return probs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T11:34:20.489605Z","iopub.execute_input":"2025-12-19T11:34:20.489944Z","iopub.status.idle":"2025-12-19T11:34:20.519623Z","shell.execute_reply.started":"2025-12-19T11:34:20.489916Z","shell.execute_reply":"2025-12-19T11:34:20.518588Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class ETransformerBlock(nn.Module):\n    def __init__(\n        self,\n        embedding_size,\n        num_head,\n        d_embed,\n        d_c,\n        d_c1,\n        d_rotate,\n    ):\n        super().__init__()\n        self.rms_n = nn.RMSNorm(embedding_size)\n        self.att = MultiHeadLatentAttention(\n            d_model = embedding_size, #after attention embedding_size\n            num_head = num_head, \n            d_embed = d_embed, #input to attention embedding_size\n            d_c = d_c, \n            d_c1 = d_c1, \n            d_rotate = d_rotate, \n            dropout=0.1, \n            bias=True,\n        )\n        self.liner = nn.Linear(embedding_size, embedding_size)\n        self.lin_act = nn.SiLU()\n\n    def forward(self, x, att_mask):\n        res = x\n        x = self.rms_n(x)\n        x = self.att(sequence = x, att_mask = att_mask)\n        x += res\n        res = x\n        x = self.rms_n(x)\n        x = self.liner(x)\n        x = self.lin_act(x)\n        x += res\n        return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T11:34:20.521295Z","iopub.execute_input":"2025-12-19T11:34:20.522054Z","iopub.status.idle":"2025-12-19T11:34:20.542406Z","shell.execute_reply.started":"2025-12-19T11:34:20.522011Z","shell.execute_reply":"2025-12-19T11:34:20.541399Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class TokenEncoder(nn.Module):\n    def __init__(\n        self, \n        embedding_size,\n        class_num,\n        num_head,\n        d_c,\n        d_c1,\n        d_rotate,\n        pool_type:str = 'cls',\n    ):\n        super().__init__()\n        #self.vembed2embed = nn.Linear(vembedding_size, embedding_size)\n        self.output_size = class_num+1\n        self.embedding_size = embedding_size\n        self.num_head = num_head\n        self.embed2embed = ETransformerBlock(\n            self.embedding_size,\n            num_head,\n            self.embedding_size,\n            d_c,\n            d_c1,\n            d_rotate,\n        )\n        self.embed2class = nn.Linear(embedding_size, self.output_size)\n        self.cls_token = nn.Parameter(torch.randn(1, 1, embedding_size))\n        self.pool_type = pool_type\n        \n    def forward(\n        self,\n        x,\n        return_NoF,\n        attention_mask:torch.Tensor = None,\n        pool:bool = True,\n    ):\n        \n        batch_size = x.shape[0]\n        cls = self.cls_token.expand(batch_size, -1, -1)\n        x = torch.cat((cls, x), dim=1) #B, T, E -> B, T+1, E\n        #todo attention mask expand\n        attention_mask = torch.cat(\n            (torch.ones(attention_mask.shape[0]).unsqueeze(-1), attention_mask), \n            dim = -1\n        ) #B, T -> B, T+1 {1, 0}\n        \n        q_mask = attention_mask.unsqueeze(2) \n        k_mask = attention_mask.unsqueeze(1)\n        attention_matrix = q_mask * k_mask # (B, T+1, T+1)\n        attention_matrix -= 1 #{0 ,-1}\n        attention_matrix *= 1e9 #{0, -inf}\n        attention_matrix = attention_matrix.unsqueeze(1).expand(-1, self.num_head, -1, -1).to(device) #(B, H, T+1, T+1)\n        output = self.embed2embed(x, att_mask = attention_matrix) #B, T+1, E -> B, T+1, E #todo probably bolleans to int in att_mask\n        #output = x #for simplicity in test setup\n\n        if pool:\n            #or in model cls_token pool\n            if self.pool_type == 'cls':\n                output = output[:, 0, :] #(B, 1, E)\n                \n            #or token_avg pool\n            elif self.pool_type == 'mean':\n                \n                output = output[:, 1:, :].to(device)\n                attention_mask = attention_mask[:, 1:].to(device)\n                last_indx = torch.sum(attention_mask, dim = 1).int().unsqueeze(-1).to(device) # (B, 1)\n                \n                #handle empty batch\n                if output.shape[0] == 0:\n                    output = torch.empty(0, self.embedding_size, dtype=output.dtype, device=output.device)\n                else:\n                    #because 'output' contains padded tokens (with zeros) calculated stats will also include them\n                    #to play it safe, I want to multiply 'output' on 'att_mask'\n                    output = (output * attention_mask.unsqueeze(-1)).sum(dim = 1) / last_indx #B, T, E -> B, E\n            else:\n                raise ValueError(f\"pool_type {self.pool_type} isn't implemented yet\")\n        else:\n            output = output[:, 1:, :]\n\n        output = self.embed2class(output) #B, ?, E -> B, ?, C+1\n    \n        if not return_NoF:\n            #last digit is NoF\n            output = output[..., :-1]\n                \n        return output","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T11:34:20.543521Z","iopub.execute_input":"2025-12-19T11:34:20.543877Z","iopub.status.idle":"2025-12-19T11:34:20.571185Z","shell.execute_reply.started":"2025-12-19T11:34:20.543847Z","shell.execute_reply":"2025-12-19T11:34:20.570173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"w = 4\nw_b = 32\nnp.array([[int(i*w**(1+i/w_b)), int((i+1)*w**(1+i/w_b))]for i in range(12)])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T11:34:20.572498Z","iopub.execute_input":"2025-12-19T11:34:20.572839Z","iopub.status.idle":"2025-12-19T11:34:20.596550Z","shell.execute_reply.started":"2025-12-19T11:34:20.572800Z","shell.execute_reply":"2025-12-19T11:34:20.595358Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os \nimport pandas as pd\nimport numpy as np\nimport glob\n#=======csv-serving\n\nref_df = pd.DataFrame()\ndirectory = \"/kaggle/input/birdclef-2025/test_soundscapes/\"\nfilename = glob.glob(os.path.join(directory, \"*ogg\"))\nif len(filename) == 0:\n    print('no-test-dir')\n    directory = \"/kaggle/input/birdclef-2025/train_soundscapes\"\n    filename = glob.glob(os.path.join(directory, \"*ogg\"))[:3]\n    \nfilename = [os.path.basename(path) for path in filename]\ndummy_primaries = ['1139490'] * len(filename)\ndummy_sec = [\"['']\"] * len(filename)\nref_df['filename'] = filename\nref_df['primary_label'] = dummy_primaries\nref_df['secondary_labels'] = dummy_sec\n#=======csv-serving\n\n\n#=======model loading\nwith open(\"/kaggle/input/cltest/other/default/9/label_encoder.pkl\", 'rb') as f: #le path\n    le = pickle.load(f)\n    \nencoder = Encoder('efficientnet_b3a', original_weights = False)\nconfig = AudioConfig(\n    n_fft = 400,\n    win_length = 400,\n    hop_length = 400 // 2,\n    slicing_func=encoder.stripe_w_overlap,\n    n_mels = 320,\n    stripe_width = 320,\n    stripe_overlap = 200,\n    prior_cut_sec = 60,\n)\nencoder.set_audiopreprocessing(config)\nsingle_target_head = TokenEncoder(\n    embedding_size = encoder.backbone.num_features,\n    class_num = len(le.classes_),\n    num_head = 4,\n    d_c = 256,\n    d_c1 = 256,\n    d_rotate = 16,\n    pool_type = 'cls',\n)\nclassifier = Classifier(\n    seq_mode = True,\n    single_head = single_target_head,\n    #multi_head = multi_target_head, #todo\n    multi_head = single_target_head, #todo same model is trained in different modes\n    single_activation = nn.Softmax(dim = -1), #dim = -1, because tokenwise softmax \n    multi_activation= nn.Sigmoid(),\n)\nmodel = CLEFModel(\n    encoder = encoder,\n    classifier = classifier,\n    padding_value = 0,\n)\nmodel.load_state_dict(torch.load('/kaggle/input/cltest/other/default/9/dmodel_354.pt', map_location=torch.device('cpu'))) #model path\nmodel.eval()\n#=======model loading\n\nref_dataset = BirdDataset(\n    meta_df = ref_df[:2], \n    label_encoder = le,\n    config = config,\n    group_mode = True,\n    return_id = True,\n    root_dir = directory,\n)\n\n#=======saving csv\n\nresults_df = pd.DataFrame(columns = le.classes_)\nfor i in range(len(filename)):\n    sub_slices, _, filename = ref_dataset.preprocess_label_file(\n        ref_df['filename'].iloc[i], \n        '1139490', \n        \"['']\",\n    )\n    filename = filename.split('.')[0]\n\n    #window tricks\n    w = 4\n    w_b = 32\n\n    \n    for i in range(12):\n        new_ids = filename + f'_{(i+1)*5}'\n        #list of booleans is multitarget\n        #single boolean is a pool mode\n        \n        prediction = model(\n            [sub_slices[..., int(i*w**(1+i/w_b)): int((i+1)*w**(1+i/w_b))]],\n            torch.tensor([False]),\n            True\n        )\n        temp_df = pd.DataFrame(\n            data = prediction.detach().numpy(), \n            columns = le.classes_, \n            index = [new_ids],\n        )\n        results_df = pd.concat([results_df, temp_df])\n\nresults_df.to_csv('submission.csv', index_label = 'row_id')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T11:34:20.598623Z","iopub.execute_input":"2025-12-19T11:34:20.598951Z","iopub.status.idle":"2025-12-19T11:34:56.121858Z","shell.execute_reply.started":"2025-12-19T11:34:20.598921Z","shell.execute_reply":"2025-12-19T11:34:56.120673Z"}},"outputs":[],"execution_count":null}]}