{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":11828260,"sourceType":"datasetVersion","datasetId":7430593},{"sourceId":167220511,"sourceType":"kernelVersion"},{"sourceId":189366517,"sourceType":"kernelVersion"},{"sourceId":243044026,"sourceType":"kernelVersion"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- https://github.com/Choiuijin1125/bclef2022/\n- https://www.kaggle.com/code/hinepo/transfer-learning-with-timm-models-and-pytorch#Read","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport warnings\nimport logging\nimport time\nimport math\nimport cv2\nfrom pathlib import Path\nimport joblib\n# import openvino.runtime as ov\nimport numpy as np\nimport pandas as pd\nimport librosa\nimport soundfile as sf\nfrom soundfile import SoundFile\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchaudio.transforms as AT\nfrom contextlib import contextmanager\nimport scipy\nfrom torch.cuda.amp import autocast, GradScaler\nimport timm\nfrom tqdm.auto import tqdm\nfrom glob import glob\nimport torchaudio\nimport random\nimport itertools\nfrom typing import Union\n\nimport concurrent.futures\n\nwarnings.filterwarnings(\"ignore\")\nlogging.basicConfig(level=logging.ERROR)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T09:49:16.380891Z","iopub.execute_input":"2025-06-05T09:49:16.381453Z","iopub.status.idle":"2025-06-05T09:49:21.365300Z","shell.execute_reply.started":"2025-06-05T09:49:16.381411Z","shell.execute_reply":"2025-06-05T09:49:21.364555Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport sys\n\nsys.path.append('/kaggle/usr/lib/kaggle_metric_utilities/')\nsys.path.append('/kaggle/usr/lib/birdclef-roc-auc/')\n\n\nimport time\nimport random # for torch seed\nimport os # for torch seed\n\nfrom sklearn.model_selection import train_test_split\n\nimport torch\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import Adam, AdamW, RMSprop # optmizers\nfrom torch.optim.lr_scheduler import CosineAnnealingLR, ReduceLROnPlateau # Learning rate schedulers\n\nimport albumentations as A\n# from albumentations.pytorch import ToTensorV2\n\nimport timm\n\n\nimport librosa\n\nimport torch.optim as optim\nimport torchaudio\nimport torchaudio.transforms as AT\nfrom torchvision import models\nimport torch.distributed as dist\nimport torch.multiprocessing as mp\nfrom torch.nn.parallel import DistributedDataParallel as DDP\nfrom torch.utils.data.distributed import DistributedSampler\n\n\nfrom tqdm import tqdm\n\n\ndef setup(rank, world_size):\n    os.environ['MASTER_ADDR'] = 'localhost'\n    os.environ['MASTER_PORT'] = '12355'\n\n    # initialize the process group\n    dist.init_process_group(\"gloo\", rank=rank, world_size=world_size)\n\ndef cleanup():\n    dist.destroy_process_group()\n\nfrom metric import score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T12:17:27.846016Z","iopub.execute_input":"2025-06-05T12:17:27.846324Z","iopub.status.idle":"2025-06-05T12:17:27.854410Z","shell.execute_reply.started":"2025-06-05T12:17:27.846296Z","shell.execute_reply":"2025-06-05T12:17:27.853555Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('timm version', timm.__version__)\nprint('torch version', torch.__version__)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T16:13:23.025667Z","iopub.execute_input":"2025-06-04T16:13:23.026701Z","iopub.status.idle":"2025-06-04T16:13:23.031450Z","shell.execute_reply.started":"2025-06-04T16:13:23.026673Z","shell.execute_reply":"2025-06-04T16:13:23.030546Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class_labels = pd.read_csv('/kaggle/input/birdclef-2025/train.csv')['primary_label'].unique()\n# class_labels = sorted(os.listdir(train_dir))\nlen(class_labels)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T12:17:30.303932Z","iopub.execute_input":"2025-06-05T12:17:30.304622Z","iopub.status.idle":"2025-06-05T12:17:30.419064Z","shell.execute_reply.started":"2025-06-05T12:17:30.304597Z","shell.execute_reply":"2025-06-05T12:17:30.418341Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"custom_train_dir = '/kaggle/input/bird-clef-2025-data-preprocessing/birdclef-2025-custom/train_audio_5s_clip/'\ntrain_dir = '/kaggle/input/birdclef-2025/train_audio/'\n\ntrain_meta = pd.read_csv('/kaggle/input/bird-clef-2025-data-preprocessing/birdclef-2025-custom/custom_train.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T12:17:32.118978Z","iopub.execute_input":"2025-06-05T12:17:32.119636Z","iopub.status.idle":"2025-06-05T12:17:32.356307Z","shell.execute_reply.started":"2025-06-05T12:17:32.119613Z","shell.execute_reply":"2025-06-05T12:17:32.355770Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"wav_sec = 5\nsample_rate = 32000\nmin_segment = sample_rate*wav_sec\n\nepochs = 8\n\n\n\nclass CFG:\n  DEBUG = False # True False\n\n  ### input: not configurable\n  IMG_HEIGHT = 28\n  IMG_WIDTH = 28\n  N_CLASS = len(class_labels)\n\n  ### split train and validation sets\n  split_fraction = 0.66\n\n  ### model\n  base_model_name = 'eca_nfnet_l0'  \n\n  ### training\n  print_freq = 100\n  BATCH_SIZE = 1024\n  N_EPOCHS = 3 if DEBUG else 40\n\n  ### set only one to True\n  save_best_loss = False\n  save_best_accuracy = True\n\n  ### optimizer\n  # optimizer = 'adam'\n  optimizer = 'adamw'\n  # optimizer = 'rmsprop'\n\n  LEARNING_RATE = 1e-7\n    \n  weight_decay = 0.1 # for adamw\n  l2_penalty = 0.01 # for RMSprop\n  rms_momentum = 0 # for RMSprop\n\n  ### learning rate scheduler (LRS)\n  # scheduler = 'ReduceLROnPlateau'\n  scheduler = 'CosineAnnealingLR'\n    \n  plateau_factor = 0.5\n  plateau_patience = 3\n  cosine_T_max = 4\n  cosine_eta_min = 1e-8\n  verbose = True\n\n  ### albumentations\n  probability = 0.6\n\n  random_seed = 88\n\ncfg = CFG()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T12:17:34.931095Z","iopub.execute_input":"2025-06-05T12:17:34.931804Z","iopub.status.idle":"2025-06-05T12:17:34.937126Z","shell.execute_reply.started":"2025-06-05T12:17:34.931781Z","shell.execute_reply":"2025-06-05T12:17:34.936474Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if cfg.DEBUG:\n  train_meta = train_meta.sample(frac = 0.1).reset_index(drop=True) \n  cfg.N_CLASS = len(np.unique(train_meta['primary_label']))\n\nprint(f'Using {cfg.N_CLASS} classes to train')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T20:59:34.140238Z","iopub.execute_input":"2025-06-04T20:59:34.140610Z","iopub.status.idle":"2025-06-04T20:59:34.147343Z","shell.execute_reply.started":"2025-06-04T20:59:34.140588Z","shell.execute_reply":"2025-06-04T20:59:34.146320Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# detect and define device \ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nprint(device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T09:49:33.502998Z","iopub.execute_input":"2025-06-05T09:49:33.503807Z","iopub.status.idle":"2025-06-05T09:49:33.567536Z","shell.execute_reply.started":"2025-06-05T09:49:33.503781Z","shell.execute_reply":"2025-06-05T09:49:33.566666Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# for reproducibility\ndef seed_torch(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_torch(seed = CFG.random_seed)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T20:59:38.574198Z","iopub.execute_input":"2025-06-04T20:59:38.574552Z","iopub.status.idle":"2025-06-04T20:59:38.584268Z","shell.execute_reply.started":"2025-06-04T20:59:38.574527Z","shell.execute_reply":"2025-06-04T20:59:38.583183Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# utilities","metadata":{}},{"cell_type":"code","source":"def cal_score(label, pred):\n    label = np.concatenate(label)\n    pred = np.concatenate(pred)\n\n    label_df = pd.DataFrame(label>0.5, columns=class_labels)\n    pred_df = pd.DataFrame(pred, columns=class_labels)\n    label_df['id'] = np.arange(len(label_df))\n    pred_df['id'] = np.arange(len(pred_df))\n\n    return score(label_df, pred_df, row_id_column_name='id')\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T20:59:41.523160Z","iopub.execute_input":"2025-06-04T20:59:41.523452Z","iopub.status.idle":"2025-06-04T20:59:41.529327Z","shell.execute_reply.started":"2025-06-04T20:59:41.523433Z","shell.execute_reply":"2025-06-04T20:59:41.528380Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Dataset Preparation","metadata":{}},{"cell_type":"code","source":"n_fft=1024\nwin_length=1024\nhop_length=512\nf_min=20\nf_max=15000\nn_mels=128\n\nmel_spectrogram = AT.MelSpectrogram(\n    sample_rate=sample_rate,\n    n_fft=n_fft,\n    win_length=win_length,\n    hop_length=hop_length,\n    center=True,\n    f_min=f_min,\n    f_max=f_max,\n    pad_mode=\"reflect\",\n    power=2.0,\n    norm='slaney',\n    n_mels=n_mels,\n    mel_scale=\"htk\",\n    # normalized=True\n)\n\nn_fft=1024\nwin_length=1024\nhop_length=512\nf_min=20\nf_max=15000\nn_mels=128\n\nmel_spectrogram = AT.MelSpectrogram(\n    sample_rate=sample_rate,\n    n_fft=n_fft,\n    win_length=win_length,\n    hop_length=hop_length,\n    center=True,\n    f_min=f_min,\n    f_max=f_max,\n    pad_mode=\"reflect\",\n    power=2.0,\n    norm='slaney',\n    n_mels=n_mels,\n    mel_scale=\"htk\",\n    # normalized=True\n)\n\nclass BirdclefDataset(Dataset):\n    def __init__(self, df, mode='train'):\n        self.df = df\n        self.mode = mode\n\n    def normalize_std(self, spec, eps=1e-23):\n        mean = torch.mean(spec)\n        std = torch.std(spec)\n        return torch.where(std == 0, spec-mean, (spec - mean) / (std+eps))\n                \n    def __getitem__(self, index):\n        # sig, _ = torchaudio.load(uri=input_path+self.df.iloc[index].filename,backend=\"soundfile\")\n        sig, _ = torchaudio.load(uri=custom_train_dir+self.df.iloc[index].filename,backend=\"soundfile\")\n        # sig_dim = sig.shape\n        \n        sig = sig / torch.max(torch.abs(sig))\n        sig = sig + 1.5849e-05*(torch.rand(1, min_segment)-0.5)      # adding noise to the data\n        melspec = mel_spectrogram(sig)\n        # melspec_dim = melspec.shape\n        \n        melspec_log = torch.log(melspec)\n        # melspec_log_dim = melspec.shape\n        \n        melspec_norm = self.normalize_std(melspec_log)\n        # melspec_norm_dim = melspec.shape\n\n        target = self.df.iloc[index].primary_label\n        y = np.array([1 if item == target else 0 for item in class_labels])\n        \n        return melspec_norm, y\n    \n    def __len__(self):\n        return len(self.df)\n        ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T09:15:37.410054Z","iopub.execute_input":"2025-06-05T09:15:37.410860Z","iopub.status.idle":"2025-06-05T09:15:37.493973Z","shell.execute_reply.started":"2025-06-05T09:15:37.410828Z","shell.execute_reply":"2025-06-05T09:15:37.492899Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dataset = BirdclefDataset(train_meta)\nX, y = dataset[0]\nX, y","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T20:59:47.347116Z","iopub.execute_input":"2025-06-04T20:59:47.347418Z","iopub.status.idle":"2025-06-04T20:59:47.398539Z","shell.execute_reply.started":"2025-06-04T20:59:47.347396Z","shell.execute_reply":"2025-06-04T20:59:47.397658Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X.shape, y.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T20:59:50.112717Z","iopub.execute_input":"2025-06-04T20:59:50.113287Z","iopub.status.idle":"2025-06-04T20:59:50.120321Z","shell.execute_reply.started":"2025-06-04T20:59:50.113229Z","shell.execute_reply":"2025-06-04T20:59:50.119332Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# multiprocessing dataloader\ndef prepare(rank, world_size, batch_size=32, pin_memory=False, num_workers=0):\n    train_df, val_df = train_test_split(train_meta, test_size=0.2, random_state=42)\n    \n    train_dataset = BirdclefDataset(train_df, mode='train')\n    train_sampler = DistributedSampler(train_dataset, num_replicas=world_size, rank=rank, shuffle=True, drop_last=False)\n    train_loader = DataLoader(train_dataset, batch_size=24, pin_memory=pin_memory, num_workers=2, drop_last=True, shuffle=False, sampler=train_sampler)\n\n    val_dataset = BirdclefDataset(val_df, mode='val')\n    val_sampler = DistributedSampler(val_dataset, num_replicas=world_size, rank=rank, shuffle=True, drop_last=False)\n    val_loader = DataLoader(val_dataset, batch_size=24, pin_memory=pin_memory, num_workers=2, drop_last=True, shuffle=False, sampler=val_sampler)\n    \n    return train_loader, val_loader","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T21:00:03.608248Z","iopub.execute_input":"2025-06-04T21:00:03.608899Z","iopub.status.idle":"2025-06-04T21:00:03.617281Z","shell.execute_reply.started":"2025-06-04T21:00:03.608874Z","shell.execute_reply":"2025-06-04T21:00:03.615758Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# MODEL","metadata":{}},{"cell_type":"code","source":"def init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n\n\ndef init_weights(model):\n    \"\"\"\n    Initializes neural network layer weights using appropriate schemes based on \n    layer type (Conv2d, BatchNorm, GRU, Linear) to improve training stability and convergence.\n    \"\"\"\n    \n    classname = model.__class__.__name__\n    if classname.find(\"Conv2d\") != -1:\n        nn.init.xavier_uniform_(model.weight, gain=np.sqrt(2))\n        model.bias.data.fill_(0)\n    elif classname.find(\"BatchNorm\") != -1:\n        model.weight.data.normal_(1.0, 0.02)\n        model.bias.data.fill_(0)\n    elif classname.find(\"GRU\") != -1:\n        for weight in model.parameters():\n            if len(weight.size()) > 1:\n                nn.init.orghogonal_(weight.data)\n    elif classname.find(\"Linear\") != -1:\n        model.weight.data.normal_(0, 0.01)\n        model.bias.data.zero_()\n\n\ndef apply_power_to_low_ranked_cols(\n    p: np.ndarray,\n    top_k: int = 30,\n    exponent: Union[int, float] = 2,\n    inplace: bool = True\n) -> np.ndarray:\n    if not inplace:\n        p = p.copy()\n\n    # Identify columns whose max value ranks below `top_k`\n    tail_cols = np.argsort(-p.max(axis=0))[top_k:]\n\n    # Apply the power transformation to those columns\n    p[:, tail_cols] = p[:, tail_cols] ** exponent\n    return p","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T19:07:06.787949Z","iopub.execute_input":"2025-06-04T19:07:06.788250Z","iopub.status.idle":"2025-06-04T19:07:06.805655Z","shell.execute_reply.started":"2025-06-04T19:07:06.788230Z","shell.execute_reply":"2025-06-04T19:07:06.804570Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class AttBlockV2(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"linear\"):\n        super().__init__()\n\n        self.activation = activation\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n\n    def forward(self, x):\n        norm_att = torch.softmax(torch.tanh(self.att(x)), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T16:22:08.833853Z","iopub.execute_input":"2025-06-04T16:22:08.834187Z","iopub.status.idle":"2025-06-04T16:22:08.841992Z","shell.execute_reply.started":"2025-06-04T16:22:08.834163Z","shell.execute_reply":"2025-06-04T16:22:08.841156Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class TimmSED(nn.Module):\n    def __init__(self, base_model_name: str, pretrained=False, num_classes=24, in_channels=1, n_mels=24):\n        super().__init__()\n\n        self.bn0 = nn.BatchNorm2d(n_mels)\n\n        base_model = timm.create_model(\n            base_model_name, pretrained=pretrained, in_chans=in_channels)\n        layers = list(base_model.children())[:-2]\n        self.encoder = nn.Sequential(*layers)\n\n        in_features = base_model.num_features\n\n        self.fc1 = nn.Linear(in_features, in_features, bias=True)\n        self.att_block2 = AttBlockV2(\n            in_features, num_classes, activation=\"sigmoid\")\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n\n    def forward(self, input_data):\n        x = input_data.transpose(2, 3)\n        x = torch.cat((x, x, x), 1)\n\n        x = x.transpose(2, 3)\n\n        x = self.encoder(x)\n\n        x = torch.mean(x, dim=2)\n\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block2(x)\n        logit = torch.sum(norm_att * self.att_block2.cla(x), dim=2)\n\n        output_dict = {\n            'logit': logit,\n        }\n\n        return output_dict\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T19:03:25.819943Z","iopub.execute_input":"2025-06-04T19:03:25.820246Z","iopub.status.idle":"2025-06-04T19:03:25.829374Z","shell.execute_reply.started":"2025-06-04T19:03:25.820225Z","shell.execute_reply":"2025-06-04T19:03:25.828388Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = TimmSED(cfg.base_model_name, num_classes=cfg.N_CLASS, in_channels=3, n_mels=128)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T19:40:46.297430Z","iopub.execute_input":"2025-06-04T19:40:46.297857Z","iopub.status.idle":"2025-06-04T19:40:46.774171Z","shell.execute_reply.started":"2025-06-04T19:40:46.297825Z","shell.execute_reply":"2025-06-04T19:40:46.773205Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"p = model(torch.rand(1, 1,128, 313))\n\np = torch.sigmoid(p['logit']).detach().cpu().numpy()\np = apply_power_to_low_ranked_cols(p, top_k=30, exponent=2)\nprint(p.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T19:41:17.851272Z","iopub.execute_input":"2025-06-04T19:41:17.851613Z","iopub.status.idle":"2025-06-04T19:41:18.056039Z","shell.execute_reply.started":"2025-06-04T19:41:17.851591Z","shell.execute_reply":"2025-06-04T19:41:18.055249Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## training and validation function","metadata":{}},{"cell_type":"code","source":"def train_fn(train_loader, model, criterion, optmizer, rank):\n  # switch to train mode\n  model.train()\n    \n    \n  size = len(train_loader.dataset)\n  num_batches = len(train_loader)\n\n  # loss, correct = 0, 0\n  pred_train = []\n  label_train = []\n  running_loss = 0.0\n\n  ################################# train #################################\n\n  for batch, (X, y) in tqdm(enumerate(train_loader, desc=f\"[Rank {rank}] Epoch {epoch+1} Training\")):\n\n    # start = time.time()\n\n    device = torch.device(rank)\n    X, y = X.to(rank), y.to(rank)  \n\n    # compute predictions and loss\n    optimizer.zero_grad()\n    pred = model(X)\n    loss = criterion(pred, y.to(torch.float32)) \n    # current = batch * len(X)\n\n    # Backpropagation: only in train function, not done in validation function\n    loss.backward()\n    optimizer.step()\n\n    # sum correct predictions\n    running_loss_val += loss.item()\n    pred_val.append(torch.softmax(outputs,dim=1).detach().cpu().numpy())\n    label_val.append(labels.detach().cpu().numpy())\n      \n    # end = time.time()\n    # time_delta = np.round(end - start, 3)\n\n    \n\n  return pred_train, label_train, running_loss_train\n\n\n\n\ndef valid_fn(valid_loader, model, criterion, rank):\n  model.eval()\n\n  size = len(valid_loader.dataset)\n  num_batches = len(valid_loader)\n\n  loss, correct = 0, 0\n\n  ################################# validation #################################\n\n  with torch.no_grad(): # disable gradients\n    pred_val = []\n    label_val = []\n    running_loss_val = 0.0\n      \n    for batch, (X, y) in tqdm(enumerate(valid_loader), desc=f\"[Rank {rank}] Epoch {epoch+1} Validation\"):\n\n      # start = time.time()\n\n      device = torch.device(rank)\n      X, y = X.to(rank), y.to(rank)\n\n      # compute predictions and loss\n      pred = model(X)\n      loss = criterion(pred, labels.to(torch.float32)) \n      # current = batch * len(X)\n      \n      # sum correct predictions\n      running_loss_val += loss.item()\n      pred_val.append(torch.softmax(outputs,dim=1).detach().cpu().numpy())\n      label_val.append(labels.detach().cpu().numpy())\n      \n      # end = time.time()\n      # time_delta = np.round(end - start, 3)\n      \n      \n  return pred_val, label_val, running_loss_val","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## optimizer\n[torch optimizer docs](https://docs.pytorch.org/docs/stable/optim.html)\n## LR Scheduler\n[torch LRS docs](https://docs.pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate)","metadata":{}},{"cell_type":"code","source":"def get_optimizer(lr = CFG.LEARNING_RATE):\n\n  if CFG.optimizer == 'adam':\n      optimizer = Adam(model.parameters(), lr=lr, weight_decay = CFG.weight_decay, amsgrad = False)\n\n  elif CFG.optimizer == 'adamw':\n      optimizer = AdamW(model.parameters(), lr = lr, weight_decay = CFG.weight_decay)\n\n  elif CFG.optimizer == 'rmsprop':\n      optimizer = RMSprop(model.parameters(), lr = lr, weight_decay = CFG.l2_penalty, momentum = CFG.rms_momentum)\n\n  else:\n      print('Optimizer is not defined')      \n\n  return optimizer\n\n\ndef get_scheduler(optimizer):\n\n  if CFG.scheduler=='ReduceLROnPlateau':\n      scheduler = ReduceLROnPlateau(optimizer, mode='max', factor = CFG.plateau_factor, patience = CFG.plateau_patience, verbose = CFG.verbose)\n\n  elif CFG.scheduler=='CosineAnnealingLR':\n      scheduler = CosineAnnealingLR(optimizer, T_max = CFG.cosine_T_max, eta_min = CFG.cosine_eta_min)\n\n  else:\n      print('LR Scheduler is not defined')\n\n  return scheduler ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T16:56:47.126222Z","iopub.execute_input":"2025-06-04T16:56:47.126593Z","iopub.status.idle":"2025-06-04T16:56:47.133681Z","shell.execute_reply.started":"2025-06-04T16:56:47.126568Z","shell.execute_reply":"2025-06-04T16:56:47.132975Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## start training","metadata":{}},{"cell_type":"code","source":"# multiprocessing training\ndef demo_basic(rank, world_size):\n    os.makedirs('/kaggle/working/weights/', exist_ok=True)\n    print(f\"Running basic DDP example on rank {rank}.\")\n    setup(rank, world_size)\n\n    # create model and move it to GPU with id rank\n    _model = TimmSED(base_model_name=cfg.base_model_name, pretrained=True).to(rank)\n    ddp_model = DDP(_model, device_ids=[rank])\n\n    train_loader, val_loader = prepare(rank, world_size)\n    criterion = nn.CrossEntropyLoss()\n    optimizer = get_optimizer(lr=cfg.LEARNING_RATE)\n    scheduler = get_scheduler(optimizer)\n\n    train_loader, val_loader = prepare(rank, world_size)\n\n    for epoch in range(epochs):\n        pred_train, label_train, running_loss = train_fn(train_loader, _model, criterion, optmizer, rank)\n\n        pred_val, label_val, running_loss_val = val_fn(val_loader, _model, criterion, rank)\n\n        if rank == 0:\n            auc_train_val = cal_score(label_train, pred_train)\n            auc_val = cal_score(label_val, pred_val)\n            print(f\"Epoch {epoch+1}/{epochs}, Loss: {running_loss/len(train_loader):.4f}, Loss_val: {running_loss_val/len(train_loader):.4f}\")\n            print(f\"Auc: {auc_train_val:.2f}% Auc_val: {auc_val:.2f}%\")\n    \n            torch.save(ddp_model.module.state_dict(), f\"weights/baseline_epoch{epoch+1}_{auc_train_val:.2f}_{auc_val:.2f}.pth\")\n\n    cleanup()\n\n\n\ndef run_demo(demo_fn, world_size):\n    mp.spawn(demo_fn,\n             args=(world_size,),\n             nprocs=world_size,\n             join=True)\n    \n\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    world_size = torch.cuda.device_count()\n    if world_size < 2:\n        raise RuntimeError(\"Need at least 2 GPUs for this demo\")\n    run_demo(demo_basic, world_size)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile demo.py\n\n# Standard library imports\nimport os\nimport shutil\nimport sys\nimport time\nimport random\nimport gc\nimport warnings\nimport logging\nimport math\nimport itertools\nfrom pathlib import Path\nfrom glob import glob\nfrom contextlib import contextmanager\nimport concurrent.futures\n\n# Scientific / data-handling libraries\nimport numpy as np\nimport pandas as pd\nimport scipy\nimport cv2\nimport soundfile as sf\n\n# Plotting\nimport matplotlib.pyplot as plt\n\n# Scikit-learn\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold\n\n# PyTorch core\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim import Adam, AdamW, RMSprop\nfrom torch.optim.lr_scheduler import CosineAnnealingLR, ReduceLROnPlateau\nimport torch.distributed as dist\nimport torch.multiprocessing as mp\nfrom torch.nn.parallel import DistributedDataParallel as DDP\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.utils.data.distributed import DistributedSampler\nfrom torch.utils.data import Subset\n\n\n# Audio-related\nimport torchaudio\nimport torchaudio.transforms as AT\nimport librosa\n\n# Computer vision / augmentation\nfrom torchvision import models\nimport albumentations as A\n\n# Model libraries\nimport timm\n\n# Progress bar\nfrom tqdm.auto import tqdm\n\n# Miscellaneous\nimport joblib\nfrom typing import Union\nimport gc\n\n# Custom metric utility paths\nsys.path.append('/kaggle/usr/lib/kaggle_metric_utilities/')\nsys.path.append('/kaggle/usr/lib/birdclef-roc-auc/')\n\n# Custom metric import\nfrom metric import score\n\n\n\nwarnings.filterwarnings(\"ignore\")\nlogging.basicConfig(level=logging.ERROR)\n\ndef setup(rank, world_size):\n    os.environ['MASTER_ADDR'] = 'localhost'\n    os.environ['MASTER_PORT'] = '12355'\n\n    # initialize the process group\n    dist.init_process_group(\"gloo\", rank=rank, world_size=world_size)\n\ndef cleanup():\n    dist.destroy_process_group()\n\n\n# dir name\nclass_labels = pd.read_csv('/kaggle/input/birdclef-2025/train.csv')['primary_label'].unique()\n\ncustom_train_dir = '/kaggle/input/bird-clef-2025-data-preprocessing/birdclef-2025-custom/train_audio_5s_clip/'\ntrain_dir = '/kaggle/input/birdclef-2025/train_audio/'\n\ntrain_meta = pd.read_csv('/kaggle/input/bird-clef-2025-data-preprocessing/birdclef-2025-custom/custom_train.csv')\n\n\n# config\nwav_sec = 5\nsample_rate = 32000\nmin_segment = sample_rate*wav_sec\n\nclass CFG:\n    DEBUG = True \n    \n    ### input: not configurable\n    N_CLASS = len(class_labels)\n    \n    ### split train and validation sets\n    split_fraction = 0.2\n    \n    ### model\n    base_model_name = 'eca_nfnet_l0'  \n    \n    ### training\n    print_freq = 100\n    BATCH_SIZE = 64\n    N_EPOCHS = 3 if DEBUG else 8\n    \n    ### set only one to True\n    save_best_loss = False\n    save_best_accuracy = True\n    \n    optimizer = 'adamw'\n\n    LEARNING_RATE = 1e-7\n    \n    weight_decay = 0.1 # for adamw\n\n    scheduler = 'CosineAnnealingLR'\n    \n    plateau_factor = 0.5\n    plateau_patience = 3\n    cosine_T_max = 4\n    cosine_eta_min = 1e-8\n    verbose = True\n\n    ### albumentations\n    probability = 0.6\n    \n    random_seed = 88\n\n\ncfg = CFG()\n\n\n\nif cfg.DEBUG:\n  train_meta = train_meta.sample(frac = 0.1).reset_index(drop=True) \n  cfg.N_CLASS = len(np.unique(train_meta['primary_label']))\n\nprint(f'Using {cfg.N_CLASS} classes to train')\n\n\n# detect and define device \ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nprint(device)\n\n# for reproducibility\ndef seed_torch(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_torch(seed = CFG.random_seed)\n\n\n##### utilities function\ndef cal_score(label, pred):\n    label = np.concatenate(label)\n    pred = np.concatenate(pred)\n\n    label_df = pd.DataFrame(label>0.5, columns=class_labels)\n    pred_df = pd.DataFrame(pred, columns=class_labels)\n    label_df['id'] = np.arange(len(label_df))\n    pred_df['id'] = np.arange(len(pred_df))\n\n    return score(label_df, pred_df, row_id_column_name='id')\n\n\n\n##### Dataset Preparation\nn_fft=1024\nwin_length=1024\nhop_length=512\nf_min=20\nf_max=15000\nn_mels=128\n\nmel_spectrogram = AT.MelSpectrogram(\n    sample_rate=sample_rate,\n    n_fft=n_fft,\n    win_length=win_length,\n    hop_length=hop_length,\n    center=True,\n    f_min=f_min,\n    f_max=f_max,\n    pad_mode=\"reflect\",\n    power=2.0,\n    norm='slaney',\n    n_mels=n_mels,\n    mel_scale=\"htk\",\n    # normalized=True\n)\n\nclass BirdclefDataset(Dataset):\n    def __init__(self, df, mode='train'):\n        self.df = df\n        self.mode = mode\n\n    def normalize_std(self, spec, eps=1e-23):\n        mean = torch.mean(spec)\n        std = torch.std(spec)\n        return torch.where(std == 0, spec-mean, (spec - mean) / (std+eps))\n                \n    def __getitem__(self, index):\n        sig, _ = torchaudio.load(uri=custom_train_dir+self.df.iloc[index].filename,backend=\"soundfile\")\n        \n        sig = sig / torch.max(torch.abs(sig))\n        sig = sig + 1.5849e-05*(torch.rand(1, min_segment)-0.5)      # adding noise to the data\n        melspec = mel_spectrogram(sig)\n        # melspec_dim = melspec.shape\n        \n        melspec_log = torch.log(melspec)\n        # melspec_log_dim = melspec.shape\n        \n        melspec_norm = self.normalize_std(melspec_log)\n        # melspec_norm_dim = melspec.shape\n\n        target = self.df.iloc[index].primary_label\n        y = np.array([1 if item == target else 0 for item in class_labels])\n        \n        return melspec_norm, y\n    \n    def __len__(self):\n        return len(self.df)\n\n\n\ndef n_fold_datasplit(df, n_splits):\n    kfold = KFold(n_splits=n_splits, shuffle=True, random_state=42)\n    train_dfs, val_dfs = [], []\n    \n    for fold, (train_idx, val_idx) in enumerate(kfold.split(df)):\n        # Create subsets for training and validation\n        train_df = df.iloc[train_idx].reset_index(drop=True)\n        val_df   = df.iloc[val_idx].reset_index(drop=True)\n\n        train_dfs.append(train_df)\n        val_dfs.append(val_df)\n        # train_subset = Subset(df, train_idx)\n        # val_subset = Subset(df, val_idx)\n    \n        # print(f\"Train subset size: {len(train_subset)}, Validation subset size: {len(val_subset)}\")\n    return train_dfs, val_dfs\n    \n\n# multiprocessing dataloader\ndef prepare(rank, world_size, test_size=cfg.split_fraction, batch_size=cfg.BATCH_SIZE, pin_memory=False):\n    # train_df, val_df = train_test_split(train_meta, test_size=test_size, random_state=42) \n\n    # train_dfs, val_dfs = n_fold_datasplit(train_meta, 3)\n    train_df = pd.read_csv('/kaggle/working/data_splits/split1/train_meta.csv')\n    val_df = pd.read_csv('/kaggle/working/data_splits/split1/val_meta.csv')\n\n\n    train_dataset = BirdclefDataset(train_df, mode='train')\n    train_sampler = DistributedSampler(train_dataset, num_replicas=world_size, rank=rank, shuffle=True, drop_last=False)\n    train_loader = DataLoader(train_dataset, batch_size=batch_size, pin_memory=pin_memory, num_workers=4, drop_last=True, shuffle=False, sampler=train_sampler)\n    \n    val_dataset = BirdclefDataset(val_df, mode='val')\n    val_sampler = DistributedSampler(val_dataset, num_replicas=world_size, rank=rank, shuffle=True, drop_last=False)\n    val_loader = DataLoader(val_dataset, batch_size=batch_size, pin_memory=pin_memory, num_workers=4, drop_last=True, shuffle=False, sampler=val_sampler)\n        \n    return train_loader, val_loader\n\n\n\n\n# <---------------------------------    Model  ------------------------------>\ndef init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n\n\ndef init_weights(model):\n    \"\"\"\n    Initializes neural network layer weights using appropriate schemes based on \n    layer type (Conv2d, BatchNorm, GRU, Linear) to improve training stability and convergence.\n    \"\"\"\n    \n    classname = model.__class__.__name__\n    if classname.find(\"Conv2d\") != -1:\n        nn.init.xavier_uniform_(model.weight, gain=np.sqrt(2))\n        model.bias.data.fill_(0)\n    elif classname.find(\"BatchNorm\") != -1:\n        model.weight.data.normal_(1.0, 0.02)\n        model.bias.data.fill_(0)\n    elif classname.find(\"GRU\") != -1:\n        for weight in model.parameters():\n            if len(weight.size()) > 1:\n                nn.init.orghogonal_(weight.data)\n    elif classname.find(\"Linear\") != -1:\n        model.weight.data.normal_(0, 0.01)\n        model.bias.data.zero_()\n\n\ndef apply_power_to_low_ranked_cols(\n    p: np.ndarray,\n    top_k: int = 30,\n    exponent: Union[int, float] = 2,\n    inplace: bool = True\n) -> np.ndarray:\n    if not inplace:\n        p = p.copy()\n\n    # Identify columns whose max value ranks below `top_k`\n    tail_cols = np.argsort(-p.max(axis=0))[top_k:]\n\n    # Apply the power transformation to those columns\n    p[:, tail_cols] = p[:, tail_cols] ** exponent\n    return p\n\n\n\nclass AttBlockV2(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"linear\"):\n        super().__init__()\n\n        self.activation = activation\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n\n    def forward(self, x):\n        norm_att = torch.softmax(torch.tanh(self.att(x)), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)\n\n\n\nclass TimmSED(nn.Module):\n    def __init__(self, base_model_name: str, pretrained=False, num_classes=24, in_channels=1, n_mels=24):\n        super().__init__()\n\n        self.bn0 = nn.BatchNorm2d(n_mels)\n    \n        base_model = timm.create_model(\n            base_model_name, pretrained=pretrained, in_chans=in_channels)\n        layers = list(base_model.children())[:-2]\n        self.encoder = nn.Sequential(*layers)\n\n        in_features = base_model.num_features\n\n        self.fc1 = nn.Linear(in_features, in_features, bias=True)\n        self.att_block2 = AttBlockV2(\n            in_features, num_classes, activation=\"sigmoid\")\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n\n    def forward(self, input_data):\n        x = input_data.transpose(2, 3)\n        x = torch.cat((x, x, x), 1)\n\n        x = x.transpose(2, 3)\n\n        x = self.encoder(x)\n\n        x = torch.mean(x, dim=2)\n\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block2(x)\n        logit = torch.sum(norm_att * self.att_block2.cla(x), dim=2)\n\n        output_dict = {\n            'logit': logit,\n        }\n\n        return output_dict\n\n\n# <------------------------------------------------                ------------------------------------------------>\n\n\n\n# <---------------------------------   Training and validation functions     --------------------------------------->\n\ndef train_fn(train_loader, model, criterion, optimizer, rank, epoch):\n  # switch to train mode\n  model.train()\n    \n    \n  size = len(train_loader.dataset)\n  num_batches = len(train_loader)\n\n  # loss, correct = 0, 0\n  pred_train = []\n  label_train = []\n  running_loss = 0.0\n\n  ################################# train #################################\n\n  for batch, (X, y) in tqdm(enumerate(train_loader), desc=f\"[Rank {rank}] Epoch {epoch+1} Training\"):\n\n    # start = time.time()\n\n    # device = torch.device(rank)\n    X, y = X.to(rank), y.to(rank)  \n\n    # compute predictions and loss\n    optimizer.zero_grad()\n    pred = model(X)\n    # pred = torch.sigmoid(pred['logit']).detach().cpu().numpy()\n    # pred = apply_power_to_low_ranked_cols(p, top_k=30, exponent=2)\n    # print(pred)\n    loss = criterion(pred['logit'], y.to(torch.float32)) \n    # current = batch * len(X)\n\n    # Backpropagation: only in train function, not done in validation function\n    loss.backward()\n    optimizer.step()\n\n    # sum correct predictions\n    running_loss += loss.item()\n    pred_train.append(torch.sigmoid(pred['logit']).detach().cpu().numpy())\n    label_train.append(y.detach().cpu().numpy())\n      \n    # end = time.time()\n    # time_delta = np.round(end - start, 3)\n\n    \n\n  return pred_train, label_train, running_loss\n\n\n\n\ndef valid_fn(valid_loader, model, criterion, rank, epoch):\n  model.eval()\n\n  size = len(valid_loader.dataset)\n  num_batches = len(valid_loader)\n\n  pred_val = []\n  label_val = []\n  running_loss_val = 0.0\n\n  # loss, correct = 0, 0\n\n  ################################# validation #################################\n\n  with torch.no_grad(): # disable gradients   \n    for batch, (X, y) in tqdm(enumerate(valid_loader), desc=f\"[Rank {rank}] Epoch {epoch+1} Validation\"):\n\n      # start = time.time()\n\n      # device = torch.device(rank)\n      X, y = X.to(rank), y.to(rank)\n\n      # compute predictions and loss\n      pred = model(X)\n      loss = criterion(pred['logit'], y.to(torch.float32)) \n      # current = batch * len(X)\n      \n      # sum correct predictions\n      running_loss_val += loss.item()\n      pred_val.append(torch.sigmoid(pred['logit']).detach().cpu().numpy())\n      label_val.append(y.detach().cpu().numpy())\n      \n      # end = time.time()\n      # time_delta = np.round(end - start, 3)\n      \n      \n  return pred_val, label_val, running_loss_val\n\n\n\n\n\n\n# <---------------------------------------------------    optimizer and LRS        ------------------------------------------------------->\n\ndef get_optimizer(model, lr = CFG.LEARNING_RATE):\n\n  if CFG.optimizer == 'adam':\n      optimizer = Adam(model.parameters(), lr=lr, weight_decay = CFG.weight_decay, amsgrad = False)\n\n  elif CFG.optimizer == 'adamw':\n      optimizer = AdamW(model.parameters(), lr = lr, weight_decay = CFG.weight_decay)\n\n  elif CFG.optimizer == 'rmsprop':\n      optimizer = RMSprop(model.parameters(), lr = lr, weight_decay = CFG.l2_penalty, momentum = CFG.rms_momentum)\n\n  else:\n      print('Optimizer is not defined')      \n\n  return optimizer\n\n\ndef get_scheduler(optimizer):\n\n  if CFG.scheduler=='ReduceLROnPlateau':\n      scheduler = ReduceLROnPlateau(optimizer, mode='max', factor = CFG.plateau_factor, patience = CFG.plateau_patience, verbose = CFG.verbose)\n\n  elif CFG.scheduler=='CosineAnnealingLR':\n      scheduler = CosineAnnealingLR(optimizer, T_max = CFG.cosine_T_max, eta_min = CFG.cosine_eta_min)\n\n  else:\n      print('LR Scheduler is not defined')\n\n  return scheduler \n\n\n\n\n# multiprocessing training\ndef demo_basic(rank, world_size):\n    save_model_dir = f'/kaggle/working/weights/'\n    if os.path.isdir(save_model_dir):\n        shutil.rmtree(save_model_dir)\n\n    \n\n    in_channels = 3\n    n_mels = 128\n    \n\n\n    # load inference model function\n    # MODELS = [f'/kaggle/input/birdclef-2025-sed-models-p/sed{i}.pth' for i in range(3)]\n    \n    def load_inf_model(base_model_name='eca_nfnet_l0', inchannels=3, n_mels=128, num_classes=206, n_model=1):\n        model = TimmSED(base_model_name=base_model_name,\n                   pretrained=False,\n                   num_classes=len(class_labels),\n                   in_channels=in_channels,\n                   n_mels=n_mels);\n        model.load_state_dict(torch.load(MODELS[n_model-1], weights_only=True, map_location=torch.device('cpu')))\n        model.to(rank)\n        \n        inf_ddp_model = DDP(model, device_ids=[rank])\n        \n        return inf_ddp_model\n\n    \n    \n    train_loader, val_loader = prepare(rank, world_size, test_size=cfg.split_fraction, batch_size=cfg.BATCH_SIZE)\n    \n            \n    os.makedirs(save_model_dir + f'weights_1/', exist_ok=True)\n    \n    setup(rank, world_size)\n    \n\n    # create model and move it to GPU with id rank\n    _model = TimmSED(base_model_name=cfg.base_model_name, \n                        pretrained=True, \n                        num_classes=len(class_labels),\n                        in_channels=in_channels,\n                        n_mels=n_mels).to(rank)\n    \n    ddp_model = DDP(_model, device_ids=[rank], find_unused_parameters=True)\n\n    # loss fn, optimizer, lr scheduler\n    criterion = nn.CrossEntropyLoss()\n    optimizer = get_optimizer(ddp_model, lr=cfg.LEARNING_RATE)\n    scheduler = get_scheduler(optimizer)\n    \n    for epoch in range(cfg.N_EPOCHS):\n        pred_train, label_train, train_loss = train_fn(train_loader, ddp_model, criterion, optimizer, rank, epoch)\n\n        pred_val, label_val, val_loss = valid_fn(val_loader, ddp_model, criterion, rank, epoch)\n\n        if rank == 0:\n            auc_train_val = cal_score(label_train, pred_train)\n            auc_val = cal_score(label_val, pred_val)\n            print(f\"Epoch {epoch+1}/{cfg.N_EPOCHS}, Loss: {train_loss/cfg.BATCH_SIZE:.4f}, Loss_val: {val_loss/cfg.BATCH_SIZE:.4f}\")\n            print(f\"Auc: {auc_train_val:.2f}% Auc_val: {auc_val:.2f}%\")\n    \n            torch.save(ddp_model.module.state_dict(), f\"{save_model_dir}/weights_1/timmsed_epoch{epoch+1}_{auc_train_val:.2f}_{auc_val:.2f}.pth\")\n\n\n        #### validation on pretrained model\n        # pred_inf, label_inf, inf_loss = valid_fn(val_loaders[j], load_inf_model(n_model=j+1), criterion, rank, epoch)\n        # if rank == 0:\n        #     auc_inf = cal_score(label_inf, pred_inf)\n        #     print(f\"Loss_inf: {inf_loss/cfg.BATCH_SIZE:.4f}\")\n        #     print(f\"Auc_inference: {auc_inf:.2f}%\")\n\n        # apply LR scheduler after each epoch\n        if isinstance(scheduler, ReduceLROnPlateau):\n            scheduler.step(val_loss)\n    \n        elif isinstance(scheduler, CosineAnnealingLR):\n            scheduler.step()\n\n\n    # del _model, \n    cleanup()\n\n\n\ndef run_demo(demo_fn, world_size):\n    mp.spawn(demo_fn,\n             args=(world_size,),\n             nprocs=world_size,\n             join=True)\n    \n\n\n\n\n\nif __name__ == \"__main__\":\n    world_size = torch.cuda.device_count()\n    if world_size < 2:\n        raise RuntimeError(\"Need at least 2 GPUs for this demo\")\n    run_demo(demo_basic, world_size)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T12:49:11.957866Z","iopub.execute_input":"2025-06-05T12:49:11.958179Z","iopub.status.idle":"2025-06-05T12:49:11.969550Z","shell.execute_reply.started":"2025-06-05T12:49:11.958158Z","shell.execute_reply":"2025-06-05T12:49:11.968792Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"save_model_dir = f'/kaggle/working/weights/'\n\nif os.path.isdir(save_model_dir):\n        shutil.rmtree(save_model_dir)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T11:10:22.779339Z","iopub.execute_input":"2025-06-05T11:10:22.779849Z","iopub.status.idle":"2025-06-05T11:10:22.783390Z","shell.execute_reply.started":"2025-06-05T11:10:22.779825Z","shell.execute_reply":"2025-06-05T11:10:22.782729Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!python3 demo.py","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T12:49:13.554223Z","iopub.execute_input":"2025-06-05T12:49:13.554819Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_ = TimmSED(base_model_name=cfg.base_model_name, \n                        pretrained=True, \n                        num_classes=len(class_labels),\n                        in_channels=3,\n                        n_mels=128)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T05:58:15.290990Z","iopub.execute_input":"2025-06-05T05:58:15.291268Z","iopub.status.idle":"2025-06-05T05:58:15.954510Z","shell.execute_reply.started":"2025-06-05T05:58:15.291248Z","shell.execute_reply":"2025-06-05T05:58:15.953812Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold\n\n\nkfold = KFold(n_splits=3, shuffle=True, random_state=42)\n\n    \nfor fold, (train_idx, val_idx) in enumerate(datasplits):\n    print(type(train_idx), val_idx, sep='\\n')\n    print()\n    # Create subsets for training and validation\n    train_df = train_meta.iloc[train_idx].reset_index(drop=True)\n    val_df   = train_meta.iloc[val_idx].reset_index(drop=True)\n    # train_subset = Subset(df, train_idx)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T09:54:46.169567Z","iopub.execute_input":"2025-06-05T09:54:46.170112Z","iopub.status.idle":"2025-06-05T09:54:46.213316Z","shell.execute_reply.started":"2025-06-05T09:54:46.170088Z","shell.execute_reply":"2025-06-05T09:54:46.212610Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(train_df), len(val_df), len(train_meta)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T09:49:49.648628Z","iopub.execute_input":"2025-06-05T09:49:49.648900Z","iopub.status.idle":"2025-06-05T09:49:49.654674Z","shell.execute_reply.started":"2025-06-05T09:49:49.648879Z","shell.execute_reply":"2025-06-05T09:49:49.653902Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## creating and saving 3-fold df","metadata":{}},{"cell_type":"code","source":"def n_fold_datasplit(df, n_splits):\n    kfold = KFold(n_splits=n_splits, shuffle=True, random_state=42)\n    train_dfs, val_dfs = [], []\n    \n    for fold, (train_idx, val_idx) in enumerate(kfold.split(df)):\n        # Create subsets for training and validation\n        train_df = df.iloc[train_idx].reset_index(drop=True)\n        val_df   = df.iloc[val_idx].reset_index(drop=True)\n\n        train_dfs.append(train_df)\n        val_dfs.append(val_df)\n        # train_subset = Subset(df, train_idx)\n        # val_subset = Subset(df, val_idx)\n    \n        # print(f\"Train subset size: {len(train_subset)}, Validation subset size: {len(val_subset)}\")\n    return train_dfs, val_dfs\n    \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T12:17:58.585967Z","iopub.execute_input":"2025-06-05T12:17:58.586241Z","iopub.status.idle":"2025-06-05T12:17:58.591297Z","shell.execute_reply.started":"2025-06-05T12:17:58.586220Z","shell.execute_reply":"2025-06-05T12:17:58.590220Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"os.makedirs('/kaggle/working/data_splits/', exist_ok=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T12:22:01.454919Z","iopub.execute_input":"2025-06-05T12:22:01.455505Z","iopub.status.idle":"2025-06-05T12:22:01.459133Z","shell.execute_reply.started":"2025-06-05T12:22:01.455481Z","shell.execute_reply":"2025-06-05T12:22:01.458392Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dfs, val_dfs = n_fold_datasplit(train_meta, 3)\n\nfor i in range(len(train_dfs)):\n    os.makedirs(f'/kaggle/working/data_splits/split{i+1}/', exist_ok=True)\n    train_df.to_csv(f'/kaggle/working/data_splits/split{i+1}/train_meta.csv')\n    val_df.to_csv(f'/kaggle/working/data_splits/split{i+1}/val_meta.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T12:23:56.502183Z","iopub.execute_input":"2025-06-05T12:23:56.502515Z","iopub.status.idle":"2025-06-05T12:23:58.306032Z","shell.execute_reply.started":"2025-06-05T12:23:56.502494Z","shell.execute_reply":"2025-06-05T12:23:58.305457Z"}},"outputs":[],"execution_count":null}]}