{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"},{"sourceId":11075844,"sourceType":"datasetVersion","datasetId":6902807},{"sourceId":11749667,"sourceType":"datasetVersion","datasetId":7368311,"isSourceIdPinned":true}],"dockerImageVersionId":31011,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport random\nfrom tqdm import tqdm\nfrom logging import getLogger, INFO, StreamHandler, FileHandler, Formatter\nimport gc\nimport pickle as pkl\n\nimport librosa\n\nfrom torch.utils.data import DataLoader, Dataset\nimport torchaudio\nimport torchaudio.transforms as T\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.amp import autocast, GradScaler\nfrom torch.optim import lr_scheduler, Adam, AdamW\n\nimport timm\n\nfrom glob import glob\nfrom sklearn.model_selection import KFold, GroupKFold, StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nfrom scipy.special import logit, expit","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:42.819086Z","iopub.execute_input":"2025-05-09T17:45:42.819456Z","iopub.status.idle":"2025-05-09T17:45:47.850099Z","shell.execute_reply.started":"2025-05-09T17:45:42.819428Z","shell.execute_reply":"2025-05-09T17:45:47.849452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nfrom pathlib import Path\nfrom types import SimpleNamespace\n\ncfg = SimpleNamespace(**{})\ncfg.num_folds = 5\ncfg.fold = -1\ncfg.gpu = \"0\"\nos.environ['CUDA_VISIBLE_DEVICES'] = cfg.gpu\n\ncfg.fname = 'efficientvit_b0'\ncfg.seed = 2025\n\ncfg.input_path = Path('../input')\ncfg.comp_data_path = cfg.input_path / 'birdclef-2025'\ncfg.save_path = Path('../working')\ncfg.soundscape_path = cfg.comp_data_path / 'train_soundscapes'\ncfg.audio_path = Path(\"/kaggle/input/birdclef-2025/train_audio\")\n\ncfg.logger_file = True\n\n\n# # V7\n# # image size\n# cfg.image_height = 224\n# cfg.image_width = 224\n# cfg.target_shape = (224, 224)\n\n# # audio\n# cfg.duration = 10\n# cfg.sr = 32000\n# cfg.fmin = 50\n# cfg.fmax = 14000\n# cfg.n_fft = 1536\n# cfg.n_mels = cfg.image_height\n# cfg.win_length = 1024\n# cfg.hop_length = int((cfg.duration * cfg.sr - cfg.win_length + cfg.n_fft) / (cfg.image_width)) + 1 \n\n\n# V10\n# image size\ncfg.image_height = 256\ncfg.image_width = 256\ncfg.target_shape = (256, 256)\n\n# audio\ncfg.duration = 10\ncfg.sr = 32000\ncfg.fmin = 20\ncfg.fmax = 16000\ncfg.n_fft = 2048\ncfg.n_mels = 512\ncfg.win_length = 2048\ncfg.hop_length = 128\n\n\n# training HP\ncfg.num_epochs = 15\ncfg.train_batch_size = 128\ncfg.valid_batch_size = 128\ncfg.workers = 2\ncfg.grad_value = 2.0\ncfg.grad_norm = 0.0\ncfg.grad_norm_type = 2\ncfg.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ncfg.accumulate = 1\n\n# optimizer\ncfg.lr = 7e-5\ncfg.decay = 0.01\ncfg.opt_beta1 = 0.9\ncfg.opt_beta2 = 0.999\ncfg.opt_eps = 1e-8\ncfg.optimizer = 'AdamW'\ncfg.no_decay = True\n\n# scheduler\ncfg.pct_start = 0.1\ncfg.max_lr = 3e-3\ncfg.final_div_factor = 100\n\n# augmentations\ncfg.resample_train = 10\ncfg.other_samples = 1\ncfg.max_shift = 1\ncfg.loudness_range = 10.0\n\n# logging\ncfg.local_rank = 0\ncfg.verbose=True\n\n# model\ncfg.backbone = 'efficientvit_b0.r224_in1k'\ncfg.gem_pooling = False\ncfg.bce = False\ncfg.drop_rate = 0.1\n\n# tasks hp\ncfg.train_model = True\ncfg.num_folds = 5\ncfg.pl = None\ncfg.pretrained_path = None ","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:47.857207Z","iopub.execute_input":"2025-05-09T17:45:47.857493Z","iopub.status.idle":"2025-05-09T17:45:47.895492Z","shell.execute_reply.started":"2025-05-09T17:45:47.857467Z","shell.execute_reply":"2025-05-09T17:45:47.894708Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from pprint import pprint\nmodel_names = timm.list_models(pretrained=True)\npprint(model_names[::len(model_names)//10])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:47.899578Z","iopub.execute_input":"2025-05-09T17:45:47.899754Z","iopub.status.idle":"2025-05-09T17:45:47.923405Z","shell.execute_reply.started":"2025-05-09T17:45:47.899739Z","shell.execute_reply":"2025-05-09T17:45:47.922805Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cfg.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ncfg.device","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:47.924090Z","iopub.execute_input":"2025-05-09T17:45:47.924325Z","iopub.status.idle":"2025-05-09T17:45:47.942102Z","shell.execute_reply.started":"2025-05-09T17:45:47.924300Z","shell.execute_reply":"2025-05-09T17:45:47.941480Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if cfg.train_model:\n    checkpoint_path = cfg.save_path / cfg.fname\n    if not cfg.save_path.exists():\n        cfg.save_path.mkdir()\n    if not checkpoint_path.exists():\n        checkpoint_path.mkdir()\n    checkpoint_path = checkpoint_path / \"exp_0\"\n    exp = 0\n    while(checkpoint_path.exists()):\n        exp += 1\n        checkpoint_path = cfg.save_path / f\"exp_{exp}\"\n    checkpoint_path.mkdir()\n    cfg.checkpoint_path = checkpoint_path\n    print(f\"Saving checkpoint path to {checkpoint_path}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:47.942680Z","iopub.execute_input":"2025-05-09T17:45:47.942874Z","iopub.status.idle":"2025-05-09T17:45:47.955226Z","shell.execute_reply.started":"2025-05-09T17:45:47.942859Z","shell.execute_reply":"2025-05-09T17:45:47.954626Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_logger(cfg):\n    logger = getLogger(cfg.fname)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    if cfg.logger_file and cfg.train_model:\n        filename= cfg.checkpoint_path / \"run.log\"\n        handler2 = FileHandler(filename=filename)\n        handler1.setFormatter(Formatter(\"%(message)s\"))\n        logger.addHandler(handler2)\n    return logger\n\ndef seed_torch(seed_value):\n    random.seed(seed_value) # Python\n    np.random.seed(seed_value) # cpu vars\n    torch.manual_seed(seed_value) # cpu  vars    \n    if torch.cuda.is_available(): \n        torch.cuda.manual_seed(seed_value)\n        torch.cuda.manual_seed_all(seed_value) # gpu vars\n    if torch.backends.cudnn.is_available:\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:47.955978Z","iopub.execute_input":"2025-05-09T17:45:47.956219Z","iopub.status.idle":"2025-05-09T17:45:47.968640Z","shell.execute_reply.started":"2025-05-09T17:45:47.956204Z","shell.execute_reply":"2025-05-09T17:45:47.967948Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ntrain = pd.read_csv(Path(\"/kaggle/input/bc25-numpy-data-kuduck/bc25-numpy-data-kuduck\") / 'train_chunks_df.csv')\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:47.969664Z","iopub.execute_input":"2025-05-09T17:45:47.969860Z","iopub.status.idle":"2025-05-09T17:45:48.325795Z","shell.execute_reply.started":"2025-05-09T17:45:47.969845Z","shell.execute_reply":"2025-05-09T17:45:48.325166Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:48.326592Z","iopub.execute_input":"2025-05-09T17:45:48.327129Z","iopub.status.idle":"2025-05-09T17:45:48.331669Z","shell.execute_reply.started":"2025-05-09T17:45:48.327101Z","shell.execute_reply":"2025-05-09T17:45:48.330976Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train = pd.read_csv(cfg.comp_data_path / 'train.csv')\n# train\n\n# train[\"species\"] = [filename.split(\"/\")[0] for filename in train[\"filename\"]]\n# train[\"record\"] = [filename.split(\"/\")[1] for filename in train[\"filename\"]]\n# train[\"secondary_labels\"] = [eval(sls) for sls in train[\"secondary_labels\"]]\n\n# df = train.groupby(\"record\").agg({\n#     'species': ['first', 'last'],\n#     'secondary_labels': ['first', 'last'],\n# })\n\n# df.columns = [\"first_species\", \"last_species\", \"first_secondary\", \"last_secondary\"]\n# df.reset_index(inplace=True)\n\n# train = train.merge(df[['record', 'first_species', 'last_species',]],\n#                    how='left',\n#                    on='record')\n# def load_audio(file_name, isfirst, istrain, cfg):\n#     filepath = file_name.split(\"/\")[0]\n#     fname = file_name.split(\"/\")[1].split(\".\")[0]\n#     filepath = cfg.input_path / \"birdclef-data\" / \"birdclef_data\" / \"birdclef_data\" / filepath\n\n#     if istrain:\n#         max_duration = int((cfg.duration + cfg.max_shift) * cfg.sr)\n#     else:\n#         max_duration = cfg.duration * cfg.sr\n\n#     if isfirst:\n#         filepath = filepath / f\"first10_{fname}.npy\"\n#         audio = np.load(filepath)\n#         audio = audio[:max_duration]\n#     else:\n#         filepath = filepath / f\"last10_{fname}.npy\"\n#         audio = np.load(filepath)\n#         audio = audio[-max_duration:]\n\n#     return audio\n# new_train = []\n\n# kf = KFold(n_splits=cfg.num_folds, shuffle=True, random_state=0)\n# for species, df in train.groupby('species'):\n#     df = df.reset_index(drop=True)\n#     df['fold'] = -1\n#     if len(df) < 5:\n#         df['fold'] = 0\n#     else:\n#         for fold, (train_idx, valid_idx) in enumerate(kf.split(df, df.primary_label)):\n#             df.loc[valid_idx, \"fold\"] = fold\n#     new_train.append(df)\n# # new_train = pd.concat(new_train).reset_index(drop=True)    \n# # new_train.fold.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:48.332531Z","iopub.execute_input":"2025-05-09T17:45:48.332814Z","iopub.status.idle":"2025-05-09T17:45:48.348354Z","shell.execute_reply.started":"2025-05-09T17:45:48.332790Z","shell.execute_reply":"2025-05-09T17:45:48.347538Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_audio(file_name, isfirst, istrain, cfg):\n    filepath = file_name.split(\"/\")[0]\n    fname = file_name.split(\"/\")[1].split(\".\")[0]\n    filepath = cfg.input_path / \"birdclef-data\" / \"birdclef_data\" / \"birdclef_data\" / filepath\n\n    if istrain:\n        max_duration = int((cfg.duration + cfg.max_shift) * cfg.sr)\n    else:\n        max_duration = cfg.duration * cfg.sr\n\n    if isfirst:\n        filepath = filepath / f\"first10_{fname}.npy\"\n        audio = np.load(filepath)\n        audio = audio[:max_duration]\n    else:\n        filepath = filepath / f\"last10_{fname}.npy\"\n        audio = np.load(filepath)\n        audio = audio[-max_duration:]\n\n    return audio","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:48.349093Z","iopub.execute_input":"2025-05-09T17:45:48.349308Z","iopub.status.idle":"2025-05-09T17:45:48.366605Z","shell.execute_reply.started":"2025-05-09T17:45:48.349293Z","shell.execute_reply":"2025-05-09T17:45:48.366030Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.copy()\nskf = StratifiedKFold(cfg.num_folds, shuffle=True, random_state=2911)\nfor fold, (train_idx, valid_idx) in enumerate(skf.split(train, train.label)):\n    train.loc[valid_idx, 'fold'] = fold\ntrain.fold = train.fold.astype(int)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:48.369901Z","iopub.execute_input":"2025-05-09T17:45:48.370167Z","iopub.status.idle":"2025-05-09T17:45:48.419108Z","shell.execute_reply.started":"2025-05-09T17:45:48.370151Z","shell.execute_reply":"2025-05-09T17:45:48.418451Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.fold.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:48.419815Z","iopub.execute_input":"2025-05-09T17:45:48.420093Z","iopub.status.idle":"2025-05-09T17:45:48.426144Z","shell.execute_reply.started":"2025-05-09T17:45:48.420071Z","shell.execute_reply":"2025-05-09T17:45:48.425400Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"set(range(206)) - set(train.label.unique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:48.426970Z","iopub.execute_input":"2025-05-09T17:45:48.427435Z","iopub.status.idle":"2025-05-09T17:45:48.441735Z","shell.execute_reply.started":"2025-05-09T17:45:48.427416Z","shell.execute_reply":"2025-05-09T17:45:48.441165Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def metric(preds, targets, cfg):\n    score = {}\n    for j, label in enumerate(cfg.labels):\n        y_true = targets[:, j]\n        y_pred = preds[:, j]\n        if len(np.unique(y_true)) < 2:\n            score[label] = 0\n            continue\n        score[label] = roc_auc_score(y_true, y_pred)\n    score_avg = np.mean([v for v in score.values()])\n    return score_avg, score\n\ndef metric_db(train, oofs, cfg):\n    score = {}\n    for j,label in enumerate(cfg.labels):\n        score[label] = roc_auc_score(train.label == label, oofs[label])\n    score_avg = np.mean([v for k,v in score.items()])\n    return score_avg, score\n    \ndef my_softmax(preds):\n    preds = preds - preds.max(1, keepdims=True)\n    preds = np.exp(preds.clip(-20, 0))\n    preds = preds / preds.sum(1, keepdims=True)\n    return preds\n\ndef bce_with_mask(preds, targets, mask):\n    loss = nn.BCEWithLogitsLoss(reduction='none')(preds, targets)\n    loss = loss * mask\n    loss = loss.mean()\n    return loss","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:51:57.152305Z","iopub.execute_input":"2025-05-09T17:51:57.152582Z","iopub.status.idle":"2025-05-09T17:51:57.159781Z","shell.execute_reply.started":"2025-05-09T17:51:57.152559Z","shell.execute_reply":"2025-05-09T17:51:57.159010Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_metadata = pd.read_csv(\"/kaggle/input/birdclef-2025/train.csv\")\ntrain_metadata.head()\ncompetition_classes = sorted(train_metadata.primary_label.unique())\n\ncfg.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nif cfg.bce:\n    cfg.loss = bce_with_mask\nelse:\n    cfg.loss = nn.CrossEntropyLoss()\n    \ncfg.labels = np.array(competition_classes)\ncfg.num_labels = len(cfg.labels)\ncfg.targets = {v: i for i, v in enumerate(cfg.labels)}\n\ncfg.logger = get_logger(cfg)\nseed_torch(cfg.seed)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:48.455812Z","iopub.execute_input":"2025-05-09T17:45:48.456095Z","iopub.status.idle":"2025-05-09T17:45:48.574599Z","shell.execute_reply.started":"2025-05-09T17:45:48.456072Z","shell.execute_reply":"2025-05-09T17:45:48.573974Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.label.values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:48.575272Z","iopub.execute_input":"2025-05-09T17:45:48.575467Z","iopub.status.idle":"2025-05-09T17:45:48.580599Z","shell.execute_reply.started":"2025-05-09T17:45:48.575452Z","shell.execute_reply":"2025-05-09T17:45:48.580016Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BirdDataset(Dataset):\n    def __init__(self, train, istrain, cfg):\n        self.cfg = cfg\n        self.istrain = istrain\n        self.filename = train.filename.values\n        self.label = train.label.values\n        self.labels_proba = []\n        for i in range(len(train)):\n            self.labels_proba.append(\n                np.array(list(map(float, train.labels_proba.values[i].replace('[', '').replace(']', '').split()))))\n        self.labels_proba = np.array(self.labels_proba)\n        \n    def __len__(self):\n        return len(self.filename)\n    \n    def get_mel(self, idx):\n        filename = self.filename[idx]\n\n        return np.load(f'/kaggle/input/bc25-numpy-data-kuduck/{filename}')\n    \n    def __getitem__(self, idx):\n        image = self.get_mel(idx)\n\n        label = self.label[idx]\n        labels_proba = self.labels_proba[idx]\n        \n        out = {\n            'image': image,\n            'label': label,\n            'labels_proba': labels_proba\n        }\n        return out\n    \ndef batch_to_device(batch, device):\n        return {k:batch[k].to(device, non_blocking=True) for k in batch.keys() if k not in []}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:48.581376Z","iopub.execute_input":"2025-05-09T17:45:48.581586Z","iopub.status.idle":"2025-05-09T17:45:48.595548Z","shell.execute_reply.started":"2025-05-09T17:45:48.581571Z","shell.execute_reply":"2025-05-09T17:45:48.594949Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dataset = BirdDataset(train, True, cfg)\nelt = dataset[0]\nfor k,v in elt.items():\n    print(k, v.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:48.596186Z","iopub.execute_input":"2025-05-09T17:45:48.596375Z","iopub.status.idle":"2025-05-09T17:45:51.427260Z","shell.execute_reply.started":"2025-05-09T17:45:48.596361Z","shell.execute_reply":"2025-05-09T17:45:51.426544Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_pl_iter(pl_preds, fold, istrain, cfg):\n    pl_dataset = BirdPLDataset(pl_preds, fold, cfg)\n    pl_data_loader = DataLoader(\n        pl_dataset,\n        batch_size=cfg.pl_batch_size,\n        num_workers=0,\n        shuffle=istrain,\n        pin_memory=False,\n        #collate_fn=collate_pad,\n        drop_last = False,\n    )\n    \n    return iter(pl_data_loader), pl_data_loader\n\ndef get_pl_batch(pl_dataloader):\n    try:\n        batch = next(pl_dataloader[0])\n    except StopIteration:\n        pl_data_loader = pl_dataloader[1]\n        pl_data_loader = iter(pl_data_loader), pl_data_loader\n        batch = next(pl_dataloader[0])\n    new_batch = {k : v.view(v.shape[0] * v.shape[1], v.shape[2]) \n                 for k,v in batch.items()}\n    return new_batch","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:51.427964Z","iopub.execute_input":"2025-05-09T17:45:51.428200Z","iopub.status.idle":"2025-05-09T17:45:51.433517Z","shell.execute_reply.started":"2025-05-09T17:45:51.428182Z","shell.execute_reply":"2025-05-09T17:45:51.432859Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_data_loader(dataset, istrain, cfg):\n    if istrain:\n        batch_size = cfg.train_batch_size\n    else:\n        batch_size = cfg.valid_batch_size\n    data_loader = DataLoader(\n        dataset,\n        batch_size=batch_size,\n        num_workers=cfg.workers,\n        shuffle=istrain,\n        pin_memory=False,\n        #collate_fn=collate_pad,\n        drop_last = istrain,\n    )\n    return data_loader ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:51.434221Z","iopub.execute_input":"2025-05-09T17:45:51.434437Z","iopub.status.idle":"2025-05-09T17:45:51.451693Z","shell.execute_reply.started":"2025-05-09T17:45:51.434413Z","shell.execute_reply":"2025-05-09T17:45:51.451029Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def gem(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6, p_trainable=True):\n        super(GeM,self).__init__()\n        if p_trainable:\n            self.p = nn.Parameter(torch.ones(1)*p)\n        else:\n            self.p = p\n        self.eps = eps\n\n    def forward(self, x):\n        return gem(x, p=self.p, eps=self.eps)       \n    def __repr__(self):\n        return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + ', ' + 'eps=' + str(self.eps) + ')'\n\nclass BirdModel(nn.Module):\n    def __init__(self, cfg, pretrained: bool = True):\n        super(BirdModel, self).__init__()\n        self.cfg = cfg\n        self.mel = T.MelSpectrogram(\n            sample_rate=cfg.sr, n_fft=cfg.n_fft, win_length=cfg.win_length, \n            hop_length= cfg.hop_length, f_min=cfg.fmin, f_max=cfg.fmax, \n            n_mels=cfg.n_mels, mel_scale='htk', power=2.0)\n        self.A2DB = T.AmplitudeToDB(stype=\"power\")\n        self.backbone = timm.create_model(\n            cfg.backbone,\n            pretrained=pretrained,\n            drop_rate = cfg.drop_rate,\n            num_classes=cfg.num_labels\n        )\n        #if cfg.gem_pooling == \"gem\":\n        #    self.backbone.head.global_pool = GeM(p_trainable=args.p_trainable)\n         \n    # def forward(self, input_dict):\n    #     x = input_dict['audio']\n    #     with autocast(enabled=False, device_type=\"cuda\"), torch.no_grad():\n    #         x = x / torch.std(x, 1, keepdim=True)\n    #         x = x.float()\n    #         x = self.mel(x)\n    #         x = self.A2DB(x)\n    #         x = (x - 40) / 80\n    #     with torch.no_grad():\n    #         x = x.unsqueeze(1)\n    #         pos = torch.linspace(0., 1., x.size(2)).to(x.device)\n    #         pos = pos.unsqueeze(0).unsqueeze(0).unsqueeze(-1)\n    #         pos = pos.expand(x.size(0), 1, x.size(2), x.size(3))\n    #         x = x.expand(-1, 2, -1, -1)\n    #         x = torch.cat([x, pos], 1)\n\n    #     x = self.backbone(x)\n    #     return x\n\n\n    def forward(self, input_dict):\n        x = input_dict['image']\n        with torch.no_grad():\n            x = x.unsqueeze(1)\n            pos = torch.linspace(0., 1., x.size(2)).to(x.device)\n            pos = pos.unsqueeze(0).unsqueeze(0).unsqueeze(-1)\n            pos = pos.expand(x.size(0), 1, x.size(2), x.size(3))\n            x = x.expand(-1, 2, -1, -1)\n            x = torch.cat([x, pos], 1)\n\n        x = self.backbone(x)\n        return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:51.452352Z","iopub.execute_input":"2025-05-09T17:45:51.452613Z","iopub.status.idle":"2025-05-09T17:45:51.467409Z","shell.execute_reply.started":"2025-05-09T17:45:51.452594Z","shell.execute_reply":"2025-05-09T17:45:51.466705Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_loader = get_data_loader(dataset, False, cfg)\nmodel = BirdModel(cfg)\nfor batch in data_loader:\n    break\nout = model(batch)\nout.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:51.468182Z","iopub.execute_input":"2025-05-09T17:45:51.468421Z","iopub.status.idle":"2025-05-09T17:45:58.842442Z","shell.execute_reply.started":"2025-05-09T17:45:51.468399Z","shell.execute_reply":"2025-05-09T17:45:58.841644Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_epoch(loader, model, optimizer, scheduler, scaler, device, cfg):\n    model.train()\n    model.zero_grad()\n    if cfg.verbose:\n        bar = tqdm(range(len(loader)))\n    else:\n        bar = range(len(loader))\n    load_iter = iter(loader)\n    loss_l = []\n    grad_norm_l = []\n    \n    accumulate = cfg.accumulate\n    \n    for i, batch in zip(bar, load_iter):\n        input_dict = batch_to_device(batch, device)\n        with autocast(enabled=True, device_type=\"cuda\"):\n            # targets = input_dict['label']\n            targets = input_dict['labels_proba']\n            bs = targets.shape[0]\n            preds = model(input_dict)\n            if False: # cfg.bce:\n                secondary_mask = input_dict['secondary_mask']\n                loss = cfg.loss(preds, targets, secondary_mask).mean()\n            else:\n                loss = cfg.loss(preds, targets).mean()\n        loss_l.append(loss.detach().cpu().item())\n        scaler.scale(loss / cfg.accumulate).backward() \n        accumulate -= 1\n        if accumulate == 0:\n            if cfg.grad_value:\n                scaler.unscale_(optimizer)\n                nn.utils.clip_grad_value_(model.parameters(), cfg.grad_value)\n            if cfg.grad_norm:\n                scaler.unscale_(optimizer)\n                total_norm = nn.utils.clip_grad_norm_(model.parameters(), cfg.grad_norm).item()\n                if np.isnan(total_norm):\n                    total_norm = cfg.grad_norm\n                else:\n                    total_norm = np.clip(total_norm, 0, cfg.grad_norm)\n                grad_norm_l.append(total_norm)\n            scaler.step(optimizer)     \n            scaler.update()\n            optimizer.zero_grad()\n            accumulate = cfg.accumulate\n            scheduler.step()  \n        del preds, targets, loss, input_dict\n        if cfg.verbose:\n            if cfg.grad_norm:\n                bar.set_description('loss: %.4f grad norm %.1f' % (np.mean(loss_l), np.mean(grad_norm_l),))\n            else:\n                bar.set_description('loss: %.4f ' % np.mean(loss_l))\n    optimizer.zero_grad()\n    del loss_l, bar\n    gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:58.843458Z","iopub.execute_input":"2025-05-09T17:45:58.843778Z","iopub.status.idle":"2025-05-09T17:45:58.852780Z","shell.execute_reply.started":"2025-05-09T17:45:58.843750Z","shell.execute_reply":"2025-05-09T17:45:58.852046Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def valid_epoch(loader, model, device, cfg):\n    model.eval()\n    model.zero_grad()\n    if cfg.verbose:\n        bar = tqdm(range(len(loader)))\n    else:\n        bar = range(len(loader))\n    load_iter = iter(loader)\n    preds_l = []\n    targets_l = []\n    with torch.no_grad():\n        for i, batch in zip(bar, load_iter):      \n            input_dict = batch_to_device(batch, device)\n            with autocast(enabled=False, device_type=\"cuda\"):\n                preds = model(input_dict)                \n            preds_l.append(preds.detach().cpu())\n            del preds, input_dict\n            targets = batch['label']\n            targets_l.append(targets)\n        preds = torch.cat(preds_l)\n        targets = torch.cat(targets_l)\n        return preds.numpy(), targets.numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:58.853622Z","iopub.execute_input":"2025-05-09T17:45:58.853824Z","iopub.status.idle":"2025-05-09T17:45:58.906949Z","shell.execute_reply.started":"2025-05-09T17:45:58.853809Z","shell.execute_reply":"2025-05-09T17:45:58.906280Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_optimizer(model, cfg):\n    no_decay = [\"bias\", \"norm\"]\n    if cfg.no_decay:\n        optimizer_parameters = [\n            {'params': [p for n, p in model.named_parameters() \n                        if not any(nd in n for nd in no_decay)],\n             'weight_decay': cfg.decay},\n            {'params': [p for n, p in model.named_parameters() \n                        if any(nd in n for nd in no_decay)],\n             'weight_decay': 0.0},\n        ] \n    else:\n        optimizer_parameters = model.parameters()\n    optimizer = torch.optim.AdamW(optimizer_parameters, lr=cfg.lr)\n    return optimizer\n\ndef get_scheduler(optimizer, train_data_loader, cfg):\n    scheduler = lr_scheduler.OneCycleLR(\n        optimizer,\n        max_lr=cfg.max_lr,\n        epochs=cfg.num_epochs,\n        steps_per_epoch=len(train_data_loader),\n        pct_start=cfg.pct_start,\n        anneal_strategy=\"cos\",\n        final_div_factor=cfg.final_div_factor,\n    )\n    return scheduler","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:58.907681Z","iopub.execute_input":"2025-05-09T17:45:58.907879Z","iopub.status.idle":"2025-05-09T17:45:58.916625Z","shell.execute_reply.started":"2025-05-09T17:45:58.907865Z","shell.execute_reply":"2025-05-09T17:45:58.916005Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def save_checkpoint(model, fold, cfg):\n    checkpoint = {\n        'model' : model.state_dict(),\n        'fold' : fold,\n        'seed' : seed,\n        }\n    checkpoint_path = cfg.checkpoint_path\n    save_path = checkpoint_path / ('%s_%d.pth' % (cfg.fname, fold))\n    if cfg.local_rank == 0:\n        cfg.logger.info('saving %s ...' % save_path)\n    torch.save(checkpoint, save_path)\n    if cfg.local_rank == 0:\n        cfg.logger.info('done')\n\ndef load_checkpoint(fold, cfg):\n    if cfg.pretrained_path:\n        checkpoint_path = cfg.pretrained_path\n    else:\n        checkpoint_path = cfg.checkpoint_path\n    save_path = checkpoint_path / ('%s_%d.pth' % (cfg.fname, fold))\n    cfg.logger.info('loading %s ...' % save_path)\n    checkpoint = torch.load(save_path, map_location='cpu')\n    model = BirdModel(cfg, pretrained=False).to(cfg.device)\n    model.load_state_dict(checkpoint['model'], strict=True)\n    model.eval()\n    cfg.logger.info('done')\n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:58.917286Z","iopub.execute_input":"2025-05-09T17:45:58.917496Z","iopub.status.idle":"2025-05-09T17:45:58.930001Z","shell.execute_reply.started":"2025-05-09T17:45:58.917481Z","shell.execute_reply":"2025-05-09T17:45:58.929268Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def resample(train_fold, cfg):\n#     new_train = []\n#     for species, df in train.groupby('species'):\n#         new_train.append(df)\n#         if len(df) < cfg.resample_train:\n#             df = df.sample(n=(cfg.resample_train - len(df)), replace=True, random_state=cfg.seed)\n#             new_train.append(df)\n#     new_train = pd.concat(new_train).reset_index(drop=True)  \n#     return new_train  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:58.930748Z","iopub.execute_input":"2025-05-09T17:45:58.930927Z","iopub.status.idle":"2025-05-09T17:45:58.946389Z","shell.execute_reply.started":"2025-05-09T17:45:58.930913Z","shell.execute_reply":"2025-05-09T17:45:58.945694Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scores = []\noofs = pd.DataFrame(columns=cfg.labels, index=train.index, data = 0.0)\noofs['filename'] = train.filename","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:58.947248Z","iopub.execute_input":"2025-05-09T17:45:58.947494Z","iopub.status.idle":"2025-05-09T17:45:58.991475Z","shell.execute_reply.started":"2025-05-09T17:45:58.947472Z","shell.execute_reply":"2025-05-09T17:45:58.990904Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for fold in range(cfg.num_folds):\n    if cfg.fold >= 0 and cfg.fold != fold:\n        continue\n    seed = cfg.seed + fold\n    seed_torch(seed)\n    if cfg.pl is not None:\n        pl_train = get_pl_train(cfg)\n    else:\n        pl_train = None\n        train_fold = train[train.fold != fold]\n    valid_dataset = BirdDataset(train[train.fold == fold], False, cfg)\n    valid_dataloader = get_data_loader(valid_dataset, istrain=False, cfg=cfg)\n    device = cfg.device\n    \n    if cfg.pretrained_path:\n        model = load_checkpoint(fold, seed, cfg)\n    else:\n        model = BirdModel(cfg, pretrained=True).to(device)\n    optimizer = get_optimizer(model, cfg)\n    scheduler = None\n    scaler = GradScaler()\n    result = None\n    for epoch in range(cfg.num_epochs):\n        if cfg.pl is not None:\n            train_fold = sample_pl(train, fold, pl_train)\n        # if cfg.resample_train:\n        #     train_fold = resample(train_fold, cfg)\n        train_dataset = BirdDataset(train_fold, True, cfg)\n        train_dataloader = get_data_loader(train_dataset, istrain=True, cfg=cfg)\n\n        if scheduler is None:\n            scheduler = get_scheduler(optimizer, train_dataloader, cfg)\n        train_epoch(train_dataloader, model, optimizer, scheduler, scaler, device, cfg)\n        if valid_dataset is not None:\n            preds, targets = valid_epoch(valid_dataloader, model, device, cfg)\n            if cfg.bce:\n                preds = expit(preds) # model uses logits\n            else:\n                preds = my_softmax(preds) # model uses logits\n            result, _ = metric(preds, np.eye(206)[targets], cfg)\n            msg = f\"seed {cfg.seed} fold {fold} epoch {epoch} metric {result:.4f}\"\n            cfg.logger.info(msg)\n        else:\n            msg = f\"seed {cfg.seed} fold {fold} epoch {epoch}\"\n            cfg.logger.info(msg)\n    if cfg.local_rank == 0:\n        save_checkpoint(model, fold, cfg)\n    del model, optimizer, scheduler, scaler, train_dataloader, \n    if valid_dataset is not None:\n        del valid_dataloader\n    gc.collect()\n    torch.cuda.empty_cache()\n    scores.append(result)\n    \n    for j,c in enumerate(cfg.labels):\n        oofs.loc[train.fold == fold, c] = preds[:, j]\n        ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:45:58.992169Z","iopub.execute_input":"2025-05-09T17:45:58.992367Z","iopub.status.idle":"2025-05-09T17:50:41.021732Z","shell.execute_reply.started":"2025-05-09T17:45:58.992351Z","shell.execute_reply":"2025-05-09T17:50:41.020235Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\noofs.to_csv(cfg.checkpoint_path / 'oofs.csv', index=False)\n\nnp.mean(scores)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:52:36.975578Z","iopub.execute_input":"2025-05-09T17:52:36.976170Z","iopub.status.idle":"2025-05-09T17:52:45.963565Z","shell.execute_reply.started":"2025-05-09T17:52:36.976146Z","shell.execute_reply":"2025-05-09T17:52:45.962793Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# class SoundScapeDataset():\n#     def __init__(self, soundscape, istrain, cfg):\n#         self.cfg = cfg\n#         self.istrain = istrain\n#         self.soundscape = soundscape\n#         self.duration = int(np.round(soundscape.shape[0] / cfg.sr))\n#         self.starts = np.arange(0, self.duration, cfg.duration)\n#         self.ends = self.starts + cfg.duration\n#         self.starts = cfg.sr * self.starts\n#         self.ends = cfg.sr * self.ends\n        \n#     def __len__(self):\n#         return len(self.starts)\n    \n#     def __getitem__(self, idx):\n#         audio = self.soundscape[self.starts[idx] : self.ends[idx]]\n#         duration = self.cfg.sr * self.cfg.duration\n#         if len(audio) < duration:\n#             pad_length = (duration - len(audio)) // 2\n#             audio = np.pad(audio, \n#                            ((pad_length, duration - len(audio) - pad_length),), \n#                            mode='constant')\n#         out = {\n#             'audio' : torch.from_numpy(audio),\n#         }\n#         return out\n\n# def predict(loader, models, cfg):\n#     for model in models:\n#         model.eval()\n#         model.zero_grad()\n#     if cfg.verbose:\n#         bar = tqdm(range(len(loader)))\n#     else:\n#         bar = range(len(loader))\n#     load_iter = iter(loader)\n#     preds_l = [[] for model in models]\n#     targets_l = []\n#     with torch.no_grad():\n#         for i, batch in zip(bar, load_iter):      \n#             input_dict = batch_to_device(batch, cfg.device)\n#             with autocast(enabled=False, device_type=\"cuda\"):\n#                 for preds, model in zip(preds_l, models):\n#                     preds.append(model(input_dict).detach().cpu()) \n#             del input_dict\n#         preds_l = [torch.cat(preds).squeeze().numpy() for preds in preds_l]\n#         return preds_l\n\n# def load_soundscape(pathname, cfg):\n#     #audio = librosa.load(pathname, sr=32000)[0].astype(np.float32)\n#     filename = pathname.split('/')[-1].split('.')[0]\n#     audio = np.load(cfg.input_path / \"birdclef-data\" / 'unlabeled_soundscapes' / 'unlabeled_soundscapes' / (filename + '.npy'))\n\n#     return audio\n\n# def predict_soundscape(filepath, models, cfg):\n#     waveform = load_soundscape(filepath, cfg)\n#     #return len(waveform) / cfg.sr\n#     #return waveform\n#     #print('duration',  len(waveform) / cfg.sr)\n#     dataset = SoundScapeDataset(waveform, False, cfg)\n#     dataloader = get_data_loader(dataset, False, cfg)\n#     preds_l = predict(dataloader, models, cfg)\n#     if cfg.bce:\n#         preds_l = [expit(preds) for preds in preds_l] # model uses logits\n#     else:\n#         preds_l = [my_softmax(preds) for preds in preds_l] # model uses logits\n#     return preds_l","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:50:41.029831Z","iopub.status.idle":"2025-05-09T17:50:41.030155Z","shell.execute_reply.started":"2025-05-09T17:50:41.030003Z","shell.execute_reply":"2025-05-09T17:50:41.030018Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# soundscapes = sorted(glob(str(cfg.soundscape_path / '*.ogg')))\n# soundscapes[:5]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:50:41.031734Z","iopub.status.idle":"2025-05-09T17:50:41.032043Z","shell.execute_reply.started":"2025-05-09T17:50:41.031886Z","shell.execute_reply":"2025-05-09T17:50:41.031896Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# cfg.verbose = False\n# cfg.workers = 0\n# cfg.valid_batch_size = 128\n\n# models = [load_checkpoint(fold, cfg).to(cfg.device)\n#           for fold in range(cfg.num_folds)\n#          ]\n# preds = {}\n# for soundscape in tqdm(soundscapes):\n#     preds[soundscape]  = predict_soundscape(soundscape, models, cfg)\n# with open(cfg.checkpoint_path / f\"pl_all.pkl\", \"wb\") as file:\n#     pkl.dump(preds, file)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T17:50:41.032735Z","iopub.status.idle":"2025-05-09T17:50:41.033079Z","shell.execute_reply.started":"2025-05-09T17:50:41.032899Z","shell.execute_reply":"2025-05-09T17:50:41.032913Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}