{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"},{"sourceId":11754970,"sourceType":"datasetVersion","datasetId":7287224}],"dockerImageVersionId":31011,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nfrom pathlib import Path\nfrom types import SimpleNamespace\n\ncfg = SimpleNamespace(**{})\ncfg.num_folds = 5\ncfg.fold = -1\ncfg.gpu = \"0\"\nos.environ['CUDA_VISIBLE_DEVICES'] = cfg.gpu\n\ncfg.fname = 'tf_efficientnetv2_s'\ncfg.seed = 2025\n\ncfg.input_path = Path('../input')\ncfg.comp_data_path = cfg.input_path / 'birdclef-2025'\ncfg.save_path = Path('../working')\ncfg.soundscape_path = cfg.comp_data_path / 'train_soundscapes'\ncfg.audio_path = Path(\"/kaggle/input/birdclef-2025/train_audio\")\n\ncfg.logger_file = True\n\n# image size\ncfg.image_height = cfg.image_width = 224\n\n# audio\ncfg.duration = 5\ncfg.sr = 32000\ncfg.fmin = 40\ncfg.fmax = 16000\ncfg.n_fft = 2048\ncfg.n_mels = cfg.image_height\ncfg.win_length = 512\n# cfg.hop_length = int((cfg.duration * cfg.sr - cfg.win_length + cfg.n_fft) / (cfg.image_width)) + 1 \ncfg.hop_length = int((cfg.duration * cfg.sr) / (cfg.image_width)) + 1 \n\n# training HP\ncfg.num_epochs = 15\ncfg.train_batch_size = 128\ncfg.valid_batch_size = 128\ncfg.workers = 2\ncfg.grad_value = 2.0\ncfg.grad_norm = 0.0\ncfg.grad_norm_type = 2\ncfg.device = \"cuda\"\ncfg.accumulate = 1\n\n# optimizer\ncfg.lr = 7e-5\ncfg.decay = 0.01\ncfg.opt_beta1 = 0.9\ncfg.opt_beta2 = 0.999\ncfg.opt_eps = 1e-8\ncfg.optimizer = 'AdamW'\ncfg.no_decay = True\n\n# scheduler\ncfg.pct_start = 0.1\ncfg.max_lr = 3e-3\ncfg.final_div_factor = 100\n\n# augmentations\ncfg.resample_train = 10\ncfg.other_samples = 2\ncfg.max_shift = 1\ncfg.loudness_range = 10.0\n\n# logging\ncfg.local_rank = 0\ncfg.verbose=True\n\n# model\ncfg.backbone = 'tf_efficientnetv2_b0.in1k'\ncfg.gem_pooling = False\ncfg.bce = True\ncfg.drop_rate = 0.1\n\n# tasks hp\ncfg.train_model = True\ncfg.num_folds = 5\ncfg.pl = None\ncfg.pretrained_path = None ","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:38:39.339166Z","iopub.execute_input":"2025-05-19T06:38:39.339458Z","iopub.status.idle":"2025-05-19T06:38:39.353076Z","shell.execute_reply.started":"2025-05-19T06:38:39.339432Z","shell.execute_reply":"2025-05-19T06:38:39.352222Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport random\nfrom tqdm import tqdm\nfrom logging import getLogger, INFO, StreamHandler, FileHandler, Formatter\nimport gc\nimport pickle as pkl\n\nimport librosa\n\nfrom torch.utils.data import DataLoader, Dataset\nimport torchaudio\nimport torchaudio.transforms as T\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nfrom torch.amp import autocast, GradScaler\nfrom torch.optim import lr_scheduler, Adam, AdamW\n\nimport timm\n\nfrom glob import glob\nfrom sklearn.model_selection import KFold, GroupKFold\nfrom sklearn.metrics import roc_auc_score\nfrom scipy.special import logit, expit","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:38:39.353901Z","iopub.execute_input":"2025-05-19T06:38:39.354217Z","iopub.status.idle":"2025-05-19T06:38:51.258672Z","shell.execute_reply.started":"2025-05-19T06:38:39.354193Z","shell.execute_reply":"2025-05-19T06:38:51.257816Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from pprint import pprint\n# model_names = timm.list_models(pretrained=True)\n# pprint(model_names)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:38:51.260545Z","iopub.execute_input":"2025-05-19T06:38:51.260975Z","iopub.status.idle":"2025-05-19T06:38:51.264490Z","shell.execute_reply.started":"2025-05-19T06:38:51.260953Z","shell.execute_reply":"2025-05-19T06:38:51.263683Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if cfg.train_model:\n    checkpoint_path = cfg.save_path / cfg.fname\n    if not cfg.save_path.exists():\n        cfg.save_path.mkdir()\n    if not checkpoint_path.exists():\n        checkpoint_path.mkdir()\n    checkpoint_path = checkpoint_path / \"exp_0\"\n    exp = 0\n    while(checkpoint_path.exists()):\n        exp += 1\n        checkpoint_path = cfg.save_path / f\"exp_{exp}\"\n    checkpoint_path.mkdir()\n    cfg.checkpoint_path = checkpoint_path\n    print(f\"Saving checkpoint path to {checkpoint_path}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:38:51.267464Z","iopub.execute_input":"2025-05-19T06:38:51.267762Z","iopub.status.idle":"2025-05-19T06:38:51.403606Z","shell.execute_reply.started":"2025-05-19T06:38:51.267735Z","shell.execute_reply":"2025-05-19T06:38:51.402816Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_logger(cfg):\n    logger = getLogger(cfg.fname)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    if cfg.logger_file and cfg.train_model:\n        filename= cfg.checkpoint_path / \"run.log\"\n        handler2 = FileHandler(filename=filename)\n        handler1.setFormatter(Formatter(\"%(message)s\"))\n        logger.addHandler(handler2)\n    return logger\n\ndef seed_torch(seed_value):\n    random.seed(seed_value) # Python\n    np.random.seed(seed_value) # cpu vars\n    torch.manual_seed(seed_value) # cpu  vars    \n    if torch.cuda.is_available(): \n        torch.cuda.manual_seed(seed_value)\n        torch.cuda.manual_seed_all(seed_value) # gpu vars\n    if torch.backends.cudnn.is_available:\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:38:51.404429Z","iopub.execute_input":"2025-05-19T06:38:51.404734Z","iopub.status.idle":"2025-05-19T06:38:51.416480Z","shell.execute_reply.started":"2025-05-19T06:38:51.404707Z","shell.execute_reply":"2025-05-19T06:38:51.415845Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(cfg.comp_data_path / 'train.csv')\ntrain","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:38:51.417304Z","iopub.execute_input":"2025-05-19T06:38:51.417557Z","iopub.status.idle":"2025-05-19T06:38:51.628325Z","shell.execute_reply.started":"2025-05-19T06:38:51.417538Z","shell.execute_reply":"2025-05-19T06:38:51.627529Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[\"species\"] = [filename.split(\"/\")[0] for filename in train[\"filename\"]]\ntrain[\"record\"] = [filename.split(\"/\")[1] for filename in train[\"filename\"]]\ntrain[\"secondary_labels\"] = [eval(sls) for sls in train[\"secondary_labels\"]]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:38:51.629269Z","iopub.execute_input":"2025-05-19T06:38:51.629528Z","iopub.status.idle":"2025-05-19T06:38:51.813435Z","shell.execute_reply.started":"2025-05-19T06:38:51.629498Z","shell.execute_reply":"2025-05-19T06:38:51.812902Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = train.groupby(\"record\").agg({\n    'species': ['first', 'last'],\n    'secondary_labels': ['first', 'last'],\n})\n\ndf.columns = [\"first_species\", \"last_species\", \"first_secondary\", \"last_secondary\"]\ndf.reset_index(inplace=True)\ndf","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:38:51.816167Z","iopub.execute_input":"2025-05-19T06:38:51.816388Z","iopub.status.idle":"2025-05-19T06:38:51.877034Z","shell.execute_reply.started":"2025-05-19T06:38:51.816371Z","shell.execute_reply":"2025-05-19T06:38:51.876383Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.merge(df[['record', 'first_species', 'last_species',]],\n                   how='left',\n                   on='record')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:38:51.877730Z","iopub.execute_input":"2025-05-19T06:38:51.878045Z","iopub.status.idle":"2025-05-19T06:38:51.907648Z","shell.execute_reply.started":"2025-05-19T06:38:51.878019Z","shell.execute_reply":"2025-05-19T06:38:51.906900Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_audio(file_name, isfirst, istrain, cfg):\n    filepath = file_name.split(\"/\")[0]\n    fname = file_name.split(\"/\")[1].split(\".\")[0]\n    filepath = cfg.input_path / \"birdclef-data\" / \"birdclef_data\" / \"birdclef_data\" / filepath\n\n    if istrain:\n        max_duration = int((cfg.duration + cfg.max_shift) * cfg.sr)\n    else:\n        max_duration = cfg.duration * cfg.sr\n\n    if isfirst:\n        filepath = filepath / f\"first10_{fname}.npy\"\n        audio = np.load(filepath)\n        audio = audio[:max_duration]\n    else:\n        filepath = filepath / f\"last10_{fname}.npy\"\n        audio = np.load(filepath)\n        audio = audio[-max_duration:]\n\n    return audio","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:38:51.908573Z","iopub.execute_input":"2025-05-19T06:38:51.908929Z","iopub.status.idle":"2025-05-19T06:38:51.913981Z","shell.execute_reply.started":"2025-05-19T06:38:51.908907Z","shell.execute_reply":"2025-05-19T06:38:51.913376Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_train = []\n\nkf = KFold(n_splits=cfg.num_folds, shuffle=True, random_state=0)\nfor species, df in train.groupby('species'):\n    df['fold'] = -1\n    if len(df) < 5:\n        num_requires = 5 - len(df)\n        samples = df.sample(n=num_requires, replace=True)\n        df = pd.concat([df, samples])\n    df = df.reset_index(drop=True)\n    \n    for fold, (train_idx, valid_idx) in enumerate(kf.split(df, df.primary_label)):\n        df.loc[valid_idx, \"fold\"] = fold\n    new_train.append(df)\nnew_train = pd.concat(new_train).reset_index(drop=True)    \nnew_train.fold.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:38:51.914695Z","iopub.execute_input":"2025-05-19T06:38:51.914975Z","iopub.status.idle":"2025-05-19T06:38:52.543258Z","shell.execute_reply.started":"2025-05-19T06:38:51.914949Z","shell.execute_reply":"2025-05-19T06:38:52.542607Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_train.groupby(['species', 'fold']).size().unstack()\ntrain = new_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:38:52.543940Z","iopub.execute_input":"2025-05-19T06:38:52.544130Z","iopub.status.idle":"2025-05-19T06:38:52.562214Z","shell.execute_reply.started":"2025-05-19T06:38:52.544114Z","shell.execute_reply":"2025-05-19T06:38:52.561664Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def metric(preds, targets, cfg):\n    score = {}\n    for j, label in enumerate(cfg.labels):\n        y_true = targets[:, j]\n        y_pred = preds[:, j]\n        if len(np.unique(y_true)) < 2:\n            score[label] = np.nan\n            continue\n        score[label] = roc_auc_score(y_true, y_pred)\n    score_avg = np.nanmean([v for v in score.values()])\n    return score_avg, score\n\ndef metric_db(train, oofs, cfg):\n    score = {}\n    for j,label in enumerate(cfg.labels):\n        score[label] = roc_auc_score(train.primary_label == label, oofs[label])\n    score_avg = np.mean([v for k,v in score.items()])\n    return score_avg, score\n    \ndef my_softmax(preds):\n    preds = preds - preds.max(1, keepdims=True)\n    preds = np.exp(preds.clip(-20, 0))\n    preds = preds / preds.sum(1, keepdims=True)\n    return preds\n\ndef bce_with_mask(preds, targets, mask):\n    loss = nn.BCEWithLogitsLoss(reduction='none')(preds, targets)\n    loss = loss * mask\n    loss = loss.mean()\n    return loss\n\nclass FocalLossBCE(torch.nn.Module):\n    def __init__(\n            self,\n            alpha: float = 0.25,\n            gamma: float = 2,\n            reduction: str = \"mean\",\n            bce_weight: float = 0.7,\n            focal_weight: float = 1.3,\n    ):\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.reduction = reduction\n        # self.bce = torch.nn.BCEWithLogitsLoss(reduction=reduction)\n        self.bce_weight = bce_weight\n        self.focal_weight = focal_weight\n\n    def forward(self, logits, targets, mask):\n        focall_loss = torchvision.ops.focal_loss.sigmoid_focal_loss(\n            inputs=logits,\n            targets=targets,\n            alpha=self.alpha,\n            gamma=self.gamma,\n            reduction=self.reduction,\n        )\n        bce_loss = bce_with_mask(logits, targets, mask)\n        return (self.bce_weight * bce_loss + self.focal_weight * focall_loss) / 2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:38:52.562989Z","iopub.execute_input":"2025-05-19T06:38:52.563258Z","iopub.status.idle":"2025-05-19T06:38:52.572029Z","shell.execute_reply.started":"2025-05-19T06:38:52.563231Z","shell.execute_reply":"2025-05-19T06:38:52.571441Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cfg.device = torch.device(\"cuda\")\nif cfg.bce:\n    cfg.loss = bce_with_mask\nelse:\n    cfg.loss = bce_with_mask\n    \ncfg.labels = np.array(sorted(train.primary_label.unique()))\ncfg.num_labels = len(cfg.labels)\ncfg.targets = {v: i for i, v in enumerate(cfg.labels)}\n\ncfg.logger = get_logger(cfg)\nseed_torch(cfg.seed)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:38:52.572696Z","iopub.execute_input":"2025-05-19T06:38:52.572957Z","iopub.status.idle":"2025-05-19T06:38:52.635530Z","shell.execute_reply.started":"2025-05-19T06:38:52.572940Z","shell.execute_reply":"2025-05-19T06:38:52.634989Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BirdDataset(Dataset):\n    def __init__(self, train, istrain, cfg):\n        self.cfg = cfg\n        self.istrain = istrain\n        self.filename = train.filename.values\n        self.secondary_labels = train.secondary_labels.values\n        self.first_species = train.first_species.values\n        self.last_species = train.last_species.values\n        \n    def __len__(self):\n        return len(self.filename)\n    \n    def get_audio(self, idx):\n        filename = self.filename[idx]\n        duration = self.cfg.duration * self.cfg.sr\n        if self.istrain:\n            first = np.random.rand() < 0.5\n            audio = load_audio(filename, first, self.istrain, self.cfg)\n            \n            if len(audio) < duration:\n                pad_length = np.random.randint(0, duration - len(audio))\n                audio = np.pad(audio,\n                               ((pad_length, duration - len(audio) - pad_length),),\n                               mode='constant')\n            else:\n                start = np.random.randint(0, len(audio) - duration + 1)\n                audio = audio [start: start + duration]\n        else:\n            audio = load_audio(filename, True, self.istrain, self.cfg)\n            audio = audio[:duration]\n            if len(audio) < duration:\n                pad_length = (duration - len(audio)) // 2\n                audio = np.pad(audio,\n                               ((pad_length, duration - len(audio) - pad_length),),\n                               mode='constant')\n\n        return audio\n    \n    def __getitem__(self, idx):\n        audio = self.get_audio(idx)\n        \n        targets = np.zeros(self.cfg.num_labels, dtype=np.float32)\n        targets[self.cfg.targets[self.first_species[idx]]] = 1.0\n        targets[self.cfg.targets[self.last_species[idx]]] = 1.0\n        \n        secondary_mask = np.ones(self.cfg.num_labels, dtype=np.float32)\n        secondary_labels = self.secondary_labels[idx]\n        \n        if len(secondary_labels) > 0:\n            for label in secondary_labels:\n                if label in self.cfg.targets:\n                    secondary_mask[self.cfg.targets[label]] = 0.0\n                    \n        if self.istrain and self.cfg.other_samples:\n            num_samples = np.random.randint(0, self.cfg.other_samples + 1)\n            for _ in range(num_samples):\n                other_idx = np.random.randint(0, len(self.filename))\n                other_audio = self.get_audio(other_idx)\n                weight = 0.2 + 0.8 * np.random.rand()\n                audio += weight * other_audio\n                targets[self.cfg.targets[self.first_species[other_idx]]] = 1.0\n                targets[self.cfg.targets[self.last_species[other_idx]]] = 1.0\n                secondary_labels = self.secondary_labels[other_idx]\n                \n            if len(secondary_labels) > 0:\n                for label in secondary_labels:\n                    if label in self.cfg.targets:\n                        secondary_mask[self.cfg.targets[label]] = 0.0\n                    \n        secondary_mask = np.maximum(secondary_mask, targets)\n        out = {\n            'audio': audio,\n            'targets': targets,\n            'secondary_mask': secondary_mask\n        }\n        return out\n    \ndef batch_to_device(batch, device):\n        return {k:batch[k].to(device, non_blocking=True) for k in batch.keys() if k not in []}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:38:52.636308Z","iopub.execute_input":"2025-05-19T06:38:52.636553Z","iopub.status.idle":"2025-05-19T06:38:52.648057Z","shell.execute_reply.started":"2025-05-19T06:38:52.636525Z","shell.execute_reply":"2025-05-19T06:38:52.647327Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dataset = BirdDataset(train, True, cfg)\nelt = dataset[0]\nfor k,v in elt.items():\n    print(k, v.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:38:52.648867Z","iopub.execute_input":"2025-05-19T06:38:52.649185Z","iopub.status.idle":"2025-05-19T06:38:52.803344Z","shell.execute_reply.started":"2025-05-19T06:38:52.649168Z","shell.execute_reply":"2025-05-19T06:38:52.802736Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_pl_iter(pl_preds, fold, istrain, cfg):\n    pl_dataset = BirdPLDataset(pl_preds, fold, cfg)\n    pl_data_loader = DataLoader(\n        pl_dataset,\n        batch_size=cfg.pl_batch_size,\n        num_workers=0,\n        shuffle=istrain,\n        pin_memory=False,\n        drop_last = False,\n    )\n    \n    return iter(pl_data_loader), pl_data_loader\n\ndef get_pl_batch(pl_dataloader):\n    try:\n        batch = next(pl_dataloader[0])\n    except StopIteration:\n        pl_data_loader = pl_dataloader[1]\n        pl_data_loader = iter(pl_data_loader), pl_data_loader\n        batch = next(pl_dataloader[0])\n    new_batch = {k : v.view(v.shape[0] * v.shape[1], v.shape[2]) \n                 for k,v in batch.items()}\n    return new_batch","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:38:52.803992Z","iopub.execute_input":"2025-05-19T06:38:52.804180Z","iopub.status.idle":"2025-05-19T06:38:52.809980Z","shell.execute_reply.started":"2025-05-19T06:38:52.804165Z","shell.execute_reply":"2025-05-19T06:38:52.809261Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_data_loader(dataset, istrain, cfg):\n    if istrain:\n        batch_size = cfg.train_batch_size\n    else:\n        batch_size = cfg.valid_batch_size\n    data_loader = DataLoader(\n        dataset,\n        batch_size=batch_size,\n        num_workers=cfg.workers,\n        shuffle=istrain,\n        pin_memory=False,\n        #collate_fn=collate_pad,\n        drop_last = istrain,\n    )\n    return data_loader ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:38:52.810753Z","iopub.execute_input":"2025-05-19T06:38:52.810967Z","iopub.status.idle":"2025-05-19T06:38:52.823759Z","shell.execute_reply.started":"2025-05-19T06:38:52.810952Z","shell.execute_reply":"2025-05-19T06:38:52.823210Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BirdModel(nn.Module):\n    def __init__(self, cfg, pretrained: bool = True):\n        super(BirdModel, self).__init__()\n        self.cfg = cfg\n        self.mel = T.MelSpectrogram(\n            sample_rate=cfg.sr, n_fft=cfg.n_fft, win_length=cfg.win_length, \n            hop_length= cfg.hop_length, f_min=cfg.fmin, f_max=cfg.fmax, \n            n_mels=cfg.n_mels, mel_scale='htk', power=2.0)\n        self.A2DB = T.AmplitudeToDB(stype=\"power\")\n        self.backbone = timm.create_model(\n            cfg.backbone,\n            pretrained=pretrained,\n            drop_rate = cfg.drop_rate,\n            num_classes=cfg.num_labels\n        )\n         \n    def forward(self, input_dict):\n        x = input_dict['audio']\n        with autocast(enabled=False, device_type=\"cuda\"), torch.no_grad():\n            x = x / torch.std(x, 1, keepdim=True)\n            x = x.float()\n            x = self.mel(x)\n            x = self.A2DB(x)\n        with torch.no_grad():\n            x = x.unsqueeze(1)\n            pos = torch.linspace(0., 1., x.size(2)).to(x.device)\n            pos = pos.unsqueeze(0).unsqueeze(0).unsqueeze(-1)\n            pos = pos.expand(x.size(0), 1, x.size(2), x.size(3))\n            x = x.expand(-1, 2, -1, -1)\n            x = torch.cat([x, pos], 1)\n\n        x = self.backbone(x)\n        return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:38:52.824431Z","iopub.execute_input":"2025-05-19T06:38:52.824668Z","iopub.status.idle":"2025-05-19T06:38:52.836115Z","shell.execute_reply.started":"2025-05-19T06:38:52.824652Z","shell.execute_reply":"2025-05-19T06:38:52.835434Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_loader = get_data_loader(dataset, False, cfg)\nmodel = BirdModel(cfg)\nfor batch in data_loader:\n    break\nout = model(batch)\nout.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:38:52.836699Z","iopub.execute_input":"2025-05-19T06:38:52.836927Z","iopub.status.idle":"2025-05-19T06:39:02.596181Z","shell.execute_reply.started":"2025-05-19T06:38:52.836911Z","shell.execute_reply":"2025-05-19T06:39:02.595295Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_epoch(loader, model, optimizer, scheduler, scaler, device, cfg):\n    model.train()\n    model.zero_grad()\n    if cfg.verbose:\n        bar = tqdm(range(len(loader)))\n    else:\n        bar = range(len(loader))\n    load_iter = iter(loader)\n    loss_l = []\n    grad_norm_l = []\n    \n    accumulate = cfg.accumulate\n    \n    for i, batch in zip(bar, load_iter):\n        input_dict = batch_to_device(batch, device)\n        with autocast(enabled=True, device_type=\"cuda\"):\n            targets = input_dict['targets']\n            if cfg.loudness_range:\n                loudness = - np.log(cfg.loudness_range)\n                bs = targets.shape[0]\n                weight = torch.rand(bs, 1).to(targets.device)\n                weight = torch.exp(weight * loudness)\n                audio = input_dict['audio']\n                audio = audio * weight\n                input_dict['audio'] = audio\n            preds = model(input_dict)\n            secondary_mask = input_dict['secondary_mask']\n            loss = cfg.loss(preds, targets, secondary_mask).mean()\n        loss_l.append(loss.detach().cpu().item())\n        scaler.scale(loss / cfg.accumulate).backward() \n        accumulate -= 1\n        if accumulate == 0:\n            if cfg.grad_value:\n                scaler.unscale_(optimizer)\n                nn.utils.clip_grad_value_(model.parameters(), cfg.grad_value)\n            if cfg.grad_norm:\n                scaler.unscale_(optimizer)\n                total_norm = nn.utils.clip_grad_norm_(model.parameters(), cfg.grad_norm).item()\n                if np.isnan(total_norm):\n                    total_norm = cfg.grad_norm\n                else:\n                    total_norm = np.clip(total_norm, 0, cfg.grad_norm)\n                grad_norm_l.append(total_norm)\n            scaler.step(optimizer)     \n            scaler.update()\n            optimizer.zero_grad()\n            accumulate = cfg.accumulate\n            scheduler.step()  \n        del preds, targets, loss, input_dict\n        if cfg.verbose:\n            if cfg.grad_norm:\n                bar.set_description('loss: %.4f grad norm %.1f' % (np.mean(loss_l), np.mean(grad_norm_l),))\n            else:\n                bar.set_description('loss: %.4f ' % np.mean(loss_l))\n    optimizer.zero_grad()\n    del loss_l, bar\n    gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:39:02.597253Z","iopub.execute_input":"2025-05-19T06:39:02.597494Z","iopub.status.idle":"2025-05-19T06:39:02.608199Z","shell.execute_reply.started":"2025-05-19T06:39:02.597470Z","shell.execute_reply":"2025-05-19T06:39:02.607560Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def valid_epoch(loader, model, device, cfg):\n    model.eval()\n    model.zero_grad()\n    if cfg.verbose:\n        bar = tqdm(range(len(loader)))\n    else:\n        bar = range(len(loader))\n    load_iter = iter(loader)\n    preds_l = []\n    targets_l = []\n    with torch.no_grad():\n        for i, batch in zip(bar, load_iter):      \n            input_dict = batch_to_device(batch, device)\n            with autocast(enabled=False, device_type=\"cuda\"):\n                preds = model(input_dict)                \n            preds_l.append(preds.detach().cpu())\n            del preds, input_dict\n            targets = batch['targets']\n            targets_l.append(targets)\n        preds = torch.cat(preds_l)\n        targets = torch.cat(targets_l)\n        return preds.numpy(), targets.numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:39:02.608795Z","iopub.execute_input":"2025-05-19T06:39:02.609098Z","iopub.status.idle":"2025-05-19T06:39:02.641219Z","shell.execute_reply.started":"2025-05-19T06:39:02.609072Z","shell.execute_reply":"2025-05-19T06:39:02.640439Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_optimizer(model, cfg):\n    no_decay = [\"bias\", \"norm\"]\n    if cfg.no_decay:\n        optimizer_parameters = [\n            {'params': [p for n, p in model.named_parameters() \n                        if not any(nd in n for nd in no_decay)],\n             'weight_decay': cfg.decay},\n            {'params': [p for n, p in model.named_parameters() \n                        if any(nd in n for nd in no_decay)],\n             'weight_decay': 0.0},\n        ] \n    else:\n        optimizer_parameters = model.parameters()\n    optimizer = torch.optim.AdamW(optimizer_parameters, lr=cfg.lr)\n    return optimizer\n\ndef get_scheduler(optimizer, train_data_loader, cfg):\n    scheduler = lr_scheduler.OneCycleLR(\n        optimizer,\n        max_lr=cfg.max_lr,\n        epochs=cfg.num_epochs,\n        steps_per_epoch=len(train_data_loader),\n        pct_start=cfg.pct_start,\n        anneal_strategy=\"cos\",\n        final_div_factor=cfg.final_div_factor,\n    )\n    return scheduler","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:39:02.642079Z","iopub.execute_input":"2025-05-19T06:39:02.642357Z","iopub.status.idle":"2025-05-19T06:39:02.656250Z","shell.execute_reply.started":"2025-05-19T06:39:02.642335Z","shell.execute_reply":"2025-05-19T06:39:02.655581Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def save_checkpoint(model, fold, cfg):\n    checkpoint = {\n        'model' : model.state_dict(),\n        'fold' : fold,\n        'seed' : seed,\n        }\n    checkpoint_path = cfg.checkpoint_path\n    save_path = checkpoint_path / ('%s_%d.pth' % (cfg.fname, fold))\n    if cfg.local_rank == 0:\n        cfg.logger.info('saving %s ...' % save_path)\n    torch.save(checkpoint, save_path)\n    if cfg.local_rank == 0:\n        cfg.logger.info('done')\n\ndef load_checkpoint(fold, cfg):\n    if cfg.pretrained_path:\n        checkpoint_path = cfg.pretrained_path\n    else:\n        checkpoint_path = cfg.checkpoint_path\n    save_path = checkpoint_path / ('%s_%d.pth' % (cfg.fname, fold))\n    cfg.logger.info('loading %s ...' % save_path)\n    checkpoint = torch.load(save_path, map_location='cpu')\n    model = BirdModel(cfg, pretrained=False).to(cfg.device)\n    model.load_state_dict(checkpoint['model'], strict=True)\n    model.eval()\n    cfg.logger.info('done')\n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:39:02.656998Z","iopub.execute_input":"2025-05-19T06:39:02.657214Z","iopub.status.idle":"2025-05-19T06:39:02.670248Z","shell.execute_reply.started":"2025-05-19T06:39:02.657197Z","shell.execute_reply":"2025-05-19T06:39:02.669253Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def resample(train_fold, cfg):\n    new_train = []\n    for species, df in train.groupby('species'):\n        new_train.append(df)\n        if len(df) < cfg.resample_train:\n            df = df.sample(n=(cfg.resample_train - len(df)), replace=True, random_state=cfg.seed)\n            new_train.append(df)\n    new_train = pd.concat(new_train).reset_index(drop=True)  \n    return new_train  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:39:02.671231Z","iopub.execute_input":"2025-05-19T06:39:02.671467Z","iopub.status.idle":"2025-05-19T06:39:02.683647Z","shell.execute_reply.started":"2025-05-19T06:39:02.671442Z","shell.execute_reply":"2025-05-19T06:39:02.683081Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scores = []\noofs = pd.DataFrame(columns=cfg.labels, index=train.index, data = 0.0)\noofs['filename'] = train.filename","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:39:02.687338Z","iopub.execute_input":"2025-05-19T06:39:02.687582Z","iopub.status.idle":"2025-05-19T06:39:02.711760Z","shell.execute_reply.started":"2025-05-19T06:39:02.687565Z","shell.execute_reply":"2025-05-19T06:39:02.711189Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for fold in range(cfg.num_folds):\n    if cfg.fold >= 0 and cfg.fold != fold:\n        continue\n    seed = cfg.seed + fold\n    seed_torch(seed)\n    if cfg.pl is not None:\n        pl_train = get_pl_train(cfg)\n    else:\n        pl_train = None\n        train_fold = train[train.fold != fold]\n    valid_dataset = BirdDataset(train[train.fold == fold], False, cfg)\n    valid_dataloader = get_data_loader(valid_dataset, istrain=False, cfg=cfg)\n    device = cfg.device\n    \n    if cfg.pretrained_path:\n        model = load_checkpoint(fold, seed, cfg)\n    else:\n        model = BirdModel(cfg, pretrained=True).to(device)\n    optimizer = get_optimizer(model, cfg)\n    scheduler = None\n    scaler = GradScaler()\n    result = None\n    for epoch in range(cfg.num_epochs):\n        if cfg.pl is not None:\n            train_fold = sample_pl(train, fold, pl_train)\n        if cfg.resample_train:\n            train_fold = resample(train_fold, cfg)\n        train_dataset = BirdDataset(train_fold, True, cfg)\n        train_dataloader = get_data_loader(train_dataset, istrain=True, cfg=cfg)\n\n        if scheduler is None:\n            scheduler = get_scheduler(optimizer, train_dataloader, cfg)\n        train_epoch(train_dataloader, model, optimizer, scheduler, scaler, device, cfg)\n        if valid_dataset is not None:\n            preds, targets = valid_epoch(valid_dataloader, model, device, cfg)\n            if cfg.bce:\n                preds = expit(preds)\n            else:\n                preds = my_softmax(preds)\n            result, _ = metric(preds, targets, cfg)\n            msg = f\"seed {cfg.seed} fold {fold} epoch {epoch} metric {result:.4f}\"\n            cfg.logger.info(msg)\n        else:\n            msg = f\"seed {cfg.seed} fold {fold} epoch {epoch}\"\n            cfg.logger.info(msg)\n    if cfg.local_rank == 0:\n        save_checkpoint(model, fold, cfg)\n    del model, optimizer, scheduler, scaler, train_dataloader, \n    if valid_dataset is not None:\n        del valid_dataloader\n    gc.collect()\n    torch.cuda.empty_cache()\n    scores.append(result)\n    \n    for j,c in enumerate(cfg.labels):\n        oofs.loc[train.fold == fold, c] = preds[:, j]\noofs.to_csv(cfg.checkpoint_path / 'oofs.csv', index=False)\n\nnp.mean(scores), metric_db(train, oofs, cfg)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T06:39:02.712481Z","iopub.execute_input":"2025-05-19T06:39:02.712717Z","execution_failed":"2025-05-19T06:50:18.696Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"res = metric_db(train, oofs, cfg)\nres","metadata":{"trusted":true,"execution":{"execution_failed":"2025-05-19T06:50:18.697Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dfe = train.groupby('primary_label').size()\ndfe.name = 'size'\ndfe = dfe.reset_index().sort_values('primary_label').reset_index(drop=True)\ndfe","metadata":{"trusted":true,"execution":{"execution_failed":"2025-05-19T06:50:18.697Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.DataFrame({'label' : cfg.labels, \n                   'roc_auc' : [res[1][label] for label in cfg.labels],\n                   'size' : dfe['size'].values,\n                  })\ndf","metadata":{"trusted":true,"execution":{"execution_failed":"2025-05-19T06:50:18.697Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.scatter(np.log(df['size']), df['roc_auc'], marker='+')","metadata":{"trusted":true,"execution":{"execution_failed":"2025-05-19T06:50:18.698Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# class SoundScapeDataset():\n#     def __init__(self, soundscape, istrain, cfg):\n#         self.cfg = cfg\n#         self.istrain = istrain\n#         self.soundscape = soundscape\n#         self.duration = int(np.round(soundscape.shape[0] / cfg.sr))\n#         self.starts = np.arange(0, self.duration, cfg.duration)\n#         self.ends = self.starts + cfg.duration\n#         self.starts = cfg.sr * self.starts\n#         self.ends = cfg.sr * self.ends\n        \n#     def __len__(self):\n#         return len(self.starts)\n    \n#     def __getitem__(self, idx):\n#         audio = self.soundscape[self.starts[idx] : self.ends[idx]]\n#         duration = self.cfg.sr * self.cfg.duration\n#         if len(audio) < duration:\n#             pad_length = (duration - len(audio)) // 2\n#             audio = np.pad(audio, \n#                            ((pad_length, duration - len(audio) - pad_length),), \n#                            mode='constant')\n#         out = {\n#             'audio' : torch.from_numpy(audio),\n#         }\n#         return out\n\n# def predict(loader, models, cfg):\n#     for model in models:\n#         model.eval()\n#         model.zero_grad()\n#     if cfg.verbose:\n#         bar = tqdm(range(len(loader)))\n#     else:\n#         bar = range(len(loader))\n#     load_iter = iter(loader)\n#     preds_l = [[] for model in models]\n#     targets_l = []\n#     with torch.no_grad():\n#         for i, batch in zip(bar, load_iter):      \n#             input_dict = batch_to_device(batch, cfg.device)\n#             with autocast(enabled=False, device_type=\"cuda\"):\n#                 for preds, model in zip(preds_l, models):\n#                     preds.append(model(input_dict).detach().cpu()) \n#             del input_dict\n#         preds_l = [torch.cat(preds).squeeze().numpy() for preds in preds_l]\n#         return preds_l\n\n# def load_soundscape(pathname, cfg):\n#     #audio = librosa.load(pathname, sr=32000)[0].astype(np.float32)\n#     filename = pathname.split('/')[-1].split('.')[0]\n#     audio = np.load(cfg.input_path / \"birdclef-data\" / 'unlabeled_soundscapes' / 'unlabeled_soundscapes' / (filename + '.npy'))\n\n#     return audio\n\n# def predict_soundscape(filepath, models, cfg):\n#     waveform = load_soundscape(filepath, cfg)\n#     #return len(waveform) / cfg.sr\n#     #return waveform\n#     #print('duration',  len(waveform) / cfg.sr)\n#     dataset = SoundScapeDataset(waveform, False, cfg)\n#     dataloader = get_data_loader(dataset, False, cfg)\n#     preds_l = predict(dataloader, models, cfg)\n#     if cfg.bce:\n#         preds_l = [expit(preds) for preds in preds_l] # model uses logits\n#     else:\n#         preds_l = [my_softmax(preds) for preds in preds_l] # model uses logits\n#     return preds_l","metadata":{"trusted":true,"execution":{"execution_failed":"2025-05-19T06:50:18.698Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# soundscapes = sorted(glob(str(cfg.soundscape_path / '*.ogg')))\n# soundscapes[:5]","metadata":{"trusted":true,"execution":{"execution_failed":"2025-05-19T06:50:18.698Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# cfg.verbose = False\n# cfg.workers = 0\n# cfg.valid_batch_size = 128\n\n# models = [load_checkpoint(fold, cfg).to(cfg.device)\n#           for fold in range(cfg.num_folds)\n#          ]\n# preds = {}\n# for soundscape in tqdm(soundscapes):\n#     preds[soundscape]  = predict_soundscape(soundscape, models, cfg)\n# with open(cfg.checkpoint_path / f\"pl_all.pkl\", \"wb\") as file:\n#     pkl.dump(preds, file)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-05-19T06:50:18.698Z"}},"outputs":[],"execution_count":null}]}