{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"},{"sourceId":11075844,"sourceType":"datasetVersion","datasetId":6902807},{"sourceId":11749667,"sourceType":"datasetVersion","datasetId":7368311,"isSourceIdPinned":true},{"sourceId":11750659,"sourceType":"datasetVersion","datasetId":7372366},{"sourceId":11751579,"sourceType":"datasetVersion","datasetId":7377567},{"sourceId":384567,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":317334,"modelId":337866}],"dockerImageVersionId":31011,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport random\nfrom tqdm import tqdm\nfrom logging import getLogger, INFO, StreamHandler, FileHandler, Formatter\nimport gc\nimport pickle as pkl\n\nimport librosa\n\nfrom torch.utils.data import DataLoader, Dataset\nimport torchaudio\nimport torchaudio.transforms as T\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.amp import autocast, GradScaler\nfrom torch.optim import lr_scheduler, Adam, AdamW\n\nimport timm\n\nfrom glob import glob\nfrom sklearn.model_selection import KFold, GroupKFold, StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nfrom scipy.special import logit, expit","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:20.102842Z","iopub.execute_input":"2025-05-09T20:23:20.103388Z","iopub.status.idle":"2025-05-09T20:23:27.997204Z","shell.execute_reply.started":"2025-05-09T20:23:20.103364Z","shell.execute_reply":"2025-05-09T20:23:27.996327Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nfrom pathlib import Path\nfrom types import SimpleNamespace\n\ncfg = SimpleNamespace(**{})\ncfg.num_folds = 5\ncfg.fold = -1\ncfg.gpu = \"0\"\nos.environ['CUDA_VISIBLE_DEVICES'] = cfg.gpu\n\ncfg.fname = 'efficientvit_b0'\ncfg.seed = 2025\n\ncfg.input_path = Path('../input')\ncfg.comp_data_path = cfg.input_path / 'birdclef-2025'\ncfg.save_path = Path('../working')\ncfg.soundscape_path = cfg.comp_data_path / 'train_soundscapes'\ncfg.audio_path = Path(\"/kaggle/input/birdclef-2025/train_audio\")\n\ncfg.logger_file = True\n\n\n# # V7\n# # image size\n# cfg.image_height = 224\n# cfg.image_width = 224\n# cfg.target_shape = (224, 224)\n\n# # audio\n# cfg.duration = 10\n# cfg.sr = 32000\n# cfg.fmin = 50\n# cfg.fmax = 14000\n# cfg.n_fft = 1536\n# cfg.n_mels = cfg.image_height\n# cfg.win_length = 1024\n# cfg.hop_length = int((cfg.duration * cfg.sr - cfg.win_length + cfg.n_fft) / (cfg.image_width)) + 1 \n\n\n# V10\n# image size\ncfg.image_height = 256\ncfg.image_width = 256\ncfg.target_shape = (256, 256)\n\n# audio\ncfg.duration = 10\ncfg.sr = 32000\ncfg.fmin = 20\ncfg.fmax = 16000\ncfg.n_fft = 2048\ncfg.n_mels = 512\ncfg.win_length = 2048\ncfg.hop_length = 128\n\n\n# training HP\ncfg.num_epochs = 20\ncfg.train_batch_size = 64\ncfg.valid_batch_size = 128\ncfg.workers = 2\ncfg.grad_value = 2.0\ncfg.grad_norm = 0.0\ncfg.grad_norm_type = 2\ncfg.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ncfg.accumulate = 1\n\n# optimizer\ncfg.lr = 7e-5\ncfg.decay = 0.01\ncfg.opt_beta1 = 0.9\ncfg.opt_beta2 = 0.999\ncfg.opt_eps = 1e-8\ncfg.optimizer = 'AdamW'\ncfg.no_decay = True\n\n# scheduler\ncfg.pct_start = 0.1\ncfg.max_lr = 3e-3\ncfg.final_div_factor = 100\n\n# augmentations\ncfg.resample_train = 10\ncfg.other_samples = 1\ncfg.max_shift = 1\ncfg.loudness_range = 10.0\n\n# logging\ncfg.local_rank = 0\ncfg.verbose=True\n\n# model\ncfg.backbone = 'efficientvit_b0.r224_in1k'\ncfg.gem_pooling = False\ncfg.bce = False\ncfg.drop_rate = 0.1\n\n# tasks hp\ncfg.train_model = True\ncfg.num_folds = 5\ncfg.pl = None\ncfg.pretrained_path = None ","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:27.998584Z","iopub.execute_input":"2025-05-09T20:23:27.999284Z","iopub.status.idle":"2025-05-09T20:23:28.054239Z","shell.execute_reply.started":"2025-05-09T20:23:27.999262Z","shell.execute_reply":"2025-05-09T20:23:28.053573Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from pprint import pprint\nmodel_names = timm.list_models(pretrained=True)\npprint(model_names[::len(model_names)//10])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:28.055049Z","iopub.execute_input":"2025-05-09T20:23:28.055313Z","iopub.status.idle":"2025-05-09T20:23:28.083506Z","shell.execute_reply.started":"2025-05-09T20:23:28.055293Z","shell.execute_reply":"2025-05-09T20:23:28.082778Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cfg.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ncfg.device","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:28.085478Z","iopub.execute_input":"2025-05-09T20:23:28.085885Z","iopub.status.idle":"2025-05-09T20:23:28.102208Z","shell.execute_reply.started":"2025-05-09T20:23:28.085864Z","shell.execute_reply":"2025-05-09T20:23:28.101458Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if cfg.train_model:\n    checkpoint_path = cfg.save_path / cfg.fname\n    if not cfg.save_path.exists():\n        cfg.save_path.mkdir()\n    if not checkpoint_path.exists():\n        checkpoint_path.mkdir()\n    checkpoint_path = checkpoint_path / \"exp_0\"\n    exp = 0\n    while(checkpoint_path.exists()):\n        exp += 1\n        checkpoint_path = cfg.save_path / f\"exp_{exp}\"\n    checkpoint_path.mkdir()\n    cfg.checkpoint_path = checkpoint_path\n    print(f\"Saving checkpoint path to {checkpoint_path}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:28.102883Z","iopub.execute_input":"2025-05-09T20:23:28.103090Z","iopub.status.idle":"2025-05-09T20:23:28.118230Z","shell.execute_reply.started":"2025-05-09T20:23:28.103073Z","shell.execute_reply":"2025-05-09T20:23:28.117438Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_logger(cfg):\n    logger = getLogger(cfg.fname)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    if cfg.logger_file and cfg.train_model:\n        filename= cfg.checkpoint_path / \"run.log\"\n        handler2 = FileHandler(filename=filename)\n        handler1.setFormatter(Formatter(\"%(message)s\"))\n        logger.addHandler(handler2)\n    return logger\n\ndef seed_torch(seed_value):\n    random.seed(seed_value) # Python\n    np.random.seed(seed_value) # cpu vars\n    torch.manual_seed(seed_value) # cpu  vars    \n    if torch.cuda.is_available(): \n        torch.cuda.manual_seed(seed_value)\n        torch.cuda.manual_seed_all(seed_value) # gpu vars\n    if torch.backends.cudnn.is_available:\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:28.119082Z","iopub.execute_input":"2025-05-09T20:23:28.119356Z","iopub.status.idle":"2025-05-09T20:23:28.130014Z","shell.execute_reply.started":"2025-05-09T20:23:28.119329Z","shell.execute_reply":"2025-05-09T20:23:28.129254Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ntrain = pd.read_csv(Path(\"/kaggle/input/bc25-numpy-data-kuduck/bc25-numpy-data-kuduck\") / 'train_chunks_df.csv')\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:28.130792Z","iopub.execute_input":"2025-05-09T20:23:28.131049Z","iopub.status.idle":"2025-05-09T20:23:28.857722Z","shell.execute_reply.started":"2025-05-09T20:23:28.131021Z","shell.execute_reply":"2025-05-09T20:23:28.857069Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:28.858407Z","iopub.execute_input":"2025-05-09T20:23:28.858694Z","iopub.status.idle":"2025-05-09T20:23:28.863900Z","shell.execute_reply.started":"2025-05-09T20:23:28.858664Z","shell.execute_reply":"2025-05-09T20:23:28.863155Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train = pd.read_csv(cfg.comp_data_path / 'train.csv')\n# train\n\n# train[\"species\"] = [filename.split(\"/\")[0] for filename in train[\"filename\"]]\n# train[\"record\"] = [filename.split(\"/\")[1] for filename in train[\"filename\"]]\n# train[\"secondary_labels\"] = [eval(sls) for sls in train[\"secondary_labels\"]]\n\n# df = train.groupby(\"record\").agg({\n#     'species': ['first', 'last'],\n#     'secondary_labels': ['first', 'last'],\n# })\n\n# df.columns = [\"first_species\", \"last_species\", \"first_secondary\", \"last_secondary\"]\n# df.reset_index(inplace=True)\n\n# train = train.merge(df[['record', 'first_species', 'last_species',]],\n#                    how='left',\n#                    on='record')\n# def load_audio(file_name, isfirst, istrain, cfg):\n#     filepath = file_name.split(\"/\")[0]\n#     fname = file_name.split(\"/\")[1].split(\".\")[0]\n#     filepath = cfg.input_path / \"birdclef-data\" / \"birdclef_data\" / \"birdclef_data\" / filepath\n\n#     if istrain:\n#         max_duration = int((cfg.duration + cfg.max_shift) * cfg.sr)\n#     else:\n#         max_duration = cfg.duration * cfg.sr\n\n#     if isfirst:\n#         filepath = filepath / f\"first10_{fname}.npy\"\n#         audio = np.load(filepath)\n#         audio = audio[:max_duration]\n#     else:\n#         filepath = filepath / f\"last10_{fname}.npy\"\n#         audio = np.load(filepath)\n#         audio = audio[-max_duration:]\n\n#     return audio\n# new_train = []\n\n# kf = KFold(n_splits=cfg.num_folds, shuffle=True, random_state=0)\n# for species, df in train.groupby('species'):\n#     df = df.reset_index(drop=True)\n#     df['fold'] = -1\n#     if len(df) < 5:\n#         df['fold'] = 0\n#     else:\n#         for fold, (train_idx, valid_idx) in enumerate(kf.split(df, df.primary_label)):\n#             df.loc[valid_idx, \"fold\"] = fold\n#     new_train.append(df)\n# # new_train = pd.concat(new_train).reset_index(drop=True)    \n# # new_train.fold.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:28.864807Z","iopub.execute_input":"2025-05-09T20:23:28.865105Z","iopub.status.idle":"2025-05-09T20:23:28.875032Z","shell.execute_reply.started":"2025-05-09T20:23:28.865073Z","shell.execute_reply":"2025-05-09T20:23:28.874337Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_audio(file_name, isfirst, istrain, cfg):\n    filepath = file_name.split(\"/\")[0]\n    fname = file_name.split(\"/\")[1].split(\".\")[0]\n    filepath = cfg.input_path / \"birdclef-data\" / \"birdclef_data\" / \"birdclef_data\" / filepath\n\n    if istrain:\n        max_duration = int((cfg.duration + cfg.max_shift) * cfg.sr)\n    else:\n        max_duration = cfg.duration * cfg.sr\n\n    if isfirst:\n        filepath = filepath / f\"first10_{fname}.npy\"\n        audio = np.load(filepath)\n        audio = audio[:max_duration]\n    else:\n        filepath = filepath / f\"last10_{fname}.npy\"\n        audio = np.load(filepath)\n        audio = audio[-max_duration:]\n\n    return audio","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:28.878039Z","iopub.execute_input":"2025-05-09T20:23:28.878250Z","iopub.status.idle":"2025-05-09T20:23:28.890114Z","shell.execute_reply.started":"2025-05-09T20:23:28.878235Z","shell.execute_reply":"2025-05-09T20:23:28.889560Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.copy()\nskf = StratifiedKFold(cfg.num_folds, shuffle=True, random_state=2911)\nfor fold, (train_idx, valid_idx) in enumerate(skf.split(train, train.label)):\n    train.loc[valid_idx, 'fold'] = fold\ntrain.fold = train.fold.astype(int)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:28.890896Z","iopub.execute_input":"2025-05-09T20:23:28.891388Z","iopub.status.idle":"2025-05-09T20:23:28.945359Z","shell.execute_reply.started":"2025-05-09T20:23:28.891363Z","shell.execute_reply":"2025-05-09T20:23:28.944606Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.fold.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:28.946214Z","iopub.execute_input":"2025-05-09T20:23:28.946850Z","iopub.status.idle":"2025-05-09T20:23:28.956675Z","shell.execute_reply.started":"2025-05-09T20:23:28.946830Z","shell.execute_reply":"2025-05-09T20:23:28.955986Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"set(range(206)) - set(train.label.unique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:28.957373Z","iopub.execute_input":"2025-05-09T20:23:28.957636Z","iopub.status.idle":"2025-05-09T20:23:28.968680Z","shell.execute_reply.started":"2025-05-09T20:23:28.957611Z","shell.execute_reply":"2025-05-09T20:23:28.968104Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def metric(preds, targets, cfg):\n    score = {}\n    for j, label in enumerate(cfg.labels):\n        y_true = targets[:, j]\n        y_pred = preds[:, j]\n        if len(np.unique(y_true)) < 2:\n            score[label] = 0\n            continue\n        score[label] = roc_auc_score(y_true, y_pred)\n    score_avg = np.mean([v for v in score.values()])\n    return score_avg, score\n\ndef metric_db(train, oofs, cfg):\n    score = {}\n    for j,label in enumerate(cfg.labels):\n        score[label] = roc_auc_score(train.label == label, oofs[label])\n    score_avg = np.mean([v for k,v in score.items()])\n    return score_avg, score\n    \ndef my_softmax(preds):\n    preds = preds - preds.max(1, keepdims=True)\n    preds = np.exp(preds.clip(-20, 0))\n    preds = preds / preds.sum(1, keepdims=True)\n    return preds\n\ndef bce_with_mask(preds, targets, mask):\n    loss = nn.BCEWithLogitsLoss(reduction='none')(preds, targets)\n    loss = loss * mask\n    loss = loss.mean()\n    return loss","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:28.969317Z","iopub.execute_input":"2025-05-09T20:23:28.969540Z","iopub.status.idle":"2025-05-09T20:23:28.979770Z","shell.execute_reply.started":"2025-05-09T20:23:28.969525Z","shell.execute_reply":"2025-05-09T20:23:28.979057Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_metadata = pd.read_csv(\"/kaggle/input/birdclef-2025/train.csv\")\ntrain_metadata.head()\ncompetition_classes = sorted(train_metadata.primary_label.unique())\n\ncfg.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nif cfg.bce:\n    cfg.loss = bce_with_mask\nelse:\n    cfg.loss = nn.CrossEntropyLoss()\n    \ncfg.labels = np.array(competition_classes)\ncfg.num_labels = len(cfg.labels)\ncfg.targets = {v: i for i, v in enumerate(cfg.labels)}\n\ncfg.logger = get_logger(cfg)\nseed_torch(cfg.seed)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:28.980637Z","iopub.execute_input":"2025-05-09T20:23:28.980915Z","iopub.status.idle":"2025-05-09T20:23:29.164757Z","shell.execute_reply.started":"2025-05-09T20:23:28.980898Z","shell.execute_reply":"2025-05-09T20:23:29.164063Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ss_df = pd.read_csv('/kaggle/input/bc25-numpy-ssdata-kuduck/bc25-numpy-ssdata-kuduck/ss_chunks_df.csv')\nss_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:29.165572Z","iopub.execute_input":"2025-05-09T20:23:29.165839Z","iopub.status.idle":"2025-05-09T20:23:29.262289Z","shell.execute_reply.started":"2025-05-09T20:23:29.165809Z","shell.execute_reply":"2025-05-09T20:23:29.261688Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BirdDataset(Dataset):\n    def __init__(self, train, istrain, cfg):\n        self.cfg = cfg\n        self.istrain = istrain\n        self.filename = train.filename.values\n        self.label = train.label.values\n        self.labels_proba = []\n        for i in range(len(train)):\n            self.labels_proba.append(\n                np.array(list(map(float, train.labels_proba.values[i].replace('[', '').replace(']', '').split()))))\n        self.labels_proba = np.array(self.labels_proba)\n        \n    def __len__(self):\n        return len(self.filename)\n    \n    def get_mel(self, idx):\n        filename = self.filename[idx]\n\n        return np.load(f'/kaggle/input/bc25-numpy-data-kuduck/{filename}')\n    \n    def __getitem__(self, idx):\n        image = self.get_mel(idx)\n\n        label = self.label[idx]\n        labels_proba = self.labels_proba[idx]\n        \n        out = {\n            'image': image,\n            'label': label,\n            'labels_proba': labels_proba\n        }\n        return out\n\nclass BirdUnlabeledDataset(Dataset):\n    def __init__(self, ss_df, cfg):\n        self.cfg = cfg\n        self.filename = ss_df.filename.values\n        \n    def __len__(self):\n        return len(self.filename)\n    \n    def get_mel(self, idx):\n        filename = self.filename[idx]\n\n        return np.load(f'/kaggle/input/bc25-numpy-ssdata-kuduck/{filename}')\n    \n    def __getitem__(self, idx):\n        image = self.get_mel(idx)\n\n        out = {\n            'image': image,\n        }\n        return out\n\ndef batch_to_device(batch, device):\n        return {k:batch[k].to(device, non_blocking=True) for k in batch.keys() if k not in []}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:29.263013Z","iopub.execute_input":"2025-05-09T20:23:29.263228Z","iopub.status.idle":"2025-05-09T20:23:29.271416Z","shell.execute_reply.started":"2025-05-09T20:23:29.263208Z","shell.execute_reply":"2025-05-09T20:23:29.270706Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dataset = BirdDataset(train, True, cfg)\nelt = dataset[0]\nfor k,v in elt.items():\n    print(k, v.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:29.272175Z","iopub.execute_input":"2025-05-09T20:23:29.272387Z","iopub.status.idle":"2025-05-09T20:23:32.105702Z","shell.execute_reply.started":"2025-05-09T20:23:29.272363Z","shell.execute_reply":"2025-05-09T20:23:32.104976Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ss_dataset = BirdUnlabeledDataset(ss_df, cfg)\nelt = ss_dataset[0]\nfor k,v in elt.items():\n    print(k, v.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:32.106453Z","iopub.execute_input":"2025-05-09T20:23:32.106688Z","iopub.status.idle":"2025-05-09T20:23:32.112133Z","shell.execute_reply.started":"2025-05-09T20:23:32.106664Z","shell.execute_reply":"2025-05-09T20:23:32.111444Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_data_loader(dataset, istrain, cfg):\n    if istrain:\n        batch_size = cfg.train_batch_size\n    else:\n        batch_size = cfg.valid_batch_size\n    data_loader = DataLoader(\n        dataset,\n        batch_size=batch_size,\n        num_workers=cfg.workers,\n        shuffle=istrain,\n        pin_memory=False,\n        #collate_fn=collate_pad,\n        drop_last = istrain,\n    )\n    return data_loader ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:32.112803Z","iopub.execute_input":"2025-05-09T20:23:32.113003Z","iopub.status.idle":"2025-05-09T20:23:32.124849Z","shell.execute_reply.started":"2025-05-09T20:23:32.112987Z","shell.execute_reply":"2025-05-09T20:23:32.124079Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def gem(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6, p_trainable=True):\n        super(GeM,self).__init__()\n        if p_trainable:\n            self.p = nn.Parameter(torch.ones(1)*p)\n        else:\n            self.p = p\n        self.eps = eps\n\n    def forward(self, x):\n        return gem(x, p=self.p, eps=self.eps)       \n    def __repr__(self):\n        return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + ', ' + 'eps=' + str(self.eps) + ')'\n\nclass BirdModel(nn.Module):\n    def __init__(self, cfg, pretrained: bool = True):\n        super(BirdModel, self).__init__()\n        self.cfg = cfg\n        self.mel = T.MelSpectrogram(\n            sample_rate=cfg.sr, n_fft=cfg.n_fft, win_length=cfg.win_length, \n            hop_length= cfg.hop_length, f_min=cfg.fmin, f_max=cfg.fmax, \n            n_mels=cfg.n_mels, mel_scale='htk', power=2.0)\n        self.A2DB = T.AmplitudeToDB(stype=\"power\")\n        self.backbone = timm.create_model(\n            cfg.backbone,\n            pretrained=pretrained,\n            drop_rate = cfg.drop_rate,\n            num_classes=cfg.num_labels\n        )\n        #if cfg.gem_pooling == \"gem\":\n        #    self.backbone.head.global_pool = GeM(p_trainable=args.p_trainable)\n         \n    # def forward(self, input_dict):\n    #     x = input_dict['audio']\n    #     with autocast(enabled=False, device_type=\"cuda\"), torch.no_grad():\n    #         x = x / torch.std(x, 1, keepdim=True)\n    #         x = x.float()\n    #         x = self.mel(x)\n    #         x = self.A2DB(x)\n    #         x = (x - 40) / 80\n    #     with torch.no_grad():\n    #         x = x.unsqueeze(1)\n    #         pos = torch.linspace(0., 1., x.size(2)).to(x.device)\n    #         pos = pos.unsqueeze(0).unsqueeze(0).unsqueeze(-1)\n    #         pos = pos.expand(x.size(0), 1, x.size(2), x.size(3))\n    #         x = x.expand(-1, 2, -1, -1)\n    #         x = torch.cat([x, pos], 1)\n\n    #     x = self.backbone(x)\n    #     return x\n\n\n    def forward(self, inp):\n        if isinstance(inp, dict):\n            x = inp['image']\n        else:\n            x = inp\n        with torch.no_grad():\n            x = x.unsqueeze(1)\n            pos = torch.linspace(0., 1., x.size(2)).to(x.device)\n            pos = pos.unsqueeze(0).unsqueeze(0).unsqueeze(-1)\n            pos = pos.expand(x.size(0), 1, x.size(2), x.size(3))\n            x = x.expand(-1, 2, -1, -1)\n            x = torch.cat([x, pos], 1)\n\n        x = self.backbone(x)\n        return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:32.125684Z","iopub.execute_input":"2025-05-09T20:23:32.125933Z","iopub.status.idle":"2025-05-09T20:23:32.139622Z","shell.execute_reply.started":"2025-05-09T20:23:32.125907Z","shell.execute_reply":"2025-05-09T20:23:32.138776Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_loader = get_data_loader(dataset, False, cfg)\nmodel = BirdModel(cfg)\nfor batch in data_loader:\n    break\nout = model(batch)\nout.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:32.140359Z","iopub.execute_input":"2025-05-09T20:23:32.140639Z","iopub.status.idle":"2025-05-09T20:23:38.305377Z","shell.execute_reply.started":"2025-05-09T20:23:32.140614Z","shell.execute_reply":"2025-05-09T20:23:38.304410Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import collections\n\ndef build_uda_cross_entropy(args, model, all_samples):\n    masks = {}\n    logits = {}\n    labels = {}\n    cross_entropy = {}\n\n    batch_size = args.batch_size\n    uda_data = args.uda_data\n\n    (l_batch, l_batch_y), u_batch_ori = all_samples\n\n    labels['l'] = l_batch_y\n    logits['l'] = model(l_batch)\n    logits['u_ori'] = model(u_batch_ori)\n    # logits['u_aug'] = model(u_batch_aug)\n    \n    all_logits = torch.cat([logits['l'], logits['u_ori']], dim=0)\n\n    cross_entropy['l'] = F.cross_entropy(\n        logits['l'], labels['l'], reduction='none')\n    probs_l = logits['l'].softmax(dim=-1).detach()\n    correct_probs = (labels['l'] * probs_l).sum(dim=-1)\n    r = args.global_step / float(args.num_train_steps)\n    l_threshold = r * (1. - 1. / 4) + 1. / 4\n    masks['l'] = (correct_probs <= l_threshold)\n    cross_entropy['l'] = cross_entropy['l'].sum() / batch_size\n    \n    labels['u_ori'] = F.softmax(logits['u_ori'], dim=-1).detach()\n    cross_entropy['u'] = -labels['u_ori'] * F.log_softmax(logits['u_ori'], dim=-1)\n    largest_probs = labels['u_ori'].max(dim=-1)[0]\n    masks['u'] = largest_probs >= args.uda_threshold\n    masks['u'] = masks['u'].float().detach().unsqueeze(1)\n\n    cross_entropy['u'] = (cross_entropy['u']*masks['u']).sum() / (batch_size * uda_data)\n\n    if torch.isnan(cross_entropy['u']).any():\n        print(masks['u'])\n        print(labels['u_ori'])\n        print(-labels['u_ori'] * F.log_softmax(logits['u_ori'], dim=-1))\n        print(F.log_softmax(logits['u_ori'], dim=-1))\n        assert False\n        \n    \n    return logits, labels, masks, cross_entropy\n\n\nclass MPL:\n    def __init__(self):\n        self.moving_dot_product = None\n\n\n    def step_fn(self, args, models, optimizers, all_samples):\n        batch_size = args.batch_size\n        uda_data = args.uda_data\n        \n        teacher, student = models\n        teacher_optimizer, student_optimizer = optimizers\n\n\n        teacher.train().to(args.device)\n        student.train().to(args.device)\n        \n        (l_batch, l_batch_y), u_batch_ori = all_samples\n\n        \n        # all calls to teacher\n        logits, labels, masks, cross_entropy = build_uda_cross_entropy(args, teacher, all_samples)\n\n        # 1st call to student\n        logits['s_on_u'] = student(u_batch_ori)\n        logits['s_on_l_old'] = student(l_batch)\n\n        # for backprop\n        cross_entropy['s_on_u'] = F.cross_entropy(logits['s_on_u'], F.softmax(logits['u_ori'], dim=-1).detach())\n\n        # for Taylor\n        cross_entropy['s_on_l_old'] = F.cross_entropy(logits['s_on_l_old'], labels['l'])\n        shadow = cross_entropy['s_on_l_old'].clone().detach()\n\n            \n        w_s = {}\n        g_s = {}\n        g_n = {}\n        lr = {}\n        optim = {}\n\n        \n        if ( torch.isnan(cross_entropy['s_on_u']).any() or \n            torch.isnan(shadow).any() or \n            torch.isnan(logits['s_on_u']).any() or \n            torch.isnan(logits['s_on_l_old']).any() or \n            torch.isnan(cross_entropy['l']).any() or \n            torch.isnan(cross_entropy['s_on_u']).any() or \n            torch.isnan(cross_entropy['s_on_l_old']).any()):\n            print(logits)\n            print(cross_entropy)\n            assert False\n\n\n        # 2nd call to student\n        student_optimizer.zero_grad()\n        cross_entropy['s_on_u'].backward(retain_graph=True)\n        student_optimizer.step()\n\n        logits['s_on_l_new'] = student(l_batch)\n        cross_entropy['s_on_l_new'] = F.cross_entropy(logits['s_on_l_new'], labels['l'])\n\n        # dot_product = shadow - cross_entropy['s_on_l_new']\n        dot_product = cross_entropy['s_on_l_new'] - shadow\n        dot_product = dot_product.detach()\n\n        if self.moving_dot_product is None:\n            self.moving_dot_product = dot_product\n        self.moving_dot_product = self.moving_dot_product - 0.01 * (self.moving_dot_product - dot_product)\n\n        mpl_weight = 10.0\n        adjusted_dot_product = (dot_product - self.moving_dot_product).detach() * mpl_weight\n\n        cross_entropy['mpl'] = F.cross_entropy(logits['u_ori'], F.softmax(logits['u_ori'], dim=-1).detach())\n\n\n        # teacher train op\n        uda_weight = args.uda_weight * min(1., args.global_step / args.uda_steps)\n        teacher_loss = (\n            cross_entropy['l'] +\n            uda_weight * cross_entropy['u'] +\n            adjusted_dot_product * (cross_entropy['mpl'])\n        )\n\n        teacher_optimizer.zero_grad()\n        teacher_loss.backward()\n        teacher_optimizer.step()\n        \n        \n        logs = collections.OrderedDict()\n        logs['global_step'] = args.global_step\n        logs['cross_entropy/student_on_u'] = cross_entropy['s_on_u'].item()\n        logs['cross_entropy/student_on_l'] = cross_entropy['s_on_l_new'].item()\n        logs['cross_entropy/teacher_on_u'] = cross_entropy['u'].item()\n        logs['cross_entropy/teacher_on_l'] = cross_entropy['l'].item()\n        logs['mpl/dot_product'] = (dot_product).item()\n        logs['mpl/moving_dot_product'] = (self.moving_dot_product).item()\n        logs['mpl/mpl'] = (adjusted_dot_product * cross_entropy['mpl']).item()\n        logs['uda/u_ratio'] = (masks['u'].to(torch.float32).mean()).item()\n        logs['uda/l_ratio'] = (masks['l'].to(torch.float32).mean()).item()\n        logs['uda/weight'] = (uda_weight)\n\n        \n        if (\n            torch.isnan(cross_entropy['s_on_u']).any() or \\\n            torch.isnan(cross_entropy['s_on_l_new']).any() or \\\n            torch.isnan(cross_entropy['u']).any() or \\\n            torch.isnan(cross_entropy['l']).any() or \\\n            torch.isnan(dot_product).any() or \\\n            torch.isnan(dot_product).any() or \\\n            torch.isnan(self.moving_dot_product).any() or \\\n            torch.isnan(masks['u']).any() or \\\n            torch.isnan(masks['l']).any() \\\n        ):\n            print(logs)\n            assert False\n        \n        self.step_info = logs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:38.306613Z","iopub.execute_input":"2025-05-09T20:23:38.306866Z","iopub.status.idle":"2025-05-09T20:23:38.324526Z","shell.execute_reply.started":"2025-05-09T20:23:38.306841Z","shell.execute_reply":"2025-05-09T20:23:38.323878Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_epoch(loader, model, optimizer, scheduler, scaler, device, cfg):\n    model.train()\n    model.zero_grad()\n    if cfg.verbose:\n        bar = tqdm(range(len(loader)))\n    else:\n        bar = range(len(loader))\n    load_iter = iter(loader)\n    loss_l = []\n    grad_norm_l = []\n    \n    accumulate = cfg.accumulate\n    \n    for i, batch in zip(bar, load_iter):\n        input_dict = batch_to_device(batch, device)\n        with autocast(enabled=True, device_type=\"cuda\"):\n            # targets = input_dict['label']\n            targets = input_dict['labels_proba']\n            bs = targets.shape[0]\n            preds = model(input_dict)\n            \n            loss = cfg.loss(preds, targets).mean()\n        loss_l.append(loss.detach().cpu().item())\n        scaler.scale(loss / cfg.accumulate).backward() \n        accumulate -= 1\n        if accumulate == 0:\n            if cfg.grad_value:\n                scaler.unscale_(optimizer)\n                nn.utils.clip_grad_value_(model.parameters(), cfg.grad_value)\n            if cfg.grad_norm:\n                scaler.unscale_(optimizer)\n                total_norm = nn.utils.clip_grad_norm_(model.parameters(), cfg.grad_norm).item()\n                if np.isnan(total_norm):\n                    total_norm = cfg.grad_norm\n                else:\n                    total_norm = np.clip(total_norm, 0, cfg.grad_norm)\n                grad_norm_l.append(total_norm)\n            scaler.step(optimizer)     \n            scaler.update()\n            optimizer.zero_grad()\n            accumulate = cfg.accumulate\n            scheduler.step()  \n        del preds, targets, loss, input_dict\n        if cfg.verbose:\n            if cfg.grad_norm:\n                bar.set_description('loss: %.4f grad norm %.1f' % (np.mean(loss_l), np.mean(grad_norm_l),))\n            else:\n                bar.set_description('loss: %.4f ' % np.mean(loss_l))\n    optimizer.zero_grad()\n    del loss_l, bar\n    gc.collect()\n\ndef train_mpl_epoch(train_loader, ss_loader, models, optimizers, schedulers, device, cfg):\n    torch.cuda.empty_cache()\n    model.train()\n    model.zero_grad()\n    if cfg.verbose:\n        bar = tqdm(range(len(train_loader)))\n    else:\n        bar = range(len(train_loader))\n\n    cfg.global_step = 0\n    \n    # teacher, student = models\n    # teacher_optimizer, student_optimizer = optimizers\n    \n    train_iter = iter(train_loader)\n    ss_iter = iter(ss_loader)\n    def get_next_train_sample():\n        nonlocal train_iter\n        try: return next(train_iter)\n        except StopIteration:\n            train_iter = iter(train_loader)\n            return next(train_iter)\n    def get_next_ss_sample():\n        nonlocal ss_iter\n        try: return next(ss_iter)\n        except StopIteration:\n            ss_iter = iter(ss_loader)\n            return next(ss_iter)\n    \n    loss_l = []\n    mpl = MPL()\n    \n    for i in bar:\n        train_batch = batch_to_device(get_next_train_sample(), cfg.device)\n        ss_batch = batch_to_device(get_next_ss_sample(), cfg.device)\n        l_batch = train_batch['image']\n        l_batch_y = train_batch['labels_proba']\n        u_batch = ss_batch['image']\n        \n        all_samples = (l_batch, l_batch_y), u_batch\n        mpl.step_fn(cfg, models, optimizers, all_samples)\n        schedulers[0].step()  \n        schedulers[1].step()  \n        logs = mpl.step_info\n        \n        loss_l.append(logs['cross_entropy/teacher_on_l'])\n        if cfg.verbose:\n            bar.set_description('t: (%.4f, %.4f); s: (%.4f, %.4f); mpl: %.4f ' % (\n                np.mean(loss_l), logs['cross_entropy/teacher_on_u'], \n                logs['cross_entropy/student_on_l'], logs['cross_entropy/student_on_u'],\n                logs['mpl/mpl'],\n            ))\n    optimizer.zero_grad()\n    del loss_l, bar\n    gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:38.325269Z","iopub.execute_input":"2025-05-09T20:23:38.325523Z","iopub.status.idle":"2025-05-09T20:23:38.345244Z","shell.execute_reply.started":"2025-05-09T20:23:38.325505Z","shell.execute_reply":"2025-05-09T20:23:38.344547Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def valid_epoch(loader, model, device, cfg):\n    model.eval()\n    model.zero_grad()\n    if cfg.verbose:\n        bar = tqdm(range(len(loader)))\n    else:\n        bar = range(len(loader))\n    load_iter = iter(loader)\n    preds_l = []\n    targets_l = []\n    with torch.no_grad():\n        for i, batch in zip(bar, load_iter):      \n            input_dict = batch_to_device(batch, device)\n            with autocast(enabled=False, device_type=\"cuda\"):\n                preds = model(input_dict)                \n            preds_l.append(preds.detach().cpu())\n            del preds, input_dict\n            targets = batch['label']\n            targets_l.append(targets)\n        preds = torch.cat(preds_l)\n        targets = torch.cat(targets_l)\n        return preds.numpy(), targets.numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:38.345939Z","iopub.execute_input":"2025-05-09T20:23:38.346115Z","iopub.status.idle":"2025-05-09T20:23:38.361643Z","shell.execute_reply.started":"2025-05-09T20:23:38.346101Z","shell.execute_reply":"2025-05-09T20:23:38.360923Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_optimizer(model, cfg):\n    no_decay = [\"bias\", \"norm\"]\n    if cfg.no_decay:\n        optimizer_parameters = [\n            {'params': [p for n, p in model.named_parameters() \n                        if not any(nd in n for nd in no_decay)],\n             'weight_decay': cfg.decay},\n            {'params': [p for n, p in model.named_parameters() \n                        if any(nd in n for nd in no_decay)],\n             'weight_decay': 0.0},\n        ] \n    else:\n        optimizer_parameters = model.parameters()\n    optimizer = torch.optim.AdamW(optimizer_parameters, lr=cfg.lr)\n    return optimizer\n\ndef get_scheduler(optimizer, train_data_loader, cfg):\n    scheduler = lr_scheduler.OneCycleLR(\n        optimizer,\n        max_lr=cfg.max_lr,\n        epochs=cfg.num_epochs,\n        steps_per_epoch=len(train_data_loader),\n        pct_start=cfg.pct_start,\n        anneal_strategy=\"cos\",\n        final_div_factor=cfg.final_div_factor,\n    )\n    return scheduler","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:38.362340Z","iopub.execute_input":"2025-05-09T20:23:38.362565Z","iopub.status.idle":"2025-05-09T20:23:38.373145Z","shell.execute_reply.started":"2025-05-09T20:23:38.362550Z","shell.execute_reply":"2025-05-09T20:23:38.372580Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def save_checkpoint(model, fold, cfg):\n    checkpoint = {\n        'model' : model.state_dict(),\n        'fold' : fold,\n        'seed' : seed,\n        }\n    checkpoint_path = cfg.checkpoint_path\n    save_path = checkpoint_path / ('%s_%d.pth' % (cfg.fname, fold))\n    if cfg.local_rank == 0:\n        cfg.logger.info('saving %s ...' % save_path)\n    torch.save(checkpoint, save_path)\n    if cfg.local_rank == 0:\n        cfg.logger.info('done')\n\ndef load_checkpoint(fold, seed, cfg):\n    if cfg.pretrained_path:\n        checkpoint_path = cfg.pretrained_path\n    else:\n        checkpoint_path = cfg.checkpoint_path\n    save_path = checkpoint_path / ('%s_%d.pth' % (cfg.fname, fold))\n    cfg.logger.info('loading %s ...' % save_path)\n    checkpoint = torch.load(save_path, map_location='cpu')\n    model = BirdModel(cfg, pretrained=False).to(cfg.device)\n    model.load_state_dict(checkpoint['model'], strict=True)\n    model.eval()\n    cfg.logger.info('done')\n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:38.373842Z","iopub.execute_input":"2025-05-09T20:23:38.374087Z","iopub.status.idle":"2025-05-09T20:23:38.385646Z","shell.execute_reply.started":"2025-05-09T20:23:38.374061Z","shell.execute_reply":"2025-05-09T20:23:38.385066Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def resample(train_fold, cfg):\n#     new_train = []\n#     for species, df in train.groupby('species'):\n#         new_train.append(df)\n#         if len(df) < cfg.resample_train:\n#             df = df.sample(n=(cfg.resample_train - len(df)), replace=True, random_state=cfg.seed)\n#             new_train.append(df)\n#     new_train = pd.concat(new_train).reset_index(drop=True)  \n#     return new_train  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:38.389382Z","iopub.execute_input":"2025-05-09T20:23:38.389689Z","iopub.status.idle":"2025-05-09T20:23:38.400539Z","shell.execute_reply.started":"2025-05-09T20:23:38.389672Z","shell.execute_reply":"2025-05-09T20:23:38.399943Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# cfg.batch_size = cfg.train_batch_size\n# cfg.uda_data = 1\n# cfg.uda_weight = 1.0\n# cfg.uda_threshold = 0.8\n# cfg.global_step = 2\n# cfg.uda_steps = 1000\n\n# train_dataloader = get_data_loader(dataset, True, cfg)\n# ss_dataloader = get_data_loader(ss_dataset, True, cfg)\n\n# cfg.num_train_steps = cfg.num_epochs * len(train_dataloader)\n\n# train_batch = batch_to_device(next(iter(train_dataloader)), cfg.device)\n# ss_batch = batch_to_device(next(iter(ss_dataloader)), cfg.device)\n# l_batch = train_batch['image']\n# l_batch_y = train_batch['labels_proba']\n# u_batch = ss_batch['image']\n\n# import copy\n# student = copy.deepcopy(model)\n# teacher = copy.deepcopy(model)\n\n# student_optimizer = get_optimizer(student, cfg)\n# teacher_optimizer = get_optimizer(teacher, cfg)\n\n# mpl = MPL()\n# models = teacher, student\n# optimizers = teacher_optimizer, student_optimizer\n\n# all_samples = (l_batch, l_batch_y), u_batch\n# mpl.step_fn(cfg, models, optimizers, all_samples)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:38.401144Z","iopub.execute_input":"2025-05-09T20:23:38.401388Z","iopub.status.idle":"2025-05-09T20:23:38.413887Z","shell.execute_reply.started":"2025-05-09T20:23:38.401366Z","shell.execute_reply":"2025-05-09T20:23:38.413015Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scores = []\noofs = pd.DataFrame(columns=cfg.labels, index=train.index, data = 0.0)\noofs['filename'] = train.filename","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:38.414662Z","iopub.execute_input":"2025-05-09T20:23:38.414834Z","iopub.status.idle":"2025-05-09T20:23:38.460746Z","shell.execute_reply.started":"2025-05-09T20:23:38.414822Z","shell.execute_reply":"2025-05-09T20:23:38.459895Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fold = 0\ncfg.pretrained_path = Path(f'/kaggle/input/bc25-b0-attemp1/efficientvit_b0/exp_0/')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:38.461612Z","iopub.execute_input":"2025-05-09T20:23:38.461866Z","iopub.status.idle":"2025-05-09T20:23:38.465442Z","shell.execute_reply.started":"2025-05-09T20:23:38.461841Z","shell.execute_reply":"2025-05-09T20:23:38.464756Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import copy\n\nseed = cfg.seed + fold\nseed_torch(seed)\nif cfg.pl is not None:\n    pl_train = get_pl_train(cfg)\nelse:\n    pl_train = None\n    train_fold = train[train.fold != fold]\nvalid_dataset = BirdDataset(train[train.fold == fold], False, cfg)\nvalid_dataloader = get_data_loader(valid_dataset, istrain=False, cfg=cfg)\ndevice = cfg.device\n\nif cfg.pretrained_path:\n    model = load_checkpoint(fold, seed, cfg)\nelse:\n    model = BirdModel(cfg, pretrained=True).to(device)\noptimizer = get_optimizer(model, cfg)\nscheduler = None\nscaler = GradScaler()\nresult = None\n\ntrain_dataset = BirdDataset(train_fold, True, cfg)\ntrain_dataloader = get_data_loader(train_dataset, istrain=True, cfg=cfg)\nss_dataset = BirdUnlabeledDataset(ss_df, cfg)\nss_dataloader = get_data_loader(ss_dataset, True, cfg)\n\nstudent = BirdModel(cfg, pretrained=True).to(device)\nteacher = copy.deepcopy(model)\n\nstudent_optimizer = get_optimizer(student, cfg)\nteacher_optimizer = get_optimizer(teacher, cfg)\n\nstudent_scheduler = get_scheduler(student_optimizer, train_dataloader, cfg)\nteacher_scheduler = get_scheduler(teacher_optimizer, train_dataloader, cfg)\n\nfor epoch in range(cfg.num_epochs):\n    if cfg.pl is not None:\n        train_fold = sample_pl(train, fold, pl_train)\n    # if cfg.resample_train:\n    #     train_fold = resample(train_fold, cfg)\n\n    cfg.batch_size = cfg.train_batch_size\n    cfg.uda_data = 1\n    cfg.uda_weight = 1.0\n    cfg.uda_threshold = 0.8\n    cfg.global_step = 2\n    cfg.uda_steps = 1000\n    \n    cfg.num_train_steps = cfg.num_epochs * len(train_dataloader)\n\n    train_mpl_epoch(train_dataloader, ss_dataloader,\n                    (teacher, student), (teacher_optimizer, student_optimizer),\n                    (teacher_scheduler, student_scheduler), device, cfg)\n    model = copy.deepcopy(teacher)\n    if valid_dataset is not None:\n        preds, targets = valid_epoch(valid_dataloader, model, device, cfg)\n        if cfg.bce:\n            preds = expit(preds) # model uses logits\n        else:\n            preds = my_softmax(preds) # model uses logits\n        result, _ = metric(preds, np.eye(206)[targets], cfg)\n        msg = f\"seed {cfg.seed} fold {fold} epoch {epoch} metric {result:.4f}\"\n        cfg.logger.info(msg)\n    else:\n        msg = f\"seed {cfg.seed} fold {fold} epoch {epoch}\"\n        cfg.logger.info(msg)\nif cfg.local_rank == 0:\n    save_checkpoint(model, fold, cfg)\ndel model, optimizer, scheduler, scaler, train_dataloader, \nif valid_dataset is not None:\n    del valid_dataloader\ngc.collect()\ntorch.cuda.empty_cache()\nscores.append(result)\n\nfor j,c in enumerate(cfg.labels):\n    oofs.loc[train.fold == fold, c] = preds[:, j]\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:24:17.877157Z","iopub.execute_input":"2025-05-09T20:24:17.877968Z","execution_failed":"2025-05-09T20:43:05.435Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\noofs.to_csv(cfg.checkpoint_path / 'oofs.csv', index=False)\n\nnp.mean(scores)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:39.864002Z","iopub.status.idle":"2025-05-09T20:23:39.864220Z","shell.execute_reply.started":"2025-05-09T20:23:39.864113Z","shell.execute_reply":"2025-05-09T20:23:39.864123Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# class SoundScapeDataset():\n#     def __init__(self, soundscape, istrain, cfg):\n#         self.cfg = cfg\n#         self.istrain = istrain\n#         self.soundscape = soundscape\n#         self.duration = int(np.round(soundscape.shape[0] / cfg.sr))\n#         self.starts = np.arange(0, self.duration, cfg.duration)\n#         self.ends = self.starts + cfg.duration\n#         self.starts = cfg.sr * self.starts\n#         self.ends = cfg.sr * self.ends\n        \n#     def __len__(self):\n#         return len(self.starts)\n    \n#     def __getitem__(self, idx):\n#         audio = self.soundscape[self.starts[idx] : self.ends[idx]]\n#         duration = self.cfg.sr * self.cfg.duration\n#         if len(audio) < duration:\n#             pad_length = (duration - len(audio)) // 2\n#             audio = np.pad(audio, \n#                            ((pad_length, duration - len(audio) - pad_length),), \n#                            mode='constant')\n#         out = {\n#             'audio' : torch.from_numpy(audio),\n#         }\n#         return out\n\n# def predict(loader, models, cfg):\n#     for model in models:\n#         model.eval()\n#         model.zero_grad()\n#     if cfg.verbose:\n#         bar = tqdm(range(len(loader)))\n#     else:\n#         bar = range(len(loader))\n#     load_iter = iter(loader)\n#     preds_l = [[] for model in models]\n#     targets_l = []\n#     with torch.no_grad():\n#         for i, batch in zip(bar, load_iter):      \n#             input_dict = batch_to_device(batch, cfg.device)\n#             with autocast(enabled=False, device_type=\"cuda\"):\n#                 for preds, model in zip(preds_l, models):\n#                     preds.append(model(input_dict).detach().cpu()) \n#             del input_dict\n#         preds_l = [torch.cat(preds).squeeze().numpy() for preds in preds_l]\n#         return preds_l\n\n# def load_soundscape(pathname, cfg):\n#     #audio = librosa.load(pathname, sr=32000)[0].astype(np.float32)\n#     filename = pathname.split('/')[-1].split('.')[0]\n#     audio = np.load(cfg.input_path / \"birdclef-data\" / 'unlabeled_soundscapes' / 'unlabeled_soundscapes' / (filename + '.npy'))\n\n#     return audio\n\n# def predict_soundscape(filepath, models, cfg):\n#     waveform = load_soundscape(filepath, cfg)\n#     #return len(waveform) / cfg.sr\n#     #return waveform\n#     #print('duration',  len(waveform) / cfg.sr)\n#     dataset = SoundScapeDataset(waveform, False, cfg)\n#     dataloader = get_data_loader(dataset, False, cfg)\n#     preds_l = predict(dataloader, models, cfg)\n#     if cfg.bce:\n#         preds_l = [expit(preds) for preds in preds_l] # model uses logits\n#     else:\n#         preds_l = [my_softmax(preds) for preds in preds_l] # model uses logits\n#     return preds_l","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:39.865217Z","iopub.status.idle":"2025-05-09T20:23:39.865575Z","shell.execute_reply.started":"2025-05-09T20:23:39.865360Z","shell.execute_reply":"2025-05-09T20:23:39.865379Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# soundscapes = sorted(glob(str(cfg.soundscape_path / '*.ogg')))\n# soundscapes[:5]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:39.867619Z","iopub.status.idle":"2025-05-09T20:23:39.867968Z","shell.execute_reply.started":"2025-05-09T20:23:39.867791Z","shell.execute_reply":"2025-05-09T20:23:39.867809Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# cfg.verbose = False\n# cfg.workers = 0\n# cfg.valid_batch_size = 128\n\n# models = [load_checkpoint(fold, cfg).to(cfg.device)\n#           for fold in range(cfg.num_folds)\n#          ]\n# preds = {}\n# for soundscape in tqdm(soundscapes):\n#     preds[soundscape]  = predict_soundscape(soundscape, models, cfg)\n# with open(cfg.checkpoint_path / f\"pl_all.pkl\", \"wb\") as file:\n#     pkl.dump(preds, file)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T20:23:39.868777Z","iopub.status.idle":"2025-05-09T20:23:39.869106Z","shell.execute_reply.started":"2025-05-09T20:23:39.868938Z","shell.execute_reply":"2025-05-09T20:23:39.868955Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}