{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"},{"sourceId":11075844,"sourceType":"datasetVersion","datasetId":6902807},{"sourceId":189366517,"sourceType":"kernelVersion"}],"dockerImageVersionId":31011,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport librosa\nimport glob \nimport librosa.display\nimport torch\nimport torch.nn as nn\nimport os\n\nimport random\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\nfrom ast import literal_eval\nimport timm\nimport pandas.api.types\n\nimport kaggle_metric_utilities\n\nimport sklearn.metrics\nfrom sklearn.model_selection import StratifiedKFold\nfrom tqdm import tqdm\nimport gc\n\n\nfrom warnings import filterwarnings\nfilterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-10T15:13:34.204632Z","iopub.execute_input":"2025-05-10T15:13:34.204916Z","iopub.status.idle":"2025-05-10T15:13:34.209981Z","shell.execute_reply.started":"2025-05-10T15:13:34.204893Z","shell.execute_reply":"2025-05-10T15:13:34.209248Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Config\n","metadata":{}},{"cell_type":"code","source":"class Config:\n    train_dir = \"/kaggle/input/birdclef-2025/train_audio\"\n    seed = 42\n    train_csv = \"/kaggle/input/birdclef-2025/train.csv\"\n    sample_csv = \"/kaggle/input/birdclef-2025/sample_submission.csv\"\n    test_soundscapes = \"/kaggle/input/birdclef-2025/test_soundscapes.csv\"\n\n    sr = int(32e3)\n    num_classes = 206\n    n_fft = 1024\n    hop_length = 500\n\n    n_mels = 128\n    fmin = 50\n    fmax = 16000\n    power = 2\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T15:13:34.210789Z","iopub.execute_input":"2025-05-10T15:13:34.211062Z","iopub.status.idle":"2025-05-10T15:13:34.231258Z","shell.execute_reply.started":"2025-05-10T15:13:34.211046Z","shell.execute_reply":"2025-05-10T15:13:34.230642Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Helper function","metadata":{}},{"cell_type":"code","source":"def set_seed(seed : int = Config.seed) -> None:\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n\n    if(torch.cuda.is_available()):\n        torch.cuda.manual_seed(seed)\n        torch.cuda.manual_seed_all(seed)\n\n    torch.backends.cudnn.determinstic = True\n    torch.backends.cudnn.benchmark = False\n\n    print(f\"[INFO] Set seed: {seed}\")\n\nset_seed()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T15:13:34.232042Z","iopub.execute_input":"2025-05-10T15:13:34.232261Z","iopub.status.idle":"2025-05-10T15:13:34.327164Z","shell.execute_reply.started":"2025-05-10T15:13:34.232245Z","shell.execute_reply":"2025-05-10T15:13:34.326440Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Loading dataset","metadata":{}},{"cell_type":"code","source":"data_df = pd.read_csv(Config.train_csv)\n\nfor col in ('secondary_labels', 'type'):\n    data_df[col] = data_df[col].apply(lambda x : \"###\".join(literal_eval(x)))\ndata_df['filename'] = data_df['filename'].apply(lambda x: Config.train_dir + '/' + x)\n\ndata_df.sample(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T15:13:34.329290Z","iopub.execute_input":"2025-05-10T15:13:34.329492Z","iopub.status.idle":"2025-05-10T15:13:34.887557Z","shell.execute_reply.started":"2025-05-10T15:13:34.329476Z","shell.execute_reply":"2025-05-10T15:13:34.886928Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def rating_to_weight(rating):\n    if rating in {5.0, 4.5}:\n        return 2.5\n    elif rating in {4.0, 0}:\n        return 2.0\n    elif rating in {3.0, 3.5}:\n        return 1.5\n    else:\n        return 1.0\n\ndata_df[\"weight\"] = data_df[\"rating\"].apply(rating_to_weight)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T15:13:34.888186Z","iopub.execute_input":"2025-05-10T15:13:34.888438Z","iopub.status.idle":"2025-05-10T15:13:34.901686Z","shell.execute_reply.started":"2025-05-10T15:13:34.888393Z","shell.execute_reply":"2025-05-10T15:13:34.901063Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Null check\n# data_df.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T15:13:34.902337Z","iopub.execute_input":"2025-05-10T15:13:34.902611Z","iopub.status.idle":"2025-05-10T15:13:34.914474Z","shell.execute_reply.started":"2025-05-10T15:13:34.902586Z","shell.execute_reply":"2025-05-10T15:13:34.913862Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Distribution of the raitings\n# plt.figure(figsize = (15,3))\n# sns.histplot(data_df, x='rating')\n# plt.xticks(np.arange(0, 5.5, 0.5))\n# plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T15:13:34.915187Z","iopub.execute_input":"2025-05-10T15:13:34.915663Z","iopub.status.idle":"2025-05-10T15:13:34.927350Z","shell.execute_reply.started":"2025-05-10T15:13:34.915639Z","shell.execute_reply":"2025-05-10T15:13:34.926890Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Distribution of primary label\n# plt.figure(figsize = (15,3))\n# sns.countplot(data_df, x='primary_label')\n# plt.xticks(rotation=90)\n# plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T15:13:34.928156Z","iopub.execute_input":"2025-05-10T15:13:34.928411Z","iopub.status.idle":"2025-05-10T15:13:34.938574Z","shell.execute_reply.started":"2025-05-10T15:13:34.928375Z","shell.execute_reply":"2025-05-10T15:13:34.937940Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Distribution of primary label with different ratings\n# for r in range(0,6):\n#     plt.figure(figsize = (20,3))\n#     sns.countplot(data_df[data_df['rating'] == float(r)], x = 'primary_label')\n#     plt.title(f\"Rating {r}\")\n#     plt.xticks(rotation=90)\n#     plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T15:13:34.939189Z","iopub.execute_input":"2025-05-10T15:13:34.939415Z","iopub.status.idle":"2025-05-10T15:13:34.950345Z","shell.execute_reply.started":"2025-05-10T15:13:34.939381Z","shell.execute_reply":"2025-05-10T15:13:34.949785Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Statistic of audio durations\n# durations = []\n# for idx, row in data_df.sample(100).iterrows() :\n#     data, _ = librosa.load(row['filename'], sr = Config.sr)\n#     durations.append(librosa.get_duration(y = data, sr = Config.sr))\n\n# d_df = pd.DataFrame(columns = [\"durations\"], data = durations)\n# plt.figure(figsize = (10, 5))\n# plt.title(\"Distribution of audio lengths\")\n# sns.histplot(d_df, x = \"durations\", bins = 100)\n# plt.show();\n\n# d_df.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T15:13:34.951019Z","iopub.execute_input":"2025-05-10T15:13:34.951259Z","iopub.status.idle":"2025-05-10T15:13:34.960863Z","shell.execute_reply.started":"2025-05-10T15:13:34.951244Z","shell.execute_reply":"2025-05-10T15:13:34.960305Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Check out spectrograms","metadata":{}},{"cell_type":"code","source":"# def show_signal(file_path):\n#     class_, collector = file_path.split(\"/\")[-2:]\n#     y, sr = librosa.load(file_path, sr=Config.sr)\n#     fig, axes = plt.subplots(2, 2, figsize=(20, 10))\n#     fig.suptitle(f\"Class: {class_} | Collector: {collector}\", fontsize=16)\n\n#     # Plotting raw signal\n#     librosa.display.waveshow(y, sr=sr, ax=axes[0, 0])\n#     axes[0, 0].set_title(\"Raw signal\")\n    \n\n\n#     # Plotting fourier transformed signal\n#     ft = np.abs(librosa.stft(\n#         y,\n#         n_fft = Config.n_fft,\n#         hop_length = Config.hop_length\n#     ))\n#     im1 = librosa.display.specshow(\n#         ft,\n#         sr = sr,\n#         x_axis = 'time',\n#         y_axis = 'linear',\n#         ax = axes[0, 1]\n#     )\n#     fig.colorbar(im1, ax = axes[0, 1])\n#     axes[0, 1].set_title(\"Spectrogram\")\n\n#     # Plotting log scale fourier transformed signal\n#     ft_db = librosa.amplitude_to_db(ft, ref = np.max)\n#     im2 = librosa.display.specshow(\n#     ft_db,\n#     sr=sr,\n#     x_axis='time',\n#     y_axis='log',\n#     ax=axes[1, 0]\n#     )\n\n#     fig.colorbar(im2, ax=axes[1, 0])\n#     axes[1, 0].set_title(\"Log Scaled spectrogram\")\n\n\n#     # Pplotting mel spectrograms\n#     mel_sp = librosa.feature.melspectrogram(\n#         y = y,\n#         sr = Config.sr,\n#         fmin = Config.fmin,\n#         fmax = Config.fmax,\n#         power = Config.power,\n#         n_mels = Config.n_mels,\n#     )\n#     mel_sp = librosa.power_to_db(mel_sp, ref=np.max)\n\n#     im3 = librosa.display.specshow(\n#         mel_sp,\n#         y_axis='mel',\n#         sr=Config.sr,\n#         fmin=Config.fmin,\n#         x_axis='time',\n#         fmax=Config.fmax,\n#         ax=axes[1, 1]\n#     )\n#     fig.colorbar(im3, ax=axes[1, 1])\n#     axes[1, 1].set_title(\"Mel Spectrogram\")\n\n\n#     plt.show()\n\n\n# show_signal(data_df['filename'].values[0])\n\n# data_df['filename'].values[0]\n# data_df['filename'].values[0].split(\"/\")[-2:]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T15:13:34.961573Z","iopub.execute_input":"2025-05-10T15:13:34.961802Z","iopub.status.idle":"2025-05-10T15:13:34.972066Z","shell.execute_reply.started":"2025-05-10T15:13:34.961784Z","shell.execute_reply":"2025-05-10T15:13:34.971369Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# for idx, row in data_df.sample(10).iterrows(): show_signal(row['filename'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T15:13:34.972820Z","iopub.execute_input":"2025-05-10T15:13:34.973007Z","iopub.status.idle":"2025-05-10T15:13:34.985842Z","shell.execute_reply.started":"2025-05-10T15:13:34.972992Z","shell.execute_reply":"2025-05-10T15:13:34.985173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Dataset\n","metadata":{}},{"cell_type":"code","source":"label_mapper = {\n    label: idx \n    for idx, label in enumerate(sorted(data_df['primary_label'].unique()))\n}\n\nrev_mapper = {\n    idx: label \n    for label, idx in label_mapper.items()\n}\n\nclass BirdClefDataset(torch.utils.data.Dataset):\n    def __init__(self, df, mode=\"train\"):\n        import pickle\n\n        self.df = df\n        self.mode = mode\n\n        # Load voice_data từ file\n        with open(\"/kaggle/input/bc25-separation-voice-from-data-by-silero-vad/train_voice_data.pkl\", \"rb\") as f:\n            self.voice_data = pickle.load(f)\n\n    def __len__(self):\n        return len(self.df)\n\n    def crop_voice(self, y, audio_path, sr):\n        if audio_path in self.voice_data:\n            mask = np.ones(len(y), dtype=bool)\n            for seg in self.voice_data[audio_path]:\n                start = int(seg['start'] * sr)\n                end = int(seg['end'] * sr)\n                mask[start:end] = False\n            y = y[mask]\n        return y\n\n\n    def process(self, audio_path):\n        data, _ = librosa.load(audio_path, sr=Config.sr)\n        data = self.crop_voice(data, audio_path, Config.sr)\n        data = data * 1024\n        chunk_duration = 10\n        min_len = chunk_duration * Config.sr\n\n        # If the audio signal is less than min_len\n        if len(data) < min_len:\n            cnt = int(np.ceil(min_len / len(data)))\n            data = np.tile(data, cnt)\n        \n        # Making the data length divisible by min_len\n        leftover = len(data) % min_len\n        if leftover > 0:\n            front_crop = leftover // 2\n            back_crop = leftover - front_crop\n            data = data[front_crop : len(data) - back_crop]\n        \n        # Truncating the signal to min_len\n        data = data[:min_len]\n        data = data.reshape(-1, min_len)\n        # Creating mel spectrogram\n        mel_sp = librosa.feature.melspectrogram(\n            y = data,\n            sr = Config.sr,\n            fmin = Config.fmin,\n            fmax = Config.fmax,\n            power = Config.power,\n            n_mels = Config.n_mels,\n            n_fft = Config.n_fft,\n            hop_length = Config.hop_length\n        )\n\n        mel_sp = librosa.power_to_db(mel_sp, ref = 1)\n\n        # Normalizing the features\n        eps = 1e-12\n        mel_sp = (mel_sp - mel_sp.min()) / (mel_sp.max() - mel_sp.min() + eps)\n\n        mel_sp = mel_sp[:, :, :640]\n        return mel_sp\n\n    def __getitem__(self, idx):\n        row = self.df.loc[idx, :]\n        filename = row['filename']\n\n        # TODO: Chuyển đổi sang spectrogram\n        x = self.process(filename)\n\n        if self.mode == \"train\":\n            y = label_mapper[row['primary_label']]\n            w = row['weight']\n            return x, y, w\n\n        return x\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T15:13:34.986619Z","iopub.execute_input":"2025-05-10T15:13:34.986945Z","iopub.status.idle":"2025-05-10T15:13:35.002628Z","shell.execute_reply.started":"2025-05-10T15:13:34.986919Z","shell.execute_reply":"2025-05-10T15:13:35.002055Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# mel_sp = BirdClefDataset(data_df).process(data_df['filename'].values[0])\n# print(mel_sp.shape)\n\n# Transform from C, H, W -> H, W, C\n# plt.imshow(mel_sp.reshape(128, 640, -1))\n# plt.show();\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T15:13:35.003706Z","iopub.execute_input":"2025-05-10T15:13:35.004235Z","iopub.status.idle":"2025-05-10T15:13:35.015769Z","shell.execute_reply.started":"2025-05-10T15:13:35.004217Z","shell.execute_reply":"2025-05-10T15:13:35.014973Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":" # Định nghĩa Model\nclass Model(nn.Module):\n    def __init__(self, model_name: str):\n        super().__init__()\n        self.base_model = timm.create_model(\n            model_name=model_name,\n            num_classes=Config.num_classes,\n            pretrained=False,\n            in_chans=1\n        )\n\n    def forward(self, x):\n        return self.base_model(x)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T15:13:35.016504Z","iopub.execute_input":"2025-05-10T15:13:35.016722Z","iopub.status.idle":"2025-05-10T15:13:35.028073Z","shell.execute_reply.started":"2025-05-10T15:13:35.016706Z","shell.execute_reply":"2025-05-10T15:13:35.027413Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Checking Dataset and Model Class\n# tmp_ds = BirdClefDataset(data_df.sample(10).reset_index())\n# model = Model(\"tf_efficientnet_b0\")\n\n# for i in range(10):\n#     x, y = tmp_ds[i]\n\n#     model.eval()\n\n#     preds = model(torch.tensor([x]))\n#     preds = torch.argmax(torch.softmax(preds, dim=1), dim=1).item()\n\n#     plt.imshow(x.reshape(128, 640, -1))\n#     plt.title(f\"Label: {rev_mapper[y]} | {x.shape} | {rev_mapper[preds]}\")\n#     plt.show()\n\n# del model\n# del tmp_ds\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T15:13:35.030264Z","iopub.execute_input":"2025-05-10T15:13:35.030472Z","iopub.status.idle":"2025-05-10T15:13:35.039817Z","shell.execute_reply.started":"2025-05-10T15:13:35.030457Z","shell.execute_reply":"2025-05-10T15:13:35.039214Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"class ParticipantVisibleError(Exception):\n    pass\n\ndef score(solution: pd.DataFrame, submission: pd.DataFrame, row_id_column_name: str) -> float:\n    '''\n    Version of macro-averaged ROC-AUC score that ignores all classes that have no true positive labels.\n    '''\n    del solution[row_id_column_name]\n    del submission[row_id_column_name]\n\n    if not pandas.api.types.is_numeric_dtype(submission.values):\n        bad_dtypes = {x: submission[x].dtype for x in submission.columns if not pandas.api.types.is_numeric_dtype(submission[x])}\n        raise ParticipantVisibleError(f'Invalid submission data types found: {bad_dtypes}')\n\n    solution_sums = solution.sum(axis=0)\n    scored_columns = list(solution_sums[solution_sums > 0].index.values)\n    assert len(scored_columns) > 0\n\n    return kaggle_metric_utilities.safe_call_score(\n        sklearn.metrics.roc_auc_score,\n        solution[scored_columns].values,\n        submission[scored_columns].values,\n        average='macro'\n    )\n\ndef cal_score(labels, preds):\n    labels = np.concatenate(labels)\n    preds = np.concatenate(preds)\n\n     # Lọc những dòng có NaN trong label hoặc pred\n    valid_mask = ~np.isnan(labels).any(axis=1) & ~np.isnan(preds).any(axis=1)\n    labels = labels[valid_mask]\n    preds = preds[valid_mask]\n    \n    labels_df = pd.DataFrame(labels > 0.5, columns = list(label_mapper.keys()))\n    pred_df = pd.DataFrame(preds, columns = list(label_mapper.keys()))\n\n    labels_df['id'] = np.arange(len(labels_df))\n    pred_df['id'] = np.arange(len(pred_df))\n\n    return score(labels_df, pred_df, row_id_column_name = 'id')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T15:13:35.040497Z","iopub.execute_input":"2025-05-10T15:13:35.040710Z","iopub.status.idle":"2025-05-10T15:13:35.053952Z","shell.execute_reply.started":"2025-05-10T15:13:35.040691Z","shell.execute_reply":"2025-05-10T15:13:35.053300Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Training Configs\nepochs = 5\nnum_folds = 5\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nlr = 1e-4\ntarget_col = 'primary_label'\ndf = data_df\n\nskf = StratifiedKFold(\n    n_splits = num_folds,\n    shuffle = True,\n    random_state = Config.seed\n)\n\ndf['kfold'] = -1\n\nfor fold, (train_idx, val_idx) in enumerate(skf.split(X = df, y = df[target_col])):\n    df.loc[val_idx, 'kfold'] = fold\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T15:13:35.054657Z","iopub.execute_input":"2025-05-10T15:13:35.054846Z","iopub.status.idle":"2025-05-10T15:13:35.095674Z","shell.execute_reply.started":"2025-05-10T15:13:35.054832Z","shell.execute_reply":"2025-05-10T15:13:35.095028Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_choices = {\n    0: \"tf_efficientnet_b0\",\n    1: \"tf_efficientnet_b1\",\n    2: \"regnety_008\",\n    3: \"mobilenetv2_100\",\n    4: \"efficientvit_b0\"\n}\n\nfor fold in range(num_folds):\n    print(f\"\\n[INFO] Fold {fold} | Model: {model_choices[fold]}\")\n\n    train_df = df[df['kfold'] != fold].reset_index(drop=True)\n    val_df = df[df['kfold'] == fold].reset_index(drop=True)\n\n    train_ds = BirdClefDataset(train_df)\n    val_ds = BirdClefDataset(val_df)\n\n    train_loader = torch.utils.data.DataLoader(\n        train_ds,\n        batch_size=16,\n        shuffle=True,\n        num_workers=8,\n        drop_last=True\n    )\n\n    val_loader = torch.utils.data.DataLoader(\n        val_ds,\n        batch_size=32,\n        shuffle=False,\n        num_workers=8,\n        drop_last=False\n    )\n\n    # Khởi tạo model theo tên được chọn cho fold hiện tại\n    model_name = model_choices[fold]\n    model = Model(model_name=model_name).to(device)\n\n    criterion = nn.CrossEntropyLoss()\n    optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n\n    best_auc = 0\n\n    for epoch in range(epochs):\n        model.train()\n        pred_train, label_train = [], []\n        running_loss = 0.0\n\n        for (x, y, w) in tqdm(train_loader, desc=\"Training\"):\n            x, y, w = x.to(device), y.to(device), w.to(device)\n            y_one_hot = nn.functional.one_hot(y, num_classes=Config.num_classes).float()\n        \n            optimizer.zero_grad()\n            outputs = model(x)\n        \n           \n            losses = criterion(outputs, y) \n            weighted_loss = (losses * w).mean()  \n        \n            weighted_loss.backward()\n            optimizer.step()\n        \n            running_loss += weighted_loss.item()\n            probs = torch.softmax(outputs, dim=1)\n            pred_train.append(probs.detach().cpu().numpy())\n            label_train.append(y_one_hot.detach().cpu().numpy())\n\n\n        # Validation\n        model.eval()\n        pred_val, label_val = [], []\n        running_val_loss = 0.0\n\n        with torch.no_grad():\n            for (x, y, *_) in tqdm(val_loader, desc=\"Validation\"):\n                x, y = x.to(device), y.to(device)\n                y_one_hot = nn.functional.one_hot(y, num_classes=Config.num_classes).float()\n                outputs = model(x)\n                loss = criterion(outputs, y)\n                running_val_loss += loss.item()\n                probs = torch.softmax(outputs, dim=1)\n                pred_val.append(probs.detach().cpu().numpy())\n                label_val.append(y_one_hot.detach().cpu().numpy())\n\n        # AUC and Loss\n        auc_train = cal_score(label_train, pred_train)\n        auc_val = cal_score(label_val, pred_val)\n\n        avg_train_loss = running_loss / len(train_loader)\n        avg_val_loss = running_val_loss / len(val_loader)\n\n        print(f\"[Fold]: {fold} | [EPOCH]: {epoch} | Loss: {avg_train_loss:.4f} | Val_Loss: {avg_val_loss:.4f}\")\n        print(f\"[Fold]: {fold} | [EPOCH]: {epoch} | Train AUC: {auc_train:.4f} | Val AUC: {auc_val:.4f}\")\n\n        if best_auc <= auc_val:\n            best_auc = auc_val\n            filename = f\"fold_{fold}_{model_name}_epoch_{epoch}_val_auc_{float(auc_val):.4f}_val_loss_{float(avg_val_loss):.4f}.pth\"\n            torch.save(model.state_dict(), filename)\n            print(f\"[INFO] Model saved to: {filename}\")\n\n    del train_df, val_df, train_ds, val_ds, train_loader, val_loader, model, criterion, optimizer\n    gc.collect()\n    torch.cuda.empty_cache()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T15:13:35.096368Z","iopub.execute_input":"2025-05-10T15:13:35.096658Z","iopub.status.idle":"2025-05-10T19:35:37.961912Z","shell.execute_reply.started":"2025-05-10T15:13:35.096643Z","shell.execute_reply":"2025-05-10T19:35:37.961293Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(label_train), label_train[0].shape, len(pred_train), pred_train[0].shape\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T19:35:37.963444Z","iopub.execute_input":"2025-05-10T19:35:37.963682Z","iopub.status.idle":"2025-05-10T19:35:37.968890Z","shell.execute_reply.started":"2025-05-10T19:35:37.963661Z","shell.execute_reply":"2025-05-10T19:35:37.968355Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"np.concatenate(pred_train).shape\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T19:35:37.969531Z","iopub.execute_input":"2025-05-10T19:35:37.969769Z","iopub.status.idle":"2025-05-10T19:35:37.993704Z","shell.execute_reply.started":"2025-05-10T19:35:37.969748Z","shell.execute_reply":"2025-05-10T19:35:37.992885Z"}},"outputs":[],"execution_count":null}]}