{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import Module","metadata":{}},{"cell_type":"code","source":"import os\nimport time\nimport random\nimport gc\nimport warnings\nimport ast\nfrom pathlib import Path\nimport numpy as np\nimport pandas as pd\nimport librosa\nimport librosa.display\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score, precision_score, recall_score, f1_score\nfrom haversine import haversine\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchaudio.transforms as T\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim import AdamW, lr_scheduler\nfrom torch.amp import autocast, GradScaler\nimport timm\nimport joblib\n\n# Suppress warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Configuration","metadata":{}},{"cell_type":"code","source":"class Config:\n    OUTPUT_DIR = '/kaggle/working/'\n    DATA_ROOT = '/kaggle/input/birdclef-2025'\n    AUDIO_PATH = Path(DATA_ROOT) / 'train_audio'\n    SOUNDSCAPE_PATH = Path(DATA_ROOT) / 'train_soundscapes'\n    FS = 32000\n    N_FFT = 1024\n    HOP_LENGTH = 512\n    N_MELS = 128\n    FMIN = 50\n    FMAX = 14000\n    N_MFCC = 13\n    TARGET_DURATION = 5.0\n    TARGET_SHAPE = (256, 256)\n    N_MAX = None\n    EPOCHS = 10\n    BATCH_SIZE = 32\n    LEARNING_RATE = 0.001\n    NUM_FOLDS = 5\n    SEED = 2025\n    NOISE_FACTOR = 0.005\n    TIME_STRETCH_MIN = 0.8\n    TIME_STRETCH_MAX = 1.2\n    PITCH_SHIFT_MIN = -2\n    PITCH_SHIFT_MAX = 2\n    FREQ_MASK_PARAM = 15\n    TIME_MASK_PARAM = 20\n    CHECKPOINT_PATH = Path(OUTPUT_DIR) / 'checkpoints'\n    DURATION_CACHE = Path(OUTPUT_DIR) / 'durations.pkl'\n    FEATURE_CACHE = Path(OUTPUT_DIR) / 'features'\n\n# Initialize config\nconfig = Config()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Set random seed\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\nseed_everything(config.SEED)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load data\ntaxonomy_df = pd.read_csv(f'{config.DATA_ROOT}/taxonomy.csv')\ntrain_df = pd.read_csv(f'{config.DATA_ROOT}/train.csv')\nprint(\"Data loaded successfully\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create mappings\nspecies_class_map = dict(zip(taxonomy_df['primary_label'], taxonomy_df['class_name']))\nlabel_list = sorted(train_df['primary_label'].unique())\nlabel2id = {label: idx for idx, label in enumerate(label_list)}\nid2label = {idx: label for label, idx in label2id.items()}","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Prepare working DataFrame\nworking_df = train_df[['primary_label', 'rating', 'filename', 'latitude', 'longitude']].copy()\nworking_df['target'] = working_df['primary_label'].map(label2id)\nworking_df['filepath'] = str(config.AUDIO_PATH) + '/' + working_df['filename']\nworking_df['samplename'] = working_df['filename'].map(lambda x: x.split('/')[0] + '-' + x.split('/')[-1].split('.')[0])\nworking_df['class'] = working_df['primary_label'].map(lambda x: species_class_map.get(x, 'Unknown'))\ntotal_samples = min(len(working_df), config.N_MAX or len(working_df))\nworking_df = working_df.iloc[:total_samples].reset_index(drop=True)\nprint(f\"Total samples to process: {total_samples}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Parse secondary labels\ndef parse_secondary_labels(label_str):\n    if pd.isna(label_str):\n        return []\n    try:\n        return ast.literal_eval(label_str)\n    except:\n        return []\ntrain_df['parsed_secondary_labels'] = train_df['secondary_labels'].apply(parse_secondary_labels)\nworking_df = working_df.merge(train_df[['filename', 'parsed_secondary_labels']], on='filename', how='left')\nworking_df.rename(columns={'parsed_secondary_labels': 'secondary_labels'}, inplace=True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Add geo feature and rating weight\nworking_df['geo_distance'] = working_df.apply(lambda row: haversine((row['latitude'], row['longitude']), (0, 0)) / 1000, axis=1)\nworking_df['rating_weight'] = working_df['rating'].apply(lambda x: x / 5.0 if x > 0 else 0.5)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Audio analysis","metadata":{}},{"cell_type":"code","source":"def get_audio_duration(file_path, sr=config.FS):\n    try:\n        audio, _ = librosa.load(file_path, sr=sr, mono=True)\n        return len(audio) / sr\n    except Exception as e:\n        print(f\"Error loading {file_path}: {e}\")\n        return np.nan\n\nif os.path.exists(config.DURATION_CACHE):\n    print(\"Loading cached durations\")\n    durations = joblib.load(config.DURATION_CACHE)\nelse:\n    print(\"Calculating durations\")\n    durations = [get_audio_duration(fp) for fp in tqdm(working_df['filepath'].tolist(), desc=\"Calculating durations\")]\n    joblib.dump(durations, config.DURATION_CACHE)\nworking_df['duration'] = durations\nprint(f\"Duration stats - Mean: {np.nanmean(working_df['duration']):.2f}s, Median: {np.nanmedian(working_df['duration']):.2f}s\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# EDA: Visualizations","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nsns.countplot(data=working_df, x='class', order=working_df['class'].value_counts().index)\nplt.title('Sample Distribution by Class')\nplt.xlabel('Class')\nplt.ylabel('Number of Samples')\nplt.xticks(rotation=45)\nplt.tight_layout()\nplt.savefig(f'{config.OUTPUT_DIR}/class_distribution.png')\nplt.close()\n\nplt.figure(figsize=(10, 6))\nsns.countplot(x='rating', data=working_df)\nplt.title('Distribution of Ratings')\nplt.xlabel('Rating')\nplt.ylabel('Count')\nplt.savefig(f'{config.OUTPUT_DIR}/rating_distribution.png')\nplt.close()\nprint(f\"Average rating: {working_df['rating'].mean():.2f}, Median rating: {working_df['rating'].median():.2f}\")\n\nplt.figure(figsize=(12, 6))\nplt.hist(working_df['duration'].dropna(), bins=50, color='skyblue')\nplt.title('Distribution of Audio Durations')\nplt.xlabel('Duration (seconds)')\nplt.ylabel('Count')\nplt.savefig(f'{config.OUTPUT_DIR}/duration_distribution.png')\nplt.close()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Spectrogram samples","metadata":{}},{"cell_type":"code","source":"classes = ['Aves', 'Amphibia', 'Mammalia', 'Insecta']\nplt.figure(figsize=(15, 10))\nfor i, cls in enumerate(classes, 1):\n    sample = working_df[working_df['class'] == cls].sample(n=1)\n    audio, _ = librosa.load(sample['filepath'].iloc[0], sr=config.FS)\n    mel = audio2melspec(audio)\n    plt.subplot(2, 2, i)\n    librosa.display.specshow(mel, sr=config.FS, x_axis='time', y_axis='mel')\n    plt.title(f'Spectrogram - {cls}')\n    plt.colorbar(format='%+2.0f dB')\nplt.tight_layout()\nplt.savefig(f'{config.OUTPUT_DIR}/spectrogram_samples.png')\nplt.close()\nprint(\"Visualizations saved as spectrogram_samples.png\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature extraction (Mel Spectrogram and MFCC)","metadata":{}},{"cell_type":"code","source":"def prepare_audio(audio, target_len):\n    while len(audio) < target_len:\n        audio = np.concatenate([audio, audio])\n    start = max(0, len(audio) // 2 - target_len // 2)\n    audio = audio[start:start + target_len]\n    if len(audio) < target_len:\n        audio = np.pad(audio, (0, target_len - len(audio)), mode='constant')\n    return audio\n\ndef audio2melspec(audio_data):\n    if np.isnan(audio_data).any():\n        mean_val = np.nanmean(audio_data)\n        audio_data = np.nan_to_num(audio_data, nan=mean_val)\n    mel = librosa.feature.melspectrogram(\n        y=audio_data, sr=config.FS, n_fft=config.N_FFT, hop_length=config.HOP_LENGTH,\n        n_mels=config.N_MELS, fmin=config.FMIN, fmax=config.FMAX, power=2.0\n    )\n    mel_db = librosa.power_to_db(mel, ref=np.max)\n    mel_db = (mel_db - mel_db.min()) / (mel_db.max() - mel_db.min() + 1e-8)\n    return mel_db\n\ndef extract_mfcc(audio_data):\n    if np.isnan(audio_data).any():\n        mean_val = np.nanmean(audio_data)\n        audio_data = np.nan_to_num(audio_data, nan=mean_val)\n    mfcc = librosa.feature.mfcc(y=audio_data, sr=config.FS, n_mfcc=config.N_MFCC)\n    mfcc = (mfcc - mfcc.min()) / (mfcc.max() - mfcc.min() + 1e-8)\n    return mfcc","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Precompute features","metadata":{}},{"cell_type":"code","source":"os.makedirs(config.FEATURE_CACHE, exist_ok=True)\nfeature_cache_files = [config.FEATURE_CACHE / f'{row.samplename}.npz' for _, row in working_df.iterrows()]\nmissing_files = [f for f in feature_cache_files if not f.exists()]\n\nif missing_files:\n    print(f\"Precomputing features for {len(missing_files)} samples\")\n    for i in tqdm(range(0, total_samples, config.BATCH_SIZE), desc=\"Feature extraction\"):\n        batch_df = working_df.iloc[i:i + config.BATCH_SIZE]\n        for _, row in batch_df.iterrows():\n            cache_file = config.FEATURE_CACHE / f'{row.samplename}.npz'\n            if cache_file.exists():\n                continue\n            try:\n                audio, _ = librosa.load(row.filepath, sr=config.FS, mono=True)\n                audio = prepare_audio(audio, int(config.TARGET_DURATION * config.FS))\n                mel = audio2melspec(audio)\n                mfcc = extract_mfcc(audio)\n                if mel.shape != config.TARGET_SHAPE:\n                    mel = cv2.resize(mel, config.TARGET_SHAPE, interpolation=cv2.INTER_LINEAR)\n                if mfcc.shape != config.TARGET_SHAPE:\n                    mfcc = cv2.resize(mfcc, config.TARGET_SHAPE, interpolation=cv2.INTER_LINEAR)\n                np.savez_compressed(cache_file, mel=mel.astype(np.float32), mfcc=mfcc.astype(np.float32))\n            except Exception as e:\n                print(f\"Error processing {row.filepath}: {e}\")\n        gc.collect()\nprint(\"Feature extraction completed and cached\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Dataset class with augmentation","metadata":{}},{"cell_type":"code","source":"class BirdDataset(Dataset):\n    def __init__(self, df, is_train=True):\n        self.df = df\n        self.is_train = is_train\n        self.spec_augment = T.FrequencyMasking(freq_mask_param=config.FREQ_MASK_PARAM)\n        self.time_augment = T.TimeMasking(time_mask_param=config.TIME_MASK_PARAM)\n\n    def augment_audio(self, audio):\n        rate = random.uniform(config.TIME_STRETCH_MIN, config.TIME_STRETCH_MAX)\n        audio = librosa.effects.time_stretch(audio, rate=rate)\n        shift = random.uniform(config.PITCH_SHIFT_MIN, config.PITCH_SHIFT_MAX)\n        audio = librosa.effects.pitch_shift(audio, sr=config.FS, n_steps=shift)\n        noise = np.random.normal(0, config.NOISE_FACTOR, len(audio))\n        audio += noise\n        return audio\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        cache_file = config.FEATURE_CACHE / f'{row.samplename}.npz'\n        try:\n            data = np.load(cache_file, mmap_mode='r')\n            mel = data['mel']\n            mfcc = data['mfcc']\n        except Exception as e:\n            print(f\"Error loading {cache_file}: {e}\")\n            audio, _ = librosa.load(row.filepath, sr=config.FS, mono=True)\n            audio = prepare_audio(audio, int(config.TARGET_DURATION * config.FS))\n            mel = audio2melspec(audio)\n            mfcc = extract_mfcc(audio)\n            if mel.shape != config.TARGET_SHAPE:\n                mel = cv2.resize(mel, config.TARGET_SHAPE, interpolation=cv2.INTER_LINEAR)\n            if mfcc.shape != config.TARGET_SHAPE:\n                mfcc = cv2.resize(mfcc, config.TARGET_SHAPE, interpolation=cv2.INTER_LINEAR)\n\n        if self.is_train:\n            audio, _ = librosa.load(row.filepath, sr=config.FS, mono=True)\n            audio = self.augment_audio(audio)\n            audio = prepare_audio(audio, int(config.TARGET_DURATION * config.FS))\n            mel = audio2melspec(audio)\n            mfcc = extract_mfcc(audio)\n            if mel.shape != config.TARGET_SHAPE:\n                mel = cv2.resize(mel, config.TARGET_SHAPE, interpolation=cv2.INTER_LINEAR)\n            if mfcc.shape != config.TARGET_SHAPE:\n                mfcc = cv2.resize(mfcc, config.TARGET_SHAPE, interpolation=cv2.INTER_LINEAR)\n\n        mel = torch.tensor(mel, dtype=torch.float32).unsqueeze(0)  # [1, H, W]\n        mfcc = torch.tensor(mfcc, dtype=torch.float32).unsqueeze(0)  # [1, H, W]\n        mel = mel.permute(1, 2, 0).repeat(1, 1, 3).permute(2, 0, 1)  # [3, H, W]\n        mfcc = mfcc.permute(1, 2, 0).repeat(1, 1, 3).permute(2, 0, 1)  # [3, H, W]\n\n        if self.is_train:\n            mel = self.spec_augment(mel)\n            mel = self.time_augment(mel)\n            mfcc = self.spec_augment(mfcc)\n            mfcc = self.time_augment(mfcc)\n\n        if mel.shape != (3, config.TARGET_SHAPE[0], config.TARGET_SHAPE[1]):\n            raise ValueError(f\"Expected mel shape {(3, config.TARGET_SHAPE[0], config.TARGET_SHAPE[1])}, got {mel.shape}\")\n        if mfcc.shape != (3, config.TARGET_SHAPE[0], config.TARGET_SHAPE[1]):\n            raise ValueError(f\"Expected mfcc shape {(3, config.TARGET_SHAPE[0], config.TARGET_SHAPE[1])}, got {mfcc.shape}\")\n\n        target = np.zeros(len(label_list), dtype=np.float32)\n        target[label2id[row['primary_label']]] = 1.0\n        return {'mel': mel, 'mfcc': mfcc, 'target': target, 'geo_feature': row['geo_distance'], 'rating_weight': row['rating_weight']}","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Multi-Input CRNN Model","metadata":{}},{"cell_type":"code","source":"class BirdCRNN(nn.Module):\n    def __init__(self, num_classes):\n        super(BirdCRNN, self).__init__()\n        self.cnn_mel = timm.create_model('tf_efficientnetv2_b0.in1k', pretrained=True, num_classes=0, global_pool='', drop_rate=0.1)\n        self.cnn_mfcc = timm.create_model('tf_efficientnetv2_b0.in1k', pretrained=True, num_classes=0, global_pool='', drop_rate=0.1)\n        self.pool = nn.AdaptiveAvgPool2d((1, 1))\n        self.lstm = nn.LSTM(input_size=self.cnn_mel.num_features + self.cnn_mfcc.num_features, hidden_size=128, num_layers=2, batch_first=True)\n        self.geo_fc = nn.Linear(1, 64)\n        self.rating_fc = nn.Linear(1, 64)\n        self.fc = nn.Linear(128 + 64 + 64, num_classes)\n\n    def forward(self, mel, mfcc, geo_feature, rating_weight):\n        mel = self.cnn_mel(mel)\n        if mel.dim() != 4:\n            raise ValueError(f\"Expected 4D tensor for mel, got {mel.dim()}D tensor with shape {mel.shape}\")\n        mel = self.pool(mel).view(mel.size(0), -1)\n\n        mfcc = self.cnn_mfcc(mfcc)\n        if mfcc.dim() != 4:\n            raise ValueError(f\"Expected 4D tensor for mfcc, got {mfcc.dim()}D tensor with shape {mfcc.shape}\")\n        mfcc = self.pool(mfcc).view(mfcc.size(0), -1)\n\n        x = torch.cat([mel, mfcc], dim=1).unsqueeze(1)\n        x, _ = self.lstm(x)\n        x = x[:, -1, :]\n        geo_emb = F.relu(self.geo_fc(geo_feature.unsqueeze(-1)))\n        rating_emb = F.relu(self.rating_fc(rating_weight.unsqueeze(-1)))\n        x = torch.cat([x, geo_emb, rating_emb], dim=1)\n        return self.fc(x)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training and Evaluation","metadata":{}},{"cell_type":"code","source":"def get_dataloader(df, is_train=True):\n    dataset = BirdDataset(df, is_train)\n    return DataLoader(dataset, batch_size=config.BATCH_SIZE, shuffle=is_train, num_workers=4, pin_memory=True)\n\ndef train_epoch(model, dataloader, optimizer, scheduler, scaler, device):\n    model.train()\n    loss_l = []\n    for batch in tqdm(dataloader, desc=\"Training\"):\n        mel = batch['mel'].to(device, non_blocking=True)\n        mfcc = batch['mfcc'].to(device, non_blocking=True)\n        target = batch['target'].to(device, non_blocking=True)\n        geo_feature = torch.tensor(batch['geo_feature'], dtype=torch.float32).to(device, non_blocking=True)\n        rating_weight = torch.tensor(batch['rating_weight'], dtype=torch.float32).to(device, non_blocking=True)\n        with autocast(device_type=device.type):\n            pred = model(mel, mfcc, geo_feature, rating_weight)\n            class_counts = working_df['primary_label'].value_counts().sort_index()\n            num_samples = len(working_df)\n            class_weights = (1.0 / class_counts) / (1.0 / num_samples) * (num_samples / len(class_counts))\n            weights = torch.tensor([class_weights[label2id[id2label[i]]] for i in range(len(label_list))], dtype=torch.float32).to(device)\n            loss = nn.BCEWithLogitsLoss(weight=weights)(pred, target)\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n        optimizer.zero_grad(set_to_none=True)\n        loss_l.append(loss.item())\n        torch.cuda.empty_cache()\n        gc.collect()\n    return np.mean(loss_l)\n\ndef valid_epoch(model, dataloader, device):\n    model.eval()\n    preds_l, targets_l = [], []\n    with torch.no_grad():\n        for batch in tqdm(dataloader, desc=\"Validation\"):\n            mel = batch['mel'].to(device, non_blocking=True)\n            mfcc = batch['mfcc'].to(device, non_blocking=True)\n            target = batch['target'].to(device, non_blocking=True)\n            geo_feature = torch.tensor(batch['geo_feature'], dtype=torch.float32).to(device, non_blocking=True)\n            rating_weight = torch.tensor(batch['rating_weight'], dtype=torch.float32).to(device, non_blocking=True)\n            with autocast(device_type=device.type):\n                pred = model(mel, mfcc, geo_feature, rating_weight)\n            preds_l.append(torch.sigmoid(pred).cpu().numpy())\n            targets_l.append(target.cpu().numpy())\n    preds = np.concatenate(preds_l)\n    targets = np.concatenate(targets_l)\n    auc = roc_auc_score(targets, preds, average='macro')\n    preds_binary = (preds > 0.5).astype(int)\n    precision = precision_score(targets, preds_binary, average='macro', zero_division=0)\n    recall = recall_score(targets, preds_binary, average='macro', zero_division=0)\n    f1 = f1_score(targets, preds_binary, average='macro', zero_division=0)\n    return auc, precision, recall, f1","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Main execution\n","metadata":{}},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    skf = StratifiedKFold(n_splits=config.NUM_FOLDS, shuffle=True, random_state=config.SEED)\n    for fold, (train_idx, val_idx) in enumerate(skf.split(working_df, working_df['primary_label'])):\n        print(f\"\\nStarting fold {fold}\")\n        train_df_fold = working_df.iloc[train_idx].reset_index(drop=True)\n        val_df_fold = working_df.iloc[val_idx].reset_index(drop=True)\n\n        train_loader = get_dataloader(train_df_fold, is_train=True)\n        val_loader = get_dataloader(val_df_fold, is_train=False)\n\n        device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n        model = BirdCRNN(num_classes=len(label_list)).to(device)\n        optimizer = AdamW(model.parameters(), lr=config.LEARNING_RATE)\n        scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=1)\n        scaler = GradScaler()\n\n        best_auc = 0\n        for epoch in range(config.EPOCHS):\n            train_loss = train_epoch(model, train_loader, optimizer, scheduler, scaler, device)\n            val_auc, val_precision, val_recall, val_f1 = valid_epoch(model, val_loader, device)\n            print(f\"Fold {fold}, Epoch {epoch}, Train Loss: {train_loss:.4f}, Val AUC: {val_auc:.4f}, Precision: {val_precision:.4f}, Recall: {val_recall:.4f}, F1: {val_f1:.4f}\")\n            scheduler.step(val_auc)\n            if val_auc > best_auc:\n                best_auc = val_auc\n                os.makedirs(config.CHECKPOINT_PATH, exist_ok=True)\n                torch.save(model.state_dict(), f\"{config.CHECKPOINT_PATH}/model_fold{fold}.pth\")\n                print(f\"Saved best model for fold {fold} with AUC {best_auc:.4f}\")\n        gc.collect()\n        torch.cuda.empty_cache()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}