{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"},{"sourceId":11840064,"sourceType":"datasetVersion","datasetId":7438986},{"sourceId":11855918,"sourceType":"datasetVersion","datasetId":7449695},{"sourceId":11855938,"sourceType":"datasetVersion","datasetId":7439602}],"dockerImageVersionId":31011,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import Modules","metadata":{}},{"cell_type":"code","source":"!pip install /kaggle/input/birdclef-dependencies/audiomentations-0.33.0-py3-none-any.whl","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-18T14:19:32.554831Z","iopub.execute_input":"2025-05-18T14:19:32.554995Z","iopub.status.idle":"2025-05-18T14:19:37.085708Z","shell.execute_reply.started":"2025-05-18T14:19:32.554978Z","shell.execute_reply":"2025-05-18T14:19:37.084875Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport librosa\nimport glob\nimport torch\nimport torch.nn as nn\nimport torchaudio\nimport os\nimport random\nimport shutil\nimport signal\nimport pickle\nimport gc\nimport time\nimport logging\nfrom warnings import filterwarnings\nfrom audiomentations import Compose, AddGaussianNoise\nfrom torch.cuda.amp import autocast, GradScaler\nimport joblib\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score, roc_curve, confusion_matrix, precision_recall_fscore_support\nfrom sklearn.linear_model import LogisticRegression\nfrom tqdm import tqdm\nimport timm\nimport matplotlib.pyplot as plt\n%matplotlib inline","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-18T14:19:37.088013Z","iopub.execute_input":"2025-05-18T14:19:37.088294Z","iopub.status.idle":"2025-05-18T14:19:50.522263Z","shell.execute_reply.started":"2025-05-18T14:19:37.088267Z","shell.execute_reply":"2025-05-18T14:19:50.521469Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Logging Setup","metadata":{}},{"cell_type":"code","source":"logging.basicConfig(\n    level=logging.INFO,\n    format='%(asctime)s - %(levelname)s - %(message)s',\n    handlers=[\n        logging.FileHandler('birdclef.log'),\n        logging.StreamHandler()\n    ]\n)\nlogger = logging.getLogger(__name__)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-18T14:19:50.523064Z","iopub.execute_input":"2025-05-18T14:19:50.523519Z","iopub.status.idle":"2025-05-18T14:19:50.527915Z","shell.execute_reply.started":"2025-05-18T14:19:50.523499Z","shell.execute_reply":"2025-05-18T14:19:50.527218Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"class Config:\n    train_dir = \"/kaggle/input/birdclef-2025/train_audio\"\n    seed = 42\n    train_csv = \"/kaggle/input/birdclef-2025/train.csv\"\n    sample_csv = \"/kaggle/input/birdclef-2025/sample_submission.csv\"\n    test_soundscapes = \"/kaggle/input/birdclef-2025/test_soundscapes\"\n    weights_path_b3 = \"/kaggle/input/efficientnet-weights/tf_efficientnet_b3.pth\"\n    weights_path_convnext = \"/kaggle/input/convnext-weights/convnext_small.pth\"\n    feature_dir = \"/kaggle/working/features\"\n    cache_dir = \"/kaggle/working/cache\"\n    model_weights_dir = \"/kaggle/working/model_weights\"\n    checkpoint_dir = \"/kaggle/working/checkpoints\"\n    sr = int(32e3)\n    num_classes = 206\n    n_fft = 1024\n    hop_length = 512\n    n_mels = 128\n    n_mels_low = 32\n    fmin = 50\n    fmax = 16000\n    power = 2\n    chunk_duration = 10\n    batch_size = 16\n    inference_batch_size = 32\n    epochs = 5\n    lr = 5e-5\n    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    early_stop_patience = 3\n    tta_chunks = 3\n    max_pseudo_samples = 1000\n    dropout = 0.3\n    label_smoothing = 0.1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-18T14:19:50.528642Z","iopub.execute_input":"2025-05-18T14:19:50.528877Z","iopub.status.idle":"2025-05-18T14:19:50.721997Z","shell.execute_reply.started":"2025-05-18T14:19:50.528860Z","shell.execute_reply":"2025-05-18T14:19:50.721197Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Helper function","metadata":{}},{"cell_type":"code","source":"def set_seed(seed: int = Config.seed) -> None:\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed)\n        torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    logger.info(f\"Set Seed: {seed}\")\n\ndef check_gpu():\n    if not torch.cuda.is_available():\n        logger.error(\"GPU not available, please enable GPU P100\")\n        raise RuntimeError(\"GPU required for efficient training\")\n    logger.info(f\"Using GPU: {torch.cuda.get_device_name(0)}\")\n\ndef get_disk_usage(path=\"/kaggle/working\"):\n    total_size = sum(os.path.getsize(os.path.join(dirpath, f)) \n                     for dirpath, _, filenames in os.walk(path) for f in filenames)\n    return total_size / (1024**3)  # GiB\n\ndef save_checkpoint(model_name, fold, fold_aucs, fold_predictions, all_metrics):\n    os.makedirs(Config.checkpoint_dir, exist_ok=True)\n    checkpoint = {\n        'fold_aucs': fold_aucs,\n        'fold_predictions': fold_predictions,\n        'all_metrics': all_metrics,\n        'completed_fold': fold\n    }\n    checkpoint_path = os.path.join(Config.checkpoint_dir, f\"checkpoint_{model_name}_fold_{fold}.pkl\")\n    with open(checkpoint_path, 'wb') as f:\n        pickle.dump(checkpoint, f)\n    logger.info(f\"Saved checkpoint: {checkpoint_path}\")\n\ndef load_checkpoint(model_name, fold):\n    checkpoint_path = os.path.join(Config.checkpoint_dir, f\"checkpoint_{model_name}_fold_{fold}.pkl\")\n    if os.path.exists(checkpoint_path):\n        with open(checkpoint_path, 'rb') as f:\n            checkpoint = pickle.load(f)\n        logger.info(f\"Loaded checkpoint: {checkpoint_path}\")\n        return checkpoint\n    return None\n\nset_seed()\ncheck_gpu()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-18T14:19:50.722999Z","iopub.execute_input":"2025-05-18T14:19:50.723231Z","iopub.status.idle":"2025-05-18T14:19:50.744945Z","shell.execute_reply.started":"2025-05-18T14:19:50.723214Z","shell.execute_reply":"2025-05-18T14:19:50.744203Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Audio Augmentation","metadata":{}},{"cell_type":"code","source":"augment = Compose([\n    AddGaussianNoise(min_amplitude=0.001, max_amplitude=0.015, p=0.5),\n])\n\ntta_augment = Compose([\n    AddGaussianNoise(min_amplitude=0.001, max_amplitude=0.01, p=0.5),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-18T14:19:50.745651Z","iopub.execute_input":"2025-05-18T14:19:50.745893Z","iopub.status.idle":"2025-05-18T14:19:50.750727Z","shell.execute_reply.started":"2025-05-18T14:19:50.745874Z","shell.execute_reply":"2025-05-18T14:19:50.750023Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# SpecAugment\ndef spec_augment(spec, max_time_mask=40, max_freq_mask=20):\n    spec = torch.tensor(spec, dtype=torch.float32)\n    for _ in range(2):\n        t = random.randint(0, max_time_mask)\n        t0 = random.randint(0, spec.shape[-1] - t)\n        spec[:, :, t0:t0+t] = 0\n        f = random.randint(0, max_freq_mask)\n        f0 = random.randint(0, spec.shape[-2] - f)\n        spec[:, f0:f0+f, :] = 0\n    return spec.numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-18T14:19:50.753046Z","iopub.execute_input":"2025-05-18T14:19:50.753377Z","iopub.status.idle":"2025-05-18T14:19:50.762935Z","shell.execute_reply.started":"2025-05-18T14:19:50.753356Z","shell.execute_reply":"2025-05-18T14:19:50.762350Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"code","source":"def extract_features(audio, sr, n_mels=Config.n_mels, hop_length=Config.hop_length):\n    transform = torchaudio.transforms.MelSpectrogram(\n        sample_rate=sr, n_fft=Config.n_fft, hop_length=hop_length,\n        n_mels=n_mels, f_min=Config.fmin, f_max=Config.fmax\n    )\n    with torch.no_grad():\n        mel_sp = transform(torch.tensor(audio, dtype=torch.float32)).numpy()\n    mel_sp = np.clip(mel_sp, a_min=1e-10, a_max=1e10)\n    mel_sp = librosa.power_to_db(mel_sp, ref=1.0)\n    return np.clip(mel_sp, a_min=-100, a_max=100).astype(np.float32)\n\ndef transform_features(mel_sp, mel_sp_low):\n    mel_sp = librosa.util.normalize(mel_sp).astype(np.float32)\n    mel_sp_low = librosa.util.normalize(mel_sp_low).astype(np.float32)\n    mel_delta = librosa.feature.delta(mel_sp).astype(np.float32)\n    mel_delta2 = librosa.feature.delta(mel_sp, order=2).astype(np.float32)\n    mel_delta = np.where(np.isnan(mel_delta) | np.isinf(mel_delta), 0.0, mel_delta)\n    mel_delta2 = np.where(np.isnan(mel_delta2) | np.isinf(mel_delta2), 0.0, mel_delta2)\n    return np.stack([mel_sp, mel_delta, mel_delta2, mel_sp_low], axis=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-18T14:19:50.763650Z","iopub.execute_input":"2025-05-18T14:19:50.763872Z","iopub.status.idle":"2025-05-18T14:19:50.774935Z","shell.execute_reply.started":"2025-05-18T14:19:50.763850Z","shell.execute_reply":"2025-05-18T14:19:50.774260Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Precompute Features","metadata":{}},{"cell_type":"code","source":"corrupted_files = []\n\ndef timeout_handler(signum, frame):\n    raise TimeoutError(\"Processing timeout\")\n\ndef process_audio(row, mode=\"train\"):\n    idx, row = row\n    start_time = time.time()\n    audio_path = row['filename']\n    cache_path = os.path.join(Config.cache_dir, f\"{idx}_{mode}.npz\")\n    if os.path.exists(cache_path):\n        logger.debug(f\"Loaded cached features: {cache_path}\")\n        return cache_path\n    feature_path = os.path.join(Config.feature_dir, f\"{idx}_{mode}.npz\")\n    try:\n        signal.signal(signal.SIGALRM, timeout_handler)\n        signal.alarm(5)\n        data, _ = librosa.load(audio_path, sr=Config.sr)\n        signal.alarm(0)\n        duration = librosa.get_duration(y=data, sr=Config.sr)\n        energy = np.mean(np.abs(librosa.stft(data)))\n        if duration < 0.5 or np.all(data == 0) or np.any(np.abs(data) > 1e6) or np.var(data) < 1e-6 or energy < 1e-6:\n            raise ValueError(f\"Invalid audio: duration={duration:.2f}s, energy={energy:.2e}\")\n        data = data * 1024\n        if mode == \"train\":\n            data = augment(samples=data, sample_rate=Config.sr)\n        min_len = Config.chunk_duration * Config.sr\n        if len(data) < min_len:\n            data = np.tile(data, int(np.ceil(min_len / len(data))))\n        data = data[:min_len]\n        mel_sp = extract_features(data, Config.sr, n_mels=Config.n_mels)\n        mel_sp_low = extract_features(data, Config.sr, n_mels=Config.n_mels_low)\n        features = transform_features(mel_sp, mel_sp_low)[:, :, :641]\n        if np.any(np.isnan(features)) or np.any(np.isinf(features)):\n            raise ValueError(\"NaN/Inf in features\")\n        os.makedirs(Config.cache_dir, exist_ok=True)\n        np.savez_compressed(cache_path, features=features, allow_pickle=False)\n        np.savez_compressed(feature_path, features=features, allow_pickle=False)\n    except (Exception, TimeoutError, ValueError) as e:\n        file_size = os.path.getsize(audio_path) / (1024**2) if os.path.exists(audio_path) else 0\n        logger.warning(f\"Corrupted file {audio_path} ({file_size:.2f} MiB): {e}\")\n        corrupted_files.append({'file': audio_path, 'error': str(e), 'size_mb': file_size})\n        features = np.zeros((4, Config.n_mels, 641), dtype=np.float32)\n        np.savez_compressed(feature_path, features=features, allow_pickle=False)\n    elapsed = time.time() - start_time\n    if elapsed > 5:\n        logger.info(f\"Slow processing: {audio_path} took {elapsed:.2f} seconds\")\n    return feature_path\n\ndef precompute_features(df, mode=\"train\"):\n    start_time = time.time()\n    os.makedirs(Config.feature_dir, exist_ok=True)\n    n_jobs = max(1, joblib.cpu_count() // 2)\n    feature_paths = joblib.Parallel(n_jobs=n_jobs, backend='loky', verbose=10)(\n        joblib.delayed(process_audio)(row, mode) for row in df.iterrows()\n    )\n    df['feature_path'] = feature_paths\n    missing_files = [path for path in df['feature_path'] if not os.path.exists(path)]\n    if missing_files:\n        logger.error(f\"Missing {len(missing_files)} feature files: {missing_files[:5]}\")\n        raise FileNotFoundError(\"Feature files missing\")\n    logger.info(f\"Precompute features took {(time.time() - start_time) / 60:.2f} minutes\")\n    logger.info(f\"Disk usage: {get_disk_usage():.2f} GiB\")\n    return df\n\ndef precompute_test_features(test_files):\n    start_time = time.time()\n    test_df = pd.DataFrame({'filename': test_files})\n    test_df = precompute_features(test_df, mode=\"test\")\n    logger.info(f\"Test features took {(time.time() - start_time) / 60:.2f} minutes\")\n    return test_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-18T14:19:50.775658Z","iopub.execute_input":"2025-05-18T14:19:50.775899Z","iopub.status.idle":"2025-05-18T14:19:50.791843Z","shell.execute_reply.started":"2025-05-18T14:19:50.775883Z","shell.execute_reply":"2025-05-18T14:19:50.791145Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Dataset  ","metadata":{}},{"cell_type":"code","source":"sample_submission = pd.read_csv(Config.sample_csv)\nall_classes = sample_submission.columns[1:].tolist()\nlabel_mapper = {label: idx for idx, label in enumerate(all_classes)}\nrev_mapper = {idx: label for label, idx in label_mapper.items()}\nlogger.info(f\"Created label_mapper with {len(label_mapper)} classes\")\n\nclass BirdClefDataset(torch.utils.data.Dataset):\n    def __init__(self, df, mode=\"train\"):\n        self.df = df\n        self.mode = mode\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        feature_path = row['feature_path']\n        try:\n            x = np.load(feature_path)['features'].astype(np.float32)\n            if np.any(np.isnan(x)) or np.any(np.isinf(x)):\n                raise ValueError(\"NaN/Inf in features\")\n        except Exception as e:\n            logger.warning(f\"Error loading {feature_path}: {e}\")\n            x = np.zeros((4, Config.n_mels, 641), dtype=np.float32)\n        if self.mode == \"train\":\n            label = row['primary_label']\n            y = label_mapper.get(label, 0)\n            if random.random() < 0.3:\n                idx2 = random.randint(0, len(self.df) - 1)\n                row2 = self.df.iloc[idx2]\n                x2 = np.load(row2['feature_path'])['features'].astype(np.float32)\n                y2 = label_mapper.get(row2['primary_label'], 0)\n                alpha = 0.2\n                x = alpha * x + (1 - alpha) * x2\n                y_onehot = torch.zeros(Config.num_classes)\n                y_onehot[y] = alpha\n                y_onehot[y2] = 1 - alpha\n                return torch.tensor(x, dtype=torch.float32), y_onehot\n            return torch.tensor(x, dtype=torch.float32), y\n        return torch.tensor(x, dtype=torch.float32)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-18T14:19:50.792628Z","iopub.execute_input":"2025-05-18T14:19:50.793316Z","iopub.status.idle":"2025-05-18T14:19:50.821106Z","shell.execute_reply.started":"2025-05-18T14:19:50.793298Z","shell.execute_reply":"2025-05-18T14:19:50.820607Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self, model_name: str, weights_path: str):\n        super().__init__()\n        self.base_model = timm.create_model(\n            model_name=model_name, num_classes=Config.num_classes,\n            pretrained=False, in_chans=4, drop_rate=Config.dropout\n        )\n        if os.path.exists(weights_path):\n            pretrained_dict = torch.load(weights_path, map_location=Config.device)\n            model_dict = self.base_model.state_dict()\n            exclude_keys = ['fc.weight', 'fc.bias'] if 'convnext' in model_name else ['classifier.weight', 'classifier.bias']\n            pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict and k not in exclude_keys}\n            self.base_model.load_state_dict(pretrained_dict, strict=False)\n            logger.info(f\"Loaded weights from {weights_path}\")\n        else:\n            logger.warning(f\"Weights not found at {weights_path}\")\n\n    def forward(self, x):\n        x = torch.clamp(x, min=-1e6, max=1e6)\n        return self.base_model(x)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-18T14:19:50.821666Z","iopub.execute_input":"2025-05-18T14:19:50.821858Z","iopub.status.idle":"2025-05-18T14:19:50.827556Z","shell.execute_reply.started":"2025-05-18T14:19:50.821843Z","shell.execute_reply":"2025-05-18T14:19:50.827014Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Focal Loss","metadata":{}},{"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self, gamma=2.0, alpha=1.0, label_smoothing=0.0):\n        super().__init__()\n        self.gamma = gamma\n        self.alpha = alpha\n        self.label_smoothing = label_smoothing\n\n    def forward(self, inputs, targets):\n        if targets.dim() == 1:\n            targets = torch.nn.functional.one_hot(targets, Config.num_classes).float()\n        targets = targets * (1 - self.label_smoothing) + self.label_smoothing / Config.num_classes\n        log_probs = torch.nn.functional.log_softmax(inputs, dim=-1)\n        ce_loss = -torch.sum(targets * log_probs, dim=-1)\n        pt = torch.exp(-ce_loss)\n        return self.alpha * ((1 - pt) ** self.gamma) * ce_loss.mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-18T14:19:50.828107Z","iopub.execute_input":"2025-05-18T14:19:50.828318Z","iopub.status.idle":"2025-05-18T14:19:50.839644Z","shell.execute_reply.started":"2025-05-18T14:19:50.828296Z","shell.execute_reply":"2025-05-18T14:19:50.839002Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Models Definition\nmodels = [\n    {'name': 'tf_efficientnet_b3', 'weights_path': Config.weights_path_b3},\n    {'name': 'convnext_small', 'weights_path': Config.weights_path_convnext}\n]\nlogger.info(f\"Models defined: {[m['name'] for m in models]}\")\n\n# Training and Pseudo-Labeling\nstart_time = time.time()\ndata_df = pd.read_csv(Config.train_csv)\ndata_df['filename'] = data_df['filename'].apply(lambda x: Config.train_dir + \"/\" + x)\nlogger.info(f\"Using train set: {len(data_df)} samples\")\n\n# Class Imbalance\nclass_counts = data_df['primary_label'].value_counts()\nweights = data_df['primary_label'].map(lambda x: 1.0 / class_counts[x] if x in class_counts else 1.0)\nclass_weights = torch.tensor(weights.values, dtype=torch.float32).to(Config.device)\n\n# Load or Precompute Features\ndata_df_path = os.path.join(Config.checkpoint_dir, 'data_df.csv')\nif os.path.exists(data_df_path):\n    data_df = pd.read_csv(data_df_path)\n    missing_files = [path for path in data_df['feature_path'] if not os.path.exists(path)]\n    if missing_files:\n        logger.warning(f\"Missing {len(missing_files)} feature files\")\n        data_df = precompute_features(data_df, mode=\"train\")\nelse:\n    data_df = precompute_features(data_df, mode=\"train\")\n    os.makedirs(Config.checkpoint_dir, exist_ok=True)\n    data_df.to_csv(data_df_path, index=False)\n\n# Handle Missing Classes\nunique_classes = data_df['primary_label'].unique()\nmissing_classes = set(all_classes) - set(unique_classes)\nif missing_classes:\n    logger.warning(f\"Missing classes: {missing_classes}\")\n    dummy_rows = []\n    for cls in missing_classes:\n        dummy_row = data_df.iloc[0].copy()\n        dummy_row['primary_label'] = cls\n        dummy_rows.append(dummy_row)\n    data_df = pd.concat([data_df, pd.DataFrame(dummy_rows)], ignore_index=True)\n    data_df = precompute_features(data_df, mode=\"train\")\ndata_df.to_csv(data_df_path, index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Iterative Pseudo-Labeling","metadata":{}},{"cell_type":"code","source":"def predict_test_ensemble(test_df, device, folds=5, model_configs=None, meta_learner=None):\n    model_predictions = []\n    for model_config in model_configs:\n        model_name = model_config['name']\n        weights_path = model_config['weights_path']\n        model_preds = []\n        for fold in range(folds):\n            model = Model(model_name, weights_path).to(device)\n            weight_path = os.path.join(Config.model_weights_dir, f\"{model_name}_fold_{fold}_best.pth\")\n            if os.path.exists(weight_path):\n                model.load_state_dict(torch.load(weight_path))\n                logger.info(f\"Loaded fold {fold} weights from {weight_path}\")\n            model.eval()\n            fold_preds = []\n            with torch.no_grad():\n                for _, row in tqdm(test_df.iterrows(), total=len(test_df), desc=f\"Predicting Fold {fold} {model_name}\"):\n                    signal.signal(signal.SIGALRM, timeout_handler)\n                    signal.alarm(5)\n                    tta_preds = []\n                    for _ in range(Config.tta_chunks):\n                        try:\n                            features = np.load(row['feature_path'])['features'].astype(np.float32)\n                            if random.random() < 0.5:\n                                audio, _ = librosa.load(row['filename'], sr=Config.sr)\n                                audio = tta_augment(samples=audio, sample_rate=Config.sr)\n                                mel_sp = extract_features(audio, Config.sr, n_mels=Config.n_mels)\n                                mel_sp_low = extract_features(audio, Config.sr, n_mels=Config.n_mels_low)\n                                features = transform_features(mel_sp, mel_sp_low)[:, :, :641]\n                            features = torch.tensor(features).unsqueeze(0).to(device)\n                            outputs = model(features)\n                            probs = torch.softmax(outputs, dim=1)\n                            probs = probs / (probs.sum(dim=1, keepdim=True) + 1e-10)\n                            probs = probs.detach().cpu().numpy()\n                            tta_preds.append(probs)\n                        except (Exception, TimeoutError) as e:\n                            logger.warning(f\"Error in TTA {row['filename']}: {e}\")\n                            tta_preds.append(np.ones((1, Config.num_classes)) / Config.num_classes)\n                    signal.alarm(0)\n                    fold_preds.append(np.mean(tta_preds, axis=0))\n                fold_preds = np.array(fold_preds)\n                model_preds.append(fold_preds)\n        model_predictions.append(np.mean(model_preds, axis=0))\n    ensemble_preds = np.mean(model_predictions, axis=0)\n    if meta_learner:\n        stacked_preds = np.concatenate(model_predictions, axis=-1)\n        ensemble_preds = meta_learner.predict_proba(stacked_preds)\n    return ensemble_preds\n\ntest_files = glob.glob(os.path.join(Config.test_soundscapes, \"*.ogg\"))\ntest_df = precompute_test_features(test_files)\n\nfor pseudo_round in range(2):\n    logger.info(f\"Pseudo-Labeling Round {pseudo_round + 1}\")\n    pseudo_threshold = 0.95 if pseudo_round == 0 else 0.90\n    test_preds = predict_test_ensemble(test_df, Config.device, model_configs=models)\n    pseudo_df = test_df.copy()\n    pseudo_df['primary_label'] = [rev_mapper[np.argmax(pred)] for pred in test_preds]\n    pseudo_df['confidence'] = np.max(test_preds, axis=1)\n    pseudo_df = pseudo_df[pseudo_df['confidence'] > pseudo_threshold].reset_index(drop=True)\n    if len(pseudo_df) > 0:\n        class_counts = pseudo_df['primary_label'].value_counts()\n        max_samples_per_class = min(50, len(pseudo_df) // len(class_counts))\n        pseudo_df = pseudo_df.groupby('primary_label').head(max_samples_per_class).reset_index(drop=True)\n        pseudo_df = pseudo_df.head(Config.max_pseudo_samples)\n        data_df = pd.concat([data_df, pseudo_df[['filename', 'feature_path', 'primary_label']]], ignore_index=True)\n        data_df = precompute_features(data_df, mode=\"train\")\n        data_df.to_csv(data_df_path, index=False)\n        logger.info(f\"Added {len(pseudo_df)} pseudo-labeled samples\")\n    else:\n        logger.info(\"No pseudo-labeled samples added\")\n    gc.collect()\n    torch.cuda.empty_cache()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Final Experimental Results","metadata":{}},{"cell_type":"code","source":"def plot_roc_curves(fold_predictions, model_name):\n    plt.figure(figsize=(8, 6))\n    all_fpr = []\n    all_tpr = []\n    for fold, (pred_val, label_val) in enumerate(fold_predictions[model_name]):\n        fpr, tpr, _ = roc_curve(\n            pd.get_dummies(label_val).values.ravel(),\n            pred_val.ravel()\n        )\n        all_fpr.append(fpr)\n        all_tpr.append(tpr)\n        plt.plot(fpr, tpr, label=f'Fold {fold} (AUC = {fold_aucs[model_name][fold]:.4f})')\n    mean_fpr = np.linspace(0, 1, 100)\n    tprs = []\n    for fpr, tpr in zip(all_fpr, all_tpr):\n        tprs.append(np.interp(mean_fpr, fpr, tpr))\n    mean_tpr = np.mean(tprs, axis=0)\n    std_tpr = np.std(tprs, axis=0)\n    plt.plot(mean_fpr, mean_tpr, 'k--', label=f'Mean (AUC = {np.mean(fold_aucs[model_name]):.4f})')\n    plt.fill_between(mean_fpr, mean_tpr - std_tpr, mean_tpr + std_tpr, color='grey', alpha=0.2)\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title(f'ROC Curves for {model_name}')\n    plt.legend()\n    plt.savefig(f'roc_{model_name}.png')\n    plt.show()\n    plt.close()\n    logger.info(f\"Saved ROC plot: roc_{model_name}.png\")\n\ndef compute_metrics(fold_predictions, model_name):\n    metrics = []\n    print(f\"\\nFinal Metrics for {model_name}:\")\n    for fold, (pred_val, label_val) in enumerate(fold_predictions[model_name]):\n        pred_labels = np.argmax(pred_val, axis=1)\n        cm = confusion_matrix(label_val, pred_labels)\n        print(f\"\\nConfusion Matrix for Fold {fold} (Top 5x5 for brevity):\")\n        print(cm[:5, :5])  # แสดงเฉพาะ 5x5 เพื่อความกระชับ\n        precision, recall, f1, _ = precision_recall_fscore_support(\n            label_val, pred_labels, average='macro', zero_division=0\n        )\n        specificity = []\n        for i in range(Config.num_classes):\n            tn = cm.sum() - (cm[i, :].sum() + cm[:, i].sum() - cm[i, i])\n            fp = cm[:, i].sum() - cm[i, i]\n            specificity.append(tn / (tn + fp + 1e-10))\n        specificity = np.mean(specificity)\n        metrics.append({\n            'fold': fold,\n            'precision': precision,\n            'recall': recall,\n            'f1': f1,\n            'specificity': specificity\n        })\n        logger.info(f\"Fold {fold} {model_name} - Precision: {precision:.4f}, Recall: {recall:.4f}, F1: {f1:.4f}, Specificity: {specificity:.4f}\")\n        print(f\"Fold {fold}: Precision: {precision:.4f}, Recall: {recall:.4f}, F1: {f1:.4f}, Specificity: {specificity:.4f}\")\n    \n    # Summary Table\n    metrics_df = pd.DataFrame(metrics)\n    mean_metrics = metrics_df[['precision', 'recall', 'f1', 'specificity']].mean()\n    std_metrics = metrics_df[['precision', 'recall', 'f1', 'specificity']].std()\n    summary_df = pd.DataFrame({\n        'Metric': ['Precision', 'Recall', 'F1', 'Specificity'],\n        'Mean': mean_metrics.values,\n        'Std': std_metrics.values\n    })\n    print(f\"\\nSummary Metrics for {model_name}:\\n{summary_df.to_string(index=False)}\")\n    logger.info(f\"Summary {model_name} - Mean Precision: {mean_metrics['precision']:.4f}, Recall: {mean_metrics['recall']:.4f}, F1: {mean_metrics['f1']:.4f}, Specificity: {mean_metrics['specificity']:.4f}\")\n    return metrics\n\nfor model_name in [model['name'] for model in models]:\n    all_metrics[model_name] = compute_metrics(fold_predictions, model_name)\n    plot_roc_curves(fold_predictions, model_name)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Stacking Ensemble","metadata":{}},{"cell_type":"code","source":"meta_learner = LogisticRegression(max_iter=1000)\nstacked_val_preds = []\nstacked_val_labels = []\nfor model_name in [model['name'] for model in models]:\n    for pred_val, label_val in fold_predictions[model_name]:\n        stacked_val_preds.append(pred_val)\n        stacked_val_labels.append(label_val)\nstacked_val_preds = np.concatenate(stacked_val_preds, axis=-1)\nstacked_val_labels = stacked_val_labels[0]\nmeta_learner.fit(stacked_val_preds, stacked_val_labels)\nlogger.info(\"Trained meta-learner for stacking ensemble\")","metadata":{"trusted":true,"execution":{"execution_failed":"2025-05-18T14:34:39.873Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"# Final Submission\ntest_df = precompute_test_features(test_files)\ntest_preds = predict_test_ensemble(test_df, Config.device, model_configs=models, meta_learner=meta_learner)\nsample_submission.iloc[:, 1:] = test_preds\nsample_submission.to_csv(\"submission.csv\", index=False)\nlogger.info(f\"Created submission.csv, shape: {sample_submission.shape}\")\n\n# Clean up\nshutil.rmtree(Config.feature_dir, ignore_errors=True)\nshutil.rmtree(Config.model_weights_dir, ignore_errors=True)\nlogger.info(f\"Final disk usage: {get_disk_usage():.2f} GiB\")\nlogger.info(f\"Total Runtime: {(time.time() - start_time) / 60:.2f} minutes\")\nlogger.info(f\"Corrupted files: {[{k: v for k, v in d.items()} for d in corrupted_files]}\")","metadata":{"trusted":true,"execution":{"execution_failed":"2025-05-18T14:34:39.874Z"}},"outputs":[],"execution_count":null}]}