{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":21669,"databundleVersionId":1692278,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport sys\nimport gc\nimport copy\nimport time\nimport random\nimport csv\nimport cv2\nimport warnings\nfrom pathlib import Path\nfrom dataclasses import dataclass\nfrom typing import List, Dict, Optional\n\nimport numpy as np\nimport pandas as pd\nimport librosa\nimport soundfile as sf\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom sklearn.model_selection import KFold","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T18:17:10.250617Z","iopub.execute_input":"2025-12-15T18:17:10.25119Z","iopub.status.idle":"2025-12-15T18:17:10.256314Z","shell.execute_reply.started":"2025-12-15T18:17:10.251159Z","shell.execute_reply":"2025-12-15T18:17:10.255583Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"try:\n    from efficientnet_pytorch import EfficientNet\nexcept ImportError:\n    os.system('pip install efficientnet_pytorch')\n    from efficientnet_pytorch import EfficientNet\n\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T18:17:10.257608Z","iopub.execute_input":"2025-12-15T18:17:10.257908Z","iopub.status.idle":"2025-12-15T18:17:10.273783Z","shell.execute_reply.started":"2025-12-15T18:17:10.257889Z","shell.execute_reply":"2025-12-15T18:17:10.273263Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Конфигурация ---\n@dataclass\nclass Config:\n    # Параметры задачи\n    num_classes: int = 24\n    sr: int = 48_000\n    duration: int = 10\n    n_folds: int = 5\n    epochs: int = 18\n    \n    # Параметры модели\n    model_name: str = 'efficientnet-b0'\n    batch_size: int = 8\n    lr: float = 1e-3\n    \n    # Параметры спектрограммы\n    fmin: int = 40\n    fmax: int = 16_000\n    image_size: tuple = (224, 512)\n    \n    # Устройство\n    device: torch.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    \n    # Пути\n    root_dir: Path = Path(\"../input/rfcx-species-audio-detection\")\n    train_dir: Path = root_dir / \"train\"\n    test_dir: Path = root_dir / \"test\"\n    output_dir: Path = Path(\"./\")\n\ncfg = Config()\nprint(f\"Конфигурация загружена. Девайс: {cfg.device}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T18:17:10.274405Z","iopub.execute_input":"2025-12-15T18:17:10.274628Z","iopub.status.idle":"2025-12-15T18:17:10.293892Z","shell.execute_reply.started":"2025-12-15T18:17:10.274607Z","shell.execute_reply":"2025-12-15T18:17:10.293305Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Утилиты для Аугментации ---\nclass AudioAugmentations:\n    \"\"\"Набор методов для аугментации спектрограмм\"\"\"\n    \n    @staticmethod\n    def add_noise(image: np.ndarray) -> np.ndarray:\n        \"\"\"Добавляет случайный шум\"\"\"\n        noise = np.random.randn(*image.shape) * 0.05\n        return image + noise\n\n    @staticmethod\n    def contrast_augment(image: np.ndarray) -> np.ndarray:\n        \"\"\"Случайное изменение контраста\"\"\"\n        factor = np.random.uniform(0.8, 1.2)\n        mean = image.mean()\n        return (image - mean) * factor + mean\n\n    @staticmethod\n    def apply(image: np.ndarray) -> np.ndarray:\n        \"\"\"Применяет случайную аугментацию\"\"\"\n        if np.random.rand() < 0.3:\n            image = AudioAugmentations.add_noise(image)\n        if np.random.rand() < 0.3:\n            image = AudioAugmentations.contrast_augment(image)\n        return image","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T18:17:10.295341Z","iopub.execute_input":"2025-12-15T18:17:10.295861Z","iopub.status.idle":"2025-12-15T18:17:10.307669Z","shell.execute_reply.started":"2025-12-15T18:17:10.295826Z","shell.execute_reply":"2025-12-15T18:17:10.307083Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Датасет ---\n# class RainforestDataset(Dataset):\n#     def __init__(self, df: pd.DataFrame, file_dict: Dict, cfg: Config, mode: str = 'train'):\n#         self.df = df\n#         self.file_dict = file_dict # Словарь предзагруженных спектрограмм (кэш)\n#         self.cfg = cfg\n#         self.mode = mode\n        \n#     def __len__(self):\n#         return len(self.df)\n        \n#     def __getitem__(self, idx):\n#         row = self.df.iloc[idx]\n#         recording_id = row['recording_id']\n\n#         # Спектрограмма из кэша\n#         image = self.file_dict[recording_id].copy()\n\n#         # Аугментация только для train\n#         if self.mode == 'train':\n#             image = AudioAugmentations.apply(image)\n\n#         image = np.stack([image, image, image])\n#         image = torch.tensor(image, dtype=torch.float32)\n        \n#         if self.mode in ['train', 'valid']:\n#             label = torch.tensor(row['species_id'], dtype=torch.long)\n#             return image, label\n#         else:\n#             return image, recording_id\n\n# --- Оптимизированный Датасет (Lazy Loading) ---\nclass RainforestDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, cfg: Config, mode: str = 'train'):\n        self.df = df\n        self.cfg = cfg\n        self.mode = mode\n        self.file_paths = {\n            rec_id: cfg.train_dir / f\"{rec_id}.flac\" \n            for rec_id in df['recording_id'].values\n        }\n        \n    def __len__(self):\n        return len(self.df)\n        \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        rec_id = row['recording_id']\n        path = self.file_paths[rec_id]\n        \n        # Оптимизация: грузим только нужный кусок, если есть t_min/t_max (только для train)\n        target_len = self.cfg.duration * self.cfg.sr\n        \n        if self.mode == 'train':\n            center_sec = (row['t_min'] + row['t_max']) / 2\n            offset = max(0, center_sec - self.cfg.duration / 2)\n            audio, _ = librosa.load(path, sr=self.cfg.sr)\n            \n            center_sample = int(center_sec * self.cfg.sr)\n            start = max(0, center_sample - target_len // 2)\n            end = start + target_len\n            \n            if end > len(audio):\n                end = len(audio)\n                start = max(0, end - target_len)\n                \n            audio = audio[start:end]\n            \n            if len(audio) < target_len:\n                audio = np.pad(audio, (0, target_len - len(audio)))\n        else:\n            audio, _ = librosa.load(path, sr=self.cfg.sr, duration=self.cfg.duration)\n            if len(audio) < target_len:\n                audio = np.pad(audio, (0, target_len - len(audio)))\n\n        melspec = librosa.feature.melspectrogram(\n            y=audio, sr=self.cfg.sr, n_mels=128, fmin=self.cfg.fmin, fmax=self.cfg.fmax\n        )\n        melspec = librosa.power_to_db(melspec, top_db=80)\n        melspec = (melspec - melspec.min()) / (melspec.max() - melspec.min() + 1e-6)\n        melspec = torch.tensor(melspec).unsqueeze(0).unsqueeze(0) # (1, 1, H, W)\n        melspec = torch.nn.functional.interpolate(\n            melspec, size=(self.cfg.image_size[0], self.cfg.image_size[1]), \n            mode='bilinear', align_corners=False\n        ).squeeze()\n        \n        image = melspec.numpy()\n        if self.mode == 'train':\n            image = AudioAugmentations.apply(image)  \n        image = np.stack([image, image, image])\n        image = torch.tensor(image, dtype=torch.float32)\n        \n        label = torch.tensor(row['species_id'], dtype=torch.long)\n        return image, label\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T18:17:10.308278Z","iopub.execute_input":"2025-12-15T18:17:10.308472Z","iopub.status.idle":"2025-12-15T18:17:10.327009Z","shell.execute_reply.started":"2025-12-15T18:17:10.308458Z","shell.execute_reply":"2025-12-15T18:17:10.326423Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Подготовка данных ---\ndef load_and_preprocess_audio(cfg: Config):\n    \"\"\"Загружает все аудиофайлы, переводит в Mel-спектрограммы и сохраняет в память.\"\"\"\n    print(\"Начинаем препроцессинг данных...\")\n    df_tp = pd.read_csv(cfg.root_dir / \"train_tp.csv\")\n    cache = {}\n    unique_ids = df_tp['recording_id'].unique()\n    target_len = cfg.duration * cfg.sr\n    \n    for rec_id in tqdm(unique_ids, desc=\"Кэширование train данных\"):\n        filepath = cfg.train_dir / f\"{rec_id}.flac\"\n        audio, _ = librosa.load(filepath, sr=cfg.sr)\n\n        t_min = df_tp[df_tp.recording_id == rec_id]['t_min'].min()\n        t_max = df_tp[df_tp.recording_id == rec_id]['t_max'].max()\n        center_sec = (t_min + t_max) / 2\n        \n        center_sample = int(center_sec * cfg.sr)\n        start_sample = max(0, center_sample - target_len // 2)\n        end_sample = start_sample + target_len\n        \n        # Если вышли за границы\n        if end_sample > len(audio):\n            end_sample = len(audio)\n            start_sample = end_sample - target_len\n            \n        chunk = audio[int(start_sample):int(end_sample)]\n        \n        # Если кусок короче нужного\n        if len(chunk) < target_len:\n            pad_width = target_len - len(chunk)\n            chunk = np.pad(chunk, (0, pad_width))\n            \n        # Генерация Mel-спектрограммы\n        melspec = librosa.feature.melspectrogram(\n            y=chunk, sr=cfg.sr, n_mels=128, fmin=cfg.fmin, fmax=cfg.fmax\n        )\n        melspec = librosa.power_to_db(melspec, top_db=80)\n        \n        melspec = (melspec - melspec.min()) / (melspec.max() - melspec.min() + 1e-6)\n        melspec = (melspec * 255).astype(np.float32) / 255.0 # Сразу в 0-1 float\n        melspec = cv2.resize(melspec, (cfg.image_size[1], cfg.image_size[0]))\n        \n        cache[rec_id] = melspec\n        \n    return df_tp, cache","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T18:17:10.350159Z","iopub.execute_input":"2025-12-15T18:17:10.350649Z","iopub.status.idle":"2025-12-15T18:17:10.358321Z","shell.execute_reply.started":"2025-12-15T18:17:10.350631Z","shell.execute_reply":"2025-12-15T18:17:10.357505Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Обучение ---\ndef train_one_epoch(model, loader, optimizer, criterion, device):\n    model.train()\n    running_loss = 0.0\n    \n    for images, labels in loader:\n        images = images.to(device)\n        labels = labels.to(device)\n        \n        optimizer.zero_grad()\n        outputs = model(images)\n        loss = criterion(outputs, labels)\n        \n        loss.backward()\n        optimizer.step()\n        \n        running_loss += loss.item() * images.size(0)\n        \n    return running_loss / len(loader.dataset)\n\n@torch.no_grad()\ndef validate(model, loader, criterion, device):\n    model.eval()\n    running_loss = 0.0\n    preds = []\n    targets = []\n    \n    for images, labels in loader:\n        images = images.to(device)\n        labels = labels.to(device)\n        \n        outputs = model(images)\n        loss = criterion(outputs, labels)\n        \n        running_loss += loss.item() * images.size(0)\n        \n        preds.append(outputs.cpu().numpy().argmax(axis=1))\n        targets.append(labels.cpu().numpy())\n        \n    all_preds = np.concatenate(preds)\n    all_targets = np.concatenate(targets)\n    accuracy = (all_preds == all_targets).mean()\n    \n    return running_loss / len(loader.dataset), accuracy","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T18:17:10.359841Z","iopub.execute_input":"2025-12-15T18:17:10.360154Z","iopub.status.idle":"2025-12-15T18:17:10.378266Z","shell.execute_reply.started":"2025-12-15T18:17:10.360138Z","shell.execute_reply":"2025-12-15T18:17:10.377679Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Основной цикл ---\ndef run_training():\n    # Просто читаем CSV, ничего не грузим в память!\n    df = pd.read_csv(cfg.root_dir / \"train_tp.csv\")\n    \n    kfold = KFold(n_splits=cfg.n_folds, shuffle=True, random_state=42)\n    \n    for fold, (train_idx, val_idx) in enumerate(kfold.split(df)):\n        print(f\"\\n=== Fold {fold+1}/{cfg.n_folds} ===\")\n        \n        train_df = df.iloc[train_idx]\n        val_df = df.iloc[val_idx]\n        \n        train_ds = RainforestDataset(train_df, cfg, mode='train')\n        val_ds = RainforestDataset(val_df, cfg, mode='valid')\n        \n        train_loader = DataLoader(\n            train_ds, \n            batch_size=cfg.batch_size, \n            shuffle=True, \n            drop_last=True,\n            num_workers=2,\n            pin_memory=True\n        )\n        val_loader = DataLoader(\n            val_ds, \n            batch_size=cfg.batch_size, \n            shuffle=False,\n            num_workers=2,\n            pin_memory=True\n        )\n        \n        # Модель\n        model = EfficientNet.from_pretrained(cfg.model_name, num_classes=cfg.num_classes)\n        model = model.to(cfg.device)\n        \n        optimizer = Adam(model.parameters(), lr=cfg.lr)\n        scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=3, verbose=True)\n        criterion = nn.CrossEntropyLoss()\n        \n        best_acc = 0.0\n        \n        # Эпохи\n        for epoch in range(cfg.epochs):\n            train_loss = train_one_epoch(model, train_loader, optimizer, criterion, cfg.device)\n            val_loss, val_acc = validate(model, val_loader, criterion, cfg.device)\n            \n            scheduler.step(val_loss)\n            \n            print(f\"Epoch {epoch+1}: Train Loss {train_loss:.4f} | Val Loss {val_loss:.4f} | Val Acc {val_acc:.4f}\")\n            \n            if val_acc > best_acc:\n                best_acc = val_acc\n                torch.save(model.state_dict(), f\"model_fold_{fold}.pth\")\n                \n        # Очистка памяти\n        del model, optimizer, train_loader, val_loader\n        gc.collect()\n        torch.cuda.empty_cache()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T18:17:10.378955Z","iopub.execute_input":"2025-12-15T18:17:10.379214Z","iopub.status.idle":"2025-12-15T18:17:10.396235Z","shell.execute_reply.started":"2025-12-15T18:17:10.379192Z","shell.execute_reply":"2025-12-15T18:17:10.395513Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Инференс и Ответ ---\ndef create_submission():\n    print(\"\\nГенерация сабмита...\")\n    test_files = [f.name.split('.')[0] for f in cfg.test_dir.glob(\"*.flac\")]\n\n    class TestDataset(Dataset):\n        def __init__(self, file_list, cfg):\n            self.file_list = file_list\n            self.cfg = cfg\n            self.target_len = cfg.duration * cfg.sr\n            \n        def __len__(self):\n            return len(self.file_list)\n            \n        def __getitem__(self, idx):\n            rec_id = self.file_list[idx]\n            path = self.cfg.test_dir / f\"{rec_id}.flac\"\n            audio, _ = librosa.load(path, sr=self.cfg.sr)\n            \n            segments = []\n            num_segments = int(np.ceil(len(audio) / self.target_len))\n            num_segments = min(num_segments, 6) \n            \n            for i in range(num_segments):\n                start = i * self.target_len\n                end = start + self.target_len\n                if end > len(audio): break\n                \n                chunk = audio[start:end]\n                \n                # Mel + Image logic\n                melspec = librosa.feature.melspectrogram(\n                    y=chunk, sr=self.cfg.sr, n_mels=128, fmin=self.cfg.fmin, fmax=self.cfg.fmax\n                )\n                melspec = librosa.power_to_db(melspec, top_db=80)\n                melspec = (melspec - melspec.min()) / (melspec.max() - melspec.min() + 1e-6)\n                \n                img = cv2.resize(melspec, (self.cfg.image_size[1], self.cfg.image_size[0])) \n                img = np.stack([img, img, img])\n                segments.append(img)\n            \n            if not segments:\n                return torch.zeros((3, self.cfg.image_size[0], self.cfg.image_size[1])), rec_id\n                \n            return np.stack(segments), rec_id\n\n    test_ds = TestDataset(test_files, cfg)\n    test_loader = DataLoader(test_ds, batch_size=1, shuffle=False) \n    \n    models = []\n    for i in range(cfg.n_folds):\n        path = f\"model_fold_{i}.pth\"\n        if os.path.exists(path):\n            m = EfficientNet.from_pretrained(cfg.model_name, num_classes=cfg.num_classes)\n            m.load_state_dict(torch.load(path, map_location=cfg.device))\n            m.to(cfg.device)\n            m.eval()\n            models.append(m)\n    \n    results = {}\n    \n    with torch.no_grad():\n        for batch_segments, rec_ids in tqdm(test_loader, desc=\"Inference\"):\n            inputs = batch_segments.squeeze(0).float().to(cfg.device)\n            rec_id = rec_ids[0]\n            \n            fold_preds = []\n            for model in models:\n                logits = model(inputs) \n                max_logits, _ = torch.max(logits, dim=0) \n                probs = torch.sigmoid(max_logits)\n                fold_preds.append(probs.cpu().numpy())\n            \n            avg_pred = np.mean(fold_preds, axis=0)\n            results[rec_id] = avg_pred\n\n    with open('submission.csv', 'w', newline='') as f:\n        writer = csv.writer(f)\n        header = ['recording_id'] + [f's{i}' for i in range(24)]\n        writer.writerow(header)\n        \n        for rec_id in test_files:\n            if rec_id in results:\n                row = [rec_id] + list(results[rec_id])\n            else:\n                row = [rec_id] + [0.0]*24\n            writer.writerow(row)\n            \n    print(\"Файл submission.csv создан\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T18:17:10.397025Z","iopub.execute_input":"2025-12-15T18:17:10.3973Z","iopub.status.idle":"2025-12-15T18:17:10.416201Z","shell.execute_reply.started":"2025-12-15T18:17:10.397278Z","shell.execute_reply":"2025-12-15T18:17:10.415348Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    run_training()\n    create_submission()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T18:17:10.417703Z","iopub.execute_input":"2025-12-15T18:17:10.417914Z"}},"outputs":[],"execution_count":null}]}