{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q pydicom\n!pip install -q dicomsdl  # Faster DICOM reader, optional but recommended\n!pip install -q albumentations\n!pip install -q timm  # For pretrained models\n!pip install -q torch torchvision  # Ensure latest torch\n\nimport os\nimport gc\nimport sys\nimport glob\nimport json\nimport random\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport cv2\nfrom PIL import Image\nfrom tqdm import tqdm\nfrom sklearn.model_selection import StratifiedGroupKFold\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda.amp import autocast, GradScaler\n\nimport timm\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport pydicom\n# import dicomsdl  # If installed\n\nprint(\"Torch version:\", torch.__version__)\nprint(\"CUDA available:\", torch.cuda.is_available())  # Должен вывести True, если GPU on\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(\"Device:\", device)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-18T08:50:26.535470Z","iopub.execute_input":"2026-01-18T08:50:26.535809Z","iopub.status.idle":"2026-01-18T08:50:57.628617Z","shell.execute_reply.started":"2026-01-18T08:50:26.535781Z","shell.execute_reply":"2026-01-18T08:50:57.627820Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"study_id = 100206310\nseries_id = 1792451510","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T16:56:59.337347Z","iopub.execute_input":"2026-01-17T16:56:59.337571Z","iopub.status.idle":"2026-01-17T16:56:59.350528Z","shell.execute_reply.started":"2026-01-17T16:56:59.337550Z","shell.execute_reply":"2026-01-17T16:56:59.349918Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Пути — самые частые ошибки именно тут!\nDATA_ROOT = \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/\"\nTRAIN_IMAGES = os.path.join(DATA_ROOT, \"train_images\")\nTRAIN_CSV = os.path.join(DATA_ROOT, \"train.csv\")\nCOORDS_CSV = os.path.join(DATA_ROOT, \"train_label_coordinates.csv\")\nDESC_CSV = os.path.join(DATA_ROOT, \"train_series_descriptions.csv\")\n\n# Загружаем основные таблицы\nprint(\"Loading CSVs...\")\ndf_train = pd.read_csv(TRAIN_CSV)\ndf_coords = pd.read_csv(COORDS_CSV)\ndf_desc = pd.read_csv(DESC_CSV)\n\nprint(f\"Train shape: {df_train.shape}\")\nprint(\"Columns in train.csv:\", df_train.columns.tolist()[:10], \"...\")  # первые 10 для примера\n\n# Простая проверка наличия хотя бы одного study\nsample_study_id = df_train[\"study_id\"].iloc[0]\nprint(f\"\\nSample study_id: {sample_study_id}\")\n\n# Проверяем, существуют ли изображения для этого study\nstudy_path = os.path.join(TRAIN_IMAGES, str(sample_study_id))\nif not os.path.exists(study_path):\n    print(\"!!! CRITICAL: Папка с изображениями не найдена !!!\")\n    print(\"Проверь путь:\", study_path)\nelse:\n    all_dcm_files = []\n    for root, _, files in os.walk(study_path):\n        for f in files:\n            if f.lower().endswith('.dcm'):\n                all_dcm_files.append(os.path.join(root, f))\n    print(f\"Найдено DICOM-файлов для study {sample_study_id}: {len(all_dcm_files)}\")\n\n# Проверка меток — должны быть только 3 варианта или NaN\nseverity_cols = [c for c in df_train.columns if c not in [\"study_id\"]]\nprint(\"\\nУникальные значения в метках (должно быть normal_mild / moderate / severe / NaN):\")\nfor col in severity_cols[:5]:  # первые 5 для скорости\n    print(f\"{col}: {df_train[col].unique()}\")\n\n# Mapping меток в числа (0,1,2)\nseverity_map = {\n    \"normal_mild\": 0,\n    \"moderate\": 1,\n    \"severe\": 2\n}\n\n# Простой тест: берём одну строку и преобразуем\nsample_row = df_train.iloc[0]\nlabels = sample_row[severity_cols].map(severity_map).fillna(0).values.astype(np.float32)\nprint(\"\\nПример меток после преобразования (первые 10):\", labels[:10])\nprint(\"Всего меток должно быть 25:\", len(labels) == 25)\n\n# Тестовая загрузка одного изображения (самая частая точка отказа)\ntry:\n    # Берём первый попавшийся dicom из study\n    if all_dcm_files:\n        test_path = all_dcm_files[0]\n        ds = pydicom.dcmread(test_path)\n        img = ds.pixel_array.astype(np.float32)\n        print(f\"Успешно загружено изображение: {test_path}\")\n        print(\"Размер:\", img.shape)\n        \n        # Показываем картинку\n        plt.figure(figsize=(6,6))\n        plt.imshow(img, cmap='gray')\n        plt.title(\"Test image\")\n        plt.axis('off')\n        plt.show()\n    else:\n        print(\"Нет dicom-файлов → дальше не пойдём\")\nexcept Exception as e:\n    print(\"!!! Ошибка при чтении DICOM !!!\")\n    print(e)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T08:54:48.206264Z","iopub.execute_input":"2026-01-18T08:54:48.206967Z","iopub.status.idle":"2026-01-18T08:54:48.545081Z","shell.execute_reply.started":"2026-01-18T08:54:48.206939Z","shell.execute_reply":"2026-01-18T08:54:48.544414Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def map_severity(x):\n    if pd.isna(x):\n        return 0\n    x = str(x).strip()\n    if 'Normal' in x or 'Mild' in x:\n        return 0\n    if 'Moderate' in x:\n        return 1\n    if 'Severe' in x:\n        return 2\n    return 0  # fallback\n\n# Применяй так:\nlabels = sample_row[severity_cols].apply(map_severity).values.astype(np.float32)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T08:56:29.837184Z","iopub.execute_input":"2026-01-18T08:56:29.837535Z","iopub.status.idle":"2026-01-18T08:56:29.843806Z","shell.execute_reply.started":"2026-01-18T08:56:29.837512Z","shell.execute_reply":"2026-01-18T08:56:29.843219Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 1. Добавляем тип серии в координаты\ndf_coords = df_coords.merge(df_desc[['study_id', 'series_id', 'series_description']], \n                           on=['study_id', 'series_id'], how='left')\n\n# 2. Создаём уникальный ключ для каждой аннотации\ndf_coords['condition_level'] = df_coords['condition'].str.lower() + '_' + df_coords['level']\n\n# 3. Для каждого study-level-condition нужно найти ближайший instance_number по z-координате\n# (это упрощённо — позже можно улучшить)\n\n# Пока берём самый простой вариант — группируем и берём один снимок на уровень (позже улучшим)\n# Но для начала давай хотя бы свяжем координаты с путями\n\nprint(\"Пример координат:\")\nprint(df_coords.head(8))\nprint(\"\\nУникальные series_description:\", df_coords['series_description'].unique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T08:56:38.240719Z","iopub.execute_input":"2026-01-18T08:56:38.241264Z","iopub.status.idle":"2026-01-18T08:56:38.283738Z","shell.execute_reply.started":"2026-01-18T08:56:38.241237Z","shell.execute_reply":"2026-01-18T08:56:38.283164Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Нормализуем названия условий под колонки train.csv\ndf_coords['condition'] = df_coords['condition'].str.lower().str.replace(' ', '_')\ndf_coords['condition_level'] = df_coords['condition'] + '_' + df_coords['level'].str.lower().str.replace('/', '_')\n\n# ----------------------------------------------------------------------\n# Функция поиска ближайшего снимка по instance_number\ndef get_closest_image_path(study_id, series_id, target_instance):\n    series_dir = os.path.join(TRAIN_IMAGES, str(study_id), str(series_id))\n    if not os.path.exists(series_dir):\n        return None\n    \n    dcm_files = glob.glob(os.path.join(series_dir, \"*.dcm\"))\n    if not dcm_files:\n        return None\n    \n    instances = []\n    for f in dcm_files:\n        try:\n            inst = int(os.path.basename(f).split('.')[0])\n            instances.append((inst, f))\n        except:\n            continue\n    \n    if not instances:\n        return None\n    \n    instances.sort(key=lambda x: abs(x[0] - target_instance))\n    return instances[0][1]  # самый близкий снимок\n\n# ----------------------------------------------------------------------\n# Тест — смотрим, что получается на sample study\nsample_study = 4003253\nprint(f\"\\nТест для study {sample_study}:\\n\")\n\nsample_rows = df_coords[df_coords['study_id'] == sample_study]\n\nfor _, row in sample_rows.iterrows():\n    path = get_closest_image_path(\n        row['study_id'],\n        row['series_id'],\n        row['instance_number']\n    )\n    if path:\n        print(f\"{row['condition_level']:<45} | {row['series_description']:<18} | inst {row['instance_number']:3d} → {os.path.basename(path)}\")\n    else:\n        print(f\"{row['condition_level']:<45} | НЕ НАЙДЕН\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T08:58:56.722458Z","iopub.execute_input":"2026-01-18T08:58:56.723048Z","iopub.status.idle":"2026-01-18T08:58:56.824559Z","shell.execute_reply.started":"2026-01-18T08:58:56.723020Z","shell.execute_reply":"2026-01-18T08:58:56.823976Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Mapping severity (исправленный, с учётом 'Normal/Mild')\nseverity_map = {\n    'Normal/Mild': 0,\n    'Moderate': 1,\n    'Severe': 2\n}\n\n# Загружаем train.csv один раз\ndf_train = pd.read_csv(os.path.join(DATA_ROOT, 'train.csv'))\n\n# Создаём финальный датафрейм для обучения\nimage_label_df = []\n\nfor _, row in tqdm(df_coords.iterrows(), total=len(df_coords)):\n    study_id = row['study_id']\n    path = get_closest_image_path(\n        study_id,\n        row['series_id'],\n        row['instance_number']\n    )\n    \n    if not path:\n        continue\n    \n    # Получаем все 25 меток для этого study\n    study_labels = df_train[df_train['study_id'] == study_id].iloc[0]\n    labels = []\n    for col in df_train.columns[1:]:  # все кроме study_id\n        val = study_labels[col]\n        labels.append(severity_map.get(val, 0))  # NaN или неизвестное → 0\n    \n    image_label_df.append({\n        'study_id': study_id,\n        'image_path': path,\n        'series_description': row['series_description'],\n        'condition_level': row['condition_level'],\n        'labels': np.array(labels, dtype=np.float32)  # все 25 меток\n    })\n\n# Собираем в датафрейм\ndf_dataset = pd.DataFrame(image_label_df)\n\nprint(f\"Собрано изображений с метками: {len(df_dataset)}\")\nprint(df_dataset[['image_path', 'series_description', 'condition_level']].head(10))\n\n# Проверка распределения серий\nprint(\"\\nРаспределение по типам серий:\")\nprint(df_dataset['series_description'].value_counts())\n\n# Проверка, что метки не все нули\nsample_labels = df_dataset['labels'].iloc[0]\nprint(\"\\nПример меток для первого изображения (не должно быть все 0):\")\nprint(sample_labels)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T08:59:29.406079Z","iopub.execute_input":"2026-01-18T08:59:29.406418Z","iopub.status.idle":"2026-01-18T09:02:28.049102Z","shell.execute_reply.started":"2026-01-18T08:59:29.406393Z","shell.execute_reply":"2026-01-18T09:02:28.048346Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Простая функция загрузки и предобработки изображения ---\ndef load_image(path):\n    try:\n        ds = pydicom.dcmread(path)\n        img = ds.pixel_array\n        \n        # Нормализация и приведение к 3 каналам (многие модели требуют RGB)\n        if len(img.shape) == 2:  # grayscale\n            img = (img - img.min()) / (img.max() - img.min() + 1e-6) * 255.0\n            img = np.uint8(img)\n            img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)\n        else:\n            # Если уже 3 канала — просто нормализуем\n            img = img.astype(np.float32)\n            img = (img - img.min()) / (img.max() - img.min() + 1e-6) * 255.0\n            img = img.astype(np.uint8)\n        \n        # Ресайз до 512×512 (стандарт для EfficientNet в RSNA)\n        img = cv2.resize(img, (512, 512))\n        \n        return img\n    except Exception as e:\n        print(f\"Ошибка загрузки {path}: {e}\")\n        return np.zeros((512, 512, 3), dtype=np.uint8)\n\n# --- Трансформации ---\ntrain_transform = A.Compose([\n    A.Resize(512, 512),\n    A.HorizontalFlip(p=0.5),\n    A.VerticalFlip(p=0.2),\n    A.RandomRotate90(p=0.3),\n    A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=15, p=0.5),\n    A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.5),\n    A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),  # ImageNet stats\n    ToTensorV2(),\n])\n\nvalid_transform = A.Compose([\n    A.Resize(512, 512),\n    A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n    ToTensorV2(),\n])\n\n# --- Класс Dataset ---\nclass RSNADataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df.reset_index(drop=True)\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = row['image_path']\n        \n        img = load_image(img_path)\n        \n        if self.transform:\n            augmented = self.transform(image=img)\n            img = augmented['image']\n        \n        labels = row['labels']  # уже np.array float32\n        \n        return img, torch.from_numpy(labels)\n\n# --- Тест датасета на первых 3 примерах ---\nprint(\"Тест Dataset...\\n\")\n\ntest_ds = RSNADataset(df_dataset.head(100), transform=train_transform)  # берём первые 100 для скорости\n\nfor i in range(3):\n    img_tensor, lbl = test_ds[i]\n    print(f\"Пример {i+1}:\")\n    print(f\"  Image shape: {img_tensor.shape}\")\n    print(f\"  Labels shape: {lbl.shape}, sum: {lbl.sum().item():.1f}\")\n    print(f\"  Labels sample: {lbl[:10]}\")\n    print(\"-\" * 60)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T09:03:00.053471Z","iopub.execute_input":"2026-01-18T09:03:00.053768Z","iopub.status.idle":"2026-01-18T09:03:00.232512Z","shell.execute_reply.started":"2026-01-18T09:03:00.053742Z","shell.execute_reply":"2026-01-18T09:03:00.231751Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Модель (EfficientNet-B4 — хороший баланс скорости и качества)\nclass RSNAModel(nn.Module):\n    def __init__(self, model_name='efficientnet_b4', n_classes=25, pretrained=True):\n        super().__init__()\n        self.backbone = timm.create_model(\n            model_name,\n            pretrained=pretrained,\n            num_classes=0,           # убираем оригинальную голову\n            global_pool=''           # оставляем feature map\n        )\n        in_features = self.backbone.num_features\n        \n        self.head = nn.Sequential(\n            nn.AdaptiveAvgPool2d(1),\n            nn.Flatten(),\n            nn.Dropout(0.4),\n            nn.Linear(in_features, n_classes)\n        )\n\n    def forward(self, x):\n        features = self.backbone(x)\n        return self.head(features)\n\n# ------------------------------------------------------------\n# Создаём модель и переносим на GPU\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(\"Using device:\", device)\n\nmodel = RSNAModel(model_name='efficientnet_b4', n_classes=25, pretrained=True)\nmodel = model.to(device)\n\n# ------------------------------------------------------------\n# Loss, optimizer, scheduler, scaler\ncriterion = nn.BCEWithLogitsLoss()  # т.к. multi-label, но метки 0/1/2 → преобразуем\noptimizer = optim.AdamW(model.parameters(), lr=1e-4, weight_decay=1e-5)\nscaler = GradScaler()\n\n# Простой scheduler (можно потом улучшить)\nscheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=5)\n\n# ------------------------------------------------------------\n# DataLoader (маленький для теста)\ntrain_ds = RSNADataset(df_dataset, transform=train_transform)\ntrain_loader = DataLoader(\n    train_ds,\n    batch_size=16,           # подбери под память GPU (8–32 обычно)\n    shuffle=True,\n    num_workers=2,\n    pin_memory=True,\n    drop_last=True\n)\n\n# ------------------------------------------------------------\n# Тестовая эпоха обучения (только 1 эпоха, чтобы проверить)\nprint(\"\\n=== Запуск тестовой эпохи (1 эпоха) ===\\n\")\n\n# ------------------------------------------------------------\n# Тестовая эпоха обучения (1 эпоха, исправленная версия)\n\nprint(\"\\n=== Запуск исправленной тестовой эпохи ===\\n\")\n\nmodel.train()\nepoch_loss = 0.0\nn_batches = 0\n\nfor batch_idx, (images, labels) in enumerate(tqdm(train_loader, desc=\"Training\")):\n    images = images.to(device)\n    labels = labels.to(device)  # [batch, 25]\n    \n    # Самое важное исправление: приводим метки к [0, 1] диапазону\n    labels = labels.float() / 2.0                     # 0→0.0, 1→0.5, 2→1.0\n    # Вариант с label smoothing (рекомендуется):\n    # labels = (labels.float() / 2.0) * 0.9 + 0.05\n    \n    optimizer.zero_grad()\n    \n    with autocast():\n        outputs = model(images)           # [batch, 25] logits\n        loss = criterion(outputs, labels)\n    \n    scaler.scale(loss).backward()\n    scaler.step(optimizer)\n    scaler.update()\n    \n    epoch_loss += loss.item()\n    n_batches += 1\n    \n    if batch_idx % 50 == 0:\n        current_lr = optimizer.param_groups[0]['lr']\n        print(f\"Batch {batch_idx:4d} | Loss: {loss.item():.5f} | LR: {current_lr:.2e}\")\n    \n    # Останавливаемся раньше для теста\n    if batch_idx >= 300:\n        break\n\nif n_batches > 0:\n    avg_loss = epoch_loss / n_batches\n    print(f\"\\nСредний лосс за {n_batches} батчей: {avg_loss:.5f}\")\nelse:\n    print(\"Не удалось обработать ни одного батча\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T09:05:04.136474Z","iopub.execute_input":"2026-01-18T09:05:04.137283Z","iopub.status.idle":"2026-01-18T09:08:05.533845Z","shell.execute_reply.started":"2026-01-18T09:05:04.137249Z","shell.execute_reply":"2026-01-18T09:08:05.533044Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# После цикла обучения добавь валидацию на всём df_dataset (для грубой проверки)\nmodel.eval()\nval_loss = 0.0\nval_batches = 0\n\nval_loader = DataLoader(\n    RSNADataset(df_dataset, transform=valid_transform),\n    batch_size=16,\n    shuffle=False,\n    num_workers=2,\n    pin_memory=True\n)\n\nwith torch.no_grad():\n    for images, labels in tqdm(val_loader, desc=\"Quick Validation\"):\n        images = images.to(device)\n        labels = labels.to(device).float() / 2.0\n        \n        with autocast():\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n        val_loss += loss.item()\n        val_batches += 1\n\navg_val_loss = val_loss / val_batches\nprint(f\"Быстрая валидация на всём датасете: {avg_val_loss:.5f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T09:09:50.810492Z","iopub.execute_input":"2026-01-18T09:09:50.811411Z","iopub.status.idle":"2026-01-18T09:18:19.038679Z","shell.execute_reply.started":"2026-01-18T09:09:50.811375Z","shell.execute_reply":"2026-01-18T09:18:19.037889Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"torch.save(model.state_dict(), \"efficientnet_b4_rsna_epoch1.pth\")\nprint(\"Модель сохранена!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T09:19:36.373357Z","iopub.execute_input":"2026-01-18T09:19:36.373782Z","iopub.status.idle":"2026-01-18T09:19:36.552726Z","shell.execute_reply.started":"2026-01-18T09:19:36.373737Z","shell.execute_reply":"2026-01-18T09:19:36.551900Z"}},"outputs":[],"execution_count":null}]}