{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport sys\nimport glob\nimport json\nimport random\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport cv2\nfrom PIL import Image\nfrom tqdm import tqdm\nfrom sklearn.model_selection import StratifiedGroupKFold\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nimport torchvision.models as models\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda.amp import autocast, GradScaler\n\nimport timm\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport pydicom\n\nprint(\"Torch version:\", torch.__version__)\nprint(\"CUDA available:\", torch.cuda.is_available())  # Должен вывести True, если GPU on\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(\"Device:\", device)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Пути — самые частые ошибки именно тут!\nDATA_ROOT = \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/\"\nTRAIN_IMAGES = os.path.join(DATA_ROOT, \"train_images\")\nTRAIN_CSV = os.path.join(DATA_ROOT, \"train.csv\")\nCOORDS_CSV = os.path.join(DATA_ROOT, \"train_label_coordinates.csv\")\nDESC_CSV = os.path.join(DATA_ROOT, \"train_series_descriptions.csv\")\n\n# Загружаем основные таблицы\nprint(\"Loading CSVs...\")\ndf_train = pd.read_csv(TRAIN_CSV)\ndf_coords = pd.read_csv(COORDS_CSV)\ndf_desc = pd.read_csv(DESC_CSV)\n\n\nprint(f\"Train shape: {df_train.shape}\")\nprint(\"Columns in train.csv:\", df_train.columns.tolist()[:10], \"...\")  # первые 10 для примера\n\n# Простая проверка наличия хотя бы одного study\nsample_study_id = df_train[\"study_id\"].iloc[0]\nprint(f\"\\nSample study_id: {sample_study_id}\")\n\n# Проверяем, существуют ли изображения для этого study\nstudy_path = os.path.join(TRAIN_IMAGES, str(sample_study_id))\nif not os.path.exists(study_path):\n    print(\"!!! CRITICAL: Папка с изображениями не найдена !!!\")\n    print(\"Проверь путь:\", study_path)\nelse:\n    all_dcm_files = []\n    for root, _, files in os.walk(study_path):\n        for f in files:\n            if f.lower().endswith('.dcm'):\n                all_dcm_files.append(os.path.join(root, f))\n    print(f\"Найдено DICOM-файлов для study {sample_study_id}: {len(all_dcm_files)}\")\n\n# Проверка меток — должны быть только 3 варианта или NaN\nseverity_cols = [c for c in df_train.columns if c not in [\"study_id\"]]\nprint(\"\\nУникальные значения в метках (должно быть normal_mild / moderate / severe / NaN):\")\nfor col in severity_cols[:5]:  # первые 5 для скорости\n    print(f\"{col}: {df_train[col].unique()}\")\n\n# Mapping меток в числа (0,1,2)\nseverity_map = {\n    \"normal_mild\": 0,\n    \"moderate\": 1,\n    \"severe\": 2\n}\n\n# Простой тест: берём одну строку и преобразуем\nsample_row = df_train.iloc[0]\nlabels = sample_row[severity_cols].map(severity_map).fillna(0).values.astype(np.float32)\nprint(\"\\nПример меток после преобразования (первые 10):\", labels[:10])\nprint(\"Всего меток должно быть 25:\", len(labels) == 25)\n\n# Тестовая загрузка одного изображения (самая частая точка отказа)\ntry:\n    # Берём первый попавшийся dicom из study\n    if all_dcm_files:\n        test_path = all_dcm_files[0]\n        ds = pydicom.dcmread(test_path)\n        img = ds.pixel_array.astype(np.float32)\n        print(f\"Успешно загружено изображение: {test_path}\")\n        print(\"Размер:\", img.shape)\n        \n        # Показываем картинку\n        plt.figure(figsize=(6,6))\n        plt.imshow(img, cmap='gray')\n        plt.title(\"Test image\")\n        plt.axis('off')\n        plt.show()\n    else:\n        print(\"Нет dicom-файлов → дальше не пойдём\")\nexcept Exception as e:\n    print(\"!!! Ошибка при чтении DICOM !!!\")\n    print(e)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom scipy import interpolate\n\ndf_coords = df_coords.merge(df_desc[['study_id', 'series_id', 'series_description']],\n                            on=['study_id', 'series_id'], how='left')\ndf_coords['condition_level'] = df_coords['condition'].str.lower() + '_' + df_coords['level']\n\nprint(\"Пример координат:\")\nprint(df_coords.head(8))\nprint(\"\\nУникальные series_description:\", df_coords['series_description'].unique())\n\n# Preparing df_labels from df_train\n# Normalize column names if needed\ndf_train.columns = [col.lower().replace('/', '_') for col in df_train.columns]\n\ncondition_cols = [col for col in df_train.columns if col != 'study_id']\ndf_labels = pd.melt(df_train, id_vars=['study_id'], value_vars=condition_cols,\n                    var_name='condition_level', value_name='severity')\ndf_labels['condition_level'] = df_labels['condition_level'].str.lower()\n\n# Extract base_condition and level_str\ndef extract_base_and_level(condition_level):\n    parts = condition_level.split('_')\n    base = '_'.join(parts[:-2])\n    level = '_'.join(parts[-2:])\n    return base, level\n\ndf_labels[['base_condition', 'level_str']] = df_labels['condition_level'].apply(\n    lambda x: pd.Series(extract_base_and_level(x))\n)\nlevel_map = {'l1_l2': 1, 'l2_l3': 2, 'l3_l4': 3, 'l4_l5': 4, 'l5_s1': 5}\ndf_labels['level_num'] = df_labels['level_str'].map(level_map)\n\n# Map to series_description\ncondition_to_series = {\n    'spinal_canal_stenosis': 'Sagittal T2/STIR',\n    'left_neural_foraminal_narrowing': 'Sagittal T1',\n    'right_neural_foraminal_narrowing': 'Sagittal T1',\n    'left_subarticular_stenosis': 'Axial T2',\n    'right_subarticular_stenosis': 'Axial T2',\n}\ndf_labels['series_description'] = df_labels['base_condition'].map(condition_to_series)\n\n# Merge with df_desc to get series_id for all entries\ndf_all = df_labels.merge(df_desc[['study_id', 'series_description', 'series_id']],\n                         on=['study_id', 'series_description'], how='left')\n\n# Prepare df_coords for merge\ndf_coords['condition'] = df_coords['condition'].str.lower().str.replace(' ', '_')\ndf_coords['level_str'] = df_coords['level'].str.lower().str.replace('/', '_')\ndf_coords['base_condition'] = df_coords['condition']\ndf_coords['condition_level'] = df_coords['base_condition'] + '_' + df_coords['level_str']\n\n# Merge coordinates where available\ndf_all = df_all.merge(df_coords[['study_id', 'condition_level', 'instance_number', 'x', 'y']],\n                      on=['study_id', 'condition_level'], how='left')\n\n# Compute global averages for fallback\navg_instance = df_coords.groupby('base_condition')['instance_number'].mean().round().astype(int)\navg_x = df_coords.groupby('base_condition')['x'].mean()\navg_y = df_coords.groupby('base_condition')['y'].mean()\n\n# Function to interpolate instance_number per study and base_condition\ndef interpolate_group(group):\n    group = group.sort_values('level_num')\n    known = group[~group['instance_number'].isna()]\n    \n    if len(known) == 0:\n        bc = group['base_condition'].iloc[0]\n        group['instance_number'] = avg_instance.get(bc, 10)  # Default to 10 if no global avg\n        return group\n    \n    if len(known) == 1:\n        group['instance_number'] = known['instance_number'].iloc[0]\n        return group\n    \n    # Linear interpolation with extrapolation\n    f = interpolate.interp1d(known['level_num'], known['instance_number'],\n                             kind='linear', fill_value='extrapolate')\n    mask = group['instance_number'].isna()\n    group.loc[mask, 'instance_number'] = f(group.loc[mask, 'level_num'])\n    group['instance_number'] = group['instance_number'].round().astype(int)\n    \n    return group\n\ndf_all.dropna(inplace=True)\n\ndf_all = df_all.groupby(['study_id', 'base_condition']).apply(interpolate_group).reset_index(drop=True)\n\n# Fill missing x, y with study-specific mean, then global\ndf_all['x'] = df_all.groupby(['study_id', 'base_condition'])['x'].transform(lambda g: g.fillna(g.mean()))\ndf_all['y'] = df_all.groupby(['study_id', 'base_condition'])['y'].transform(lambda g: g.fillna(g.mean()))\ndf_all['x'] = df_all.apply(lambda row: avg_x.get(row['base_condition'], 256) if pd.isna(row['x']) else row['x'], axis=1)\ndf_all['y'] = df_all.apply(lambda row: avg_y.get(row['base_condition'], 256) if pd.isna(row['y']) else row['y'], axis=1)\n\ndf_all['series_id'] = df_all['series_id'].astype('int64')\ndf_all['instance_number'] = df_all['instance_number'].astype('int64')\n\n# Add image paths (assuming standard Kaggle directory structure)\ndf_all['image_path'] = (\n    '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/' +\n    df_all['study_id'].astype(str) + '/' +\n    df_all['series_id'].astype(str) + '/' +\n    df_all['instance_number'].astype(str) + '.dcm'\n)\n\n# Now df_all has one row per condition_level per study, with approximated instance/x/y for normal/mild\nprint(\"Пример подготовленных данных для study_id 100206310:\")\nprint(df_all[df_all['study_id'] == 100206310].head(10))\n\n# Для модели: используйте df_all для создания датасета.\n# Вход: загрузите DICOM по image_path, нормализуйте, crop вокруг (x, y) например 128x128.\n# Выход: one-hot severity (Normal/Mild -> 0, Moderate -> 1, Severe -> 2).\n# Обучите отдельные модели для каждого series_description или base_condition.\n# Для улучшения: используйте pydicom для извлечения реальных z-координат и более точной интерполяции по физическим позициям.\n# Альтернатива: обучите модель детекции ключевых точек (keypoint detection) на аннотированных сэмплах для автоматического нахождения уровней.","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# df_all.iloc[0].image_path\nimport os\n\ndef build_path(row):\n    path = f\"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/{row.study_id}/{row.series_id}/{row.instance_number}.dcm\"\n    return path if os.path.exists(path) else None\n\ndf_all['image_path'] = df_all.apply(build_path, axis=1)\ndf_all","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_all2 = df_all.copy()\ndf_all2 = df_all2.dropna()\ndf_all2['series_id'] = df_all2['series_id'].astype('int64')\ndf_all2\ndf_all2['instance_number'] = df_all2['instance_number'].astype('int64')\n\nseverity_map = {\n    'Normal/Mild': 0,\n    'Moderate': 1,\n    'Severe': 2\n}\ndf_all2['label'] = df_all2['severity'].map(severity_map)\n\nmodality_group = {\n    'Sagittal T1': 'sagittal',\n    'Sagittal T2/STIR': 'sagittal',\n    'Axial T2': 'axial'\n}\ndf_all2['modality_group'] = df_all2['series_description'].map(modality_group)\ndf_all2","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n# Предположим, твой датафрейм называется df\ndf_sag = df_all2[(df_all2['modality_group'] == 'sagittal') & \n            (df_all2['base_condition'] == 'spinal_canal_stenosis')]\n\n# Разделяем по study_id, чтобы один пациент был только в train или val\nstudy_ids = df_sag['study_id'].unique()\ntrain_ids, val_ids = train_test_split(study_ids, test_size=0.2, random_state=42)\n\ndf_train = df_sag[df_sag['study_id'].isin(train_ids)]\ndf_val   = df_sag[df_sag['study_id'].isin(val_ids)]\n\nprint(\"Train size:\", len(df_train))\nprint(\"Val size:\", len(df_val))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\nimport numpy as np\nimport cv2\nfrom PIL import Image\n\nclass Lumbar25DDataset(Dataset):\n    def __init__(self, df, levels=['l1_l2','l2_l3','l3_l4','l4_l5','l5_s1'], crop_size=224, augment=False):\n        self.levels = levels\n        self.crop_size = crop_size\n        self.augment = augment\n\n        # Фильтруем только sagittal\n        df = df[df['modality_group']=='sagittal']\n\n        # Группируем по study_id и base_condition\n        self.groups = []\n        grouped = df.groupby(['study_id', 'base_condition'])\n        for (study_id, base_condition), g in grouped:\n            # Проверяем, что все уровни есть\n            if all(l in g['level_str'].values for l in levels):\n                # Сортируем по level_num\n                g_sorted = g.sort_values('level_num')\n                self.groups.append(g_sorted)\n\n        # Трансформации\n        self.transform = transforms.Compose([\n            transforms.Resize((crop_size, crop_size)),\n            transforms.ToTensor()\n        ])\n\n    def __len__(self):\n        return len(self.groups)\n\n    def __getitem__(self, idx):\n        g = self.groups[idx]\n        imgs = []\n        for _, row in g.iterrows():\n            path = row['image_path']\n            # Загружаем DICOM\n            ds = pydicom.dcmread(path)\n            img = ds.pixel_array.astype(np.float32)\n            img = (img - img.min()) / (img.max() - img.min() + 1e-5)  # нормализация 0-1\n            img = np.stack([img]*3, axis=-1)  # 3 канала\n            img = Image.fromarray((img*255).astype(np.uint8))\n            img = self.transform(img)  # [3,H,W]\n            imgs.append(img)\n\n        # Склеиваем уровни по каналам\n        imgs = torch.cat(imgs, dim=0)  # [5*3=15, H, W]\n\n        # Метка (берём первый row — все имеют один label для base_condition)\n        label = g['label'].iloc[0]\n        return imgs, torch.tensor(label, dtype=torch.long)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Создаём датасеты\ntrain_dataset = Lumbar25DDataset(df_train, augment=True)\nval_dataset = Lumbar25DDataset(df_val, augment=False)\n\nfrom torch.utils.data import DataLoader\n\ntrain_loader = DataLoader(train_dataset, batch_size=8, shuffle=True, num_workers=2)\nval_loader = DataLoader(val_dataset, batch_size=8, shuffle=False, num_workers=2)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for x, y in train_loader:\n    print(x.shape)  # должно быть [B, 15, 224, 224]\n    print(y.shape)  # [B]\n    break","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torchvision.models import resnet18\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\n\n# Инициализация ResNet18\nmodel = resnet18(weights=None)  # или weights='IMAGENET1K_V1', если хочешь предобученные\nmodel.conv1 = nn.Conv2d(\n    in_channels=15,  # вместо стандартных 3 каналов\n    out_channels=64,\n    kernel_size=7,\n    stride=2,\n    padding=3,\n    bias=False\n)\nnum_classes = 3  # например, Normal/Mild, Moderate, Severe — под твои severity\nmodel.fc = nn.Linear(model.fc.in_features, num_classes)\n\nmodel = model.to(device)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch.optim as optim\n\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=1e-4)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_epochs = 2\n\nfor epoch in range(num_epochs):\n    model.train()\n    running_loss = 0.0\n    running_corrects = 0\n    total = 0\n\n    loop = tqdm(train_loader, desc=f\"Epoch {epoch+1}/{num_epochs} [Train]\")\n    for images, labels in loop:\n        images, labels = images.to(device), labels.to(device)\n        optimizer.zero_grad()\n        outputs = model(images)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n\n        running_loss += loss.item() * images.size(0)\n        preds = outputs.argmax(dim=1)\n        running_corrects += (preds == labels).sum().item()\n        total += labels.size(0)\n\n        loop.set_postfix(\n            loss=running_loss / total,\n            acc=running_corrects / total\n        )\n\n    epoch_loss = running_loss / total\n    epoch_acc = running_corrects / total\n\n    # Валидация\n    model.eval()\n    val_loss = 0.0\n    val_corrects = 0\n    val_total = 0\n    with torch.no_grad():\n        loop_val = tqdm(val_loader, desc=f\"Epoch {epoch+1}/{num_epochs} [Val]  \")\n        for images, labels in loop_val:\n            images, labels = images.to(device), labels.to(device)\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n\n            val_loss += loss.item() * images.size(0)\n            preds = outputs.argmax(dim=1)\n            val_corrects += (preds == labels).sum().item()\n            val_total += labels.size(0)\n\n            loop_val.set_postfix(\n                loss=val_loss / val_total,\n                acc=val_corrects / val_total\n            )\n\n    print(f\"Epoch {epoch+1} Summary: Train Loss {epoch_loss:.4f}, Train Acc {epoch_acc:.4f}, Val Loss {val_loss/val_total:.4f}, Val Acc {val_corrects/val_total:.4f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Модельді тексеру","metadata":{}},{"cell_type":"code","source":"import torch\nimport numpy as np\nimport cv2\nimport pydicom\n\ndef predict_25d(model, df_level, device, num_slices=15, img_size=224):\n    \"\"\"\n    model: обученная 2.5D модель (ResNet18 с входом [B, C=D, H, W])\n    df_level: датафрейм с одним уровнем (study_id + level_str)\n    device: 'cuda' или 'cpu'\n    num_slices: сколько срезов брать\n    img_size: размер для ресайза\n    \"\"\"\n    # Сортируем срезы по instance_number\n    df_level = df_level.sort_values('instance_number')\n    slices = df_level['image_path'].tolist()\n    \n    # Если меньше num_slices — дублируем крайние\n    if len(slices) < num_slices:\n        while len(slices) < num_slices:\n            slices.append(slices[-1])\n    elif len(slices) > num_slices:\n        start = 0  # можно random.randint(0, len(slices)-num_slices)\n        slices = slices[start:start+num_slices]\n    \n    imgs = []\n    for path in slices:\n        dicom = pydicom.dcmread(path)\n        img = dicom.pixel_array.astype(np.float32)\n        img = cv2.resize(img, (img_size, img_size))\n        img = (img - img.min()) / (img.max() - img.min())\n        imgs.append(img)\n    \n    # Формируем тензор [1, D, H, W] → для ResNet [1, C=D, H, W]\n    x = np.stack(imgs, axis=0)\n    x = torch.tensor(x).float().unsqueeze(0).to(device)  # [1, D, H, W]\n    \n    # ResNet ждет [B, C, H, W], D как каналы\n    x = x.permute(0,1,2,3)  # уже [1, D, H, W]\n    \n    model.eval()\n    with torch.no_grad():\n        outputs = model(x)\n        probs = torch.softmax(outputs, dim=1)\n        pred_class = torch.argmax(probs, dim=1)\n    \n    return pred_class.item(), probs.cpu().numpy()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Пример для одного уровня\nstudy_id = 4003253\nlevel_str = 'l3_l4'\ndf_level = df_all2[(df_all2['study_id']==study_id) & (df_all2['level_str']==level_str)]\n\npred_class, probs = predict_25d(model, df_level, device)\nprint(\"Predicted class:\", pred_class)\nprint(\"Probabilities:\", probs)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}