{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":6117,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":4600,"modelId":2797}],"dockerImageVersionId":30747,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import Libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport time\nimport random\nimport glob\nimport json\nimport collections\nfrom typing import List, Tuple, Dict\n\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\n\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as transforms\nfrom tqdm import tqdm\n\n\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-07-29T21:50:14.011511Z","iopub.execute_input":"2024-07-29T21:50:14.012126Z","iopub.status.idle":"2024-07-29T21:50:14.018861Z","shell.execute_reply.started":"2024-07-29T21:50:14.012083Z","shell.execute_reply":"2024-07-29T21:50:14.017927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:14.020524Z","iopub.execute_input":"2024-07-29T21:50:14.020913Z","iopub.status.idle":"2024-07-29T21:50:14.031684Z","shell.execute_reply.started":"2024-07-29T21:50:14.020866Z","shell.execute_reply":"2024-07-29T21:50:14.030689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Data","metadata":{}},{"cell_type":"code","source":"# Constants\nTRAIN_PATH = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/'","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:14.032846Z","iopub.execute_input":"2024-07-29T21:50:14.033167Z","iopub.status.idle":"2024-07-29T21:50:14.042193Z","shell.execute_reply.started":"2024-07-29T21:50:14.033135Z","shell.execute_reply":"2024-07-29T21:50:14.041363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data() -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame]:\n    \"\"\"Load and return all necessary dataframes.\"\"\"\n    train = pd.read_csv(os.path.join(TRAIN_PATH, 'train.csv'))\n    label = pd.read_csv(os.path.join(TRAIN_PATH, 'train_label_coordinates.csv'))\n    train_desc = pd.read_csv(os.path.join(TRAIN_PATH, 'train_series_descriptions.csv'))\n    test_desc = pd.read_csv(os.path.join(TRAIN_PATH, 'test_series_descriptions.csv'))\n    sub = pd.read_csv(os.path.join(TRAIN_PATH, 'sample_submission.csv'))\n    return train, label, train_desc, test_desc, sub","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:14.043884Z","iopub.execute_input":"2024-07-29T21:50:14.044187Z","iopub.status.idle":"2024-07-29T21:50:14.052446Z","shell.execute_reply.started":"2024-07-29T21:50:14.044150Z","shell.execute_reply":"2024-07-29T21:50:14.051783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_image_paths(df: pd.DataFrame, data_dir: str) -> List[str]:\n    \"\"\"Generate image paths based on directory structure.\"\"\"\n    image_paths = []\n    for _, row in df.iterrows():\n        study_dir = os.path.join(data_dir, str(row['study_id']))\n        series_dir = os.path.join(study_dir, str(row['series_id']))\n        images = os.listdir(series_dir)\n        image_paths.extend([os.path.join(series_dir, img) for img in images])\n    return image_paths","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:14.053736Z","iopub.execute_input":"2024-07-29T21:50:14.054506Z","iopub.status.idle":"2024-07-29T21:50:14.066772Z","shell.execute_reply.started":"2024-07-29T21:50:14.054464Z","shell.execute_reply":"2024-07-29T21:50:14.065931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load data\ntrain, label, train_desc, test_desc, sub = load_data()","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:14.068084Z","iopub.execute_input":"2024-07-29T21:50:14.068393Z","iopub.status.idle":"2024-07-29T21:50:14.184033Z","shell.execute_reply.started":"2024-07-29T21:50:14.068367Z","shell.execute_reply":"2024-07-29T21:50:14.183247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display some data\nprint(\"Test Series Descriptions:\")\nprint(test_desc.head())\nprint(\"\\nTrain Data:\")\nprint(train.head())\nprint(\"\\nTrain Series Descriptions:\")\nprint(train_desc.head())","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:14.186525Z","iopub.execute_input":"2024-07-29T21:50:14.187427Z","iopub.status.idle":"2024-07-29T21:50:14.203625Z","shell.execute_reply.started":"2024-07-29T21:50:14.187391Z","shell.execute_reply":"2024-07-29T21:50:14.202693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generate image paths\ntrain_image_paths = generate_image_paths(train_desc, os.path.join(TRAIN_PATH, 'train_images'))\ntest_image_paths = generate_image_paths(test_desc, os.path.join(TRAIN_PATH, 'test_images'))","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:14.204890Z","iopub.execute_input":"2024-07-29T21:50:14.205235Z","iopub.status.idle":"2024-07-29T21:50:19.010957Z","shell.execute_reply.started":"2024-07-29T21:50:14.205201Z","shell.execute_reply":"2024-07-29T21:50:19.009803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"\\nNumber of train images: {len(train_image_paths)}\")\nprint(f\"Number of test images: {len(test_image_paths)}\")","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:19.012556Z","iopub.execute_input":"2024-07-29T21:50:19.012913Z","iopub.status.idle":"2024-07-29T21:50:19.018007Z","shell.execute_reply.started":"2024-07-29T21:50:19.012877Z","shell.execute_reply":"2024-07-29T21:50:19.017045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_desc)","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:19.019298Z","iopub.execute_input":"2024-07-29T21:50:19.019627Z","iopub.status.idle":"2024-07-29T21:50:19.030854Z","shell.execute_reply.started":"2024-07-29T21:50:19.019602Z","shell.execute_reply":"2024-07-29T21:50:19.029887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dicom_files(path_to_folder: str) -> List[str]:\n    \"\"\"Load DICOM files from a folder and sort them.\"\"\"\n    files = [os.path.join(path_to_folder, f) for f in os.listdir(path_to_folder) if f.endswith('.dcm')]\n    return sorted(files, key=lambda x: int(os.path.splitext(os.path.basename(x))[0].split('-')[-1]))","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:19.032071Z","iopub.execute_input":"2024-07-29T21:50:19.032355Z","iopub.status.idle":"2024-07-29T21:50:19.041332Z","shell.execute_reply.started":"2024-07-29T21:50:19.032325Z","shell.execute_reply":"2024-07-29T21:50:19.040296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display DICOM images\ndisplay_dicom_images(image_paths)","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:19.042355Z","iopub.execute_input":"2024-07-29T21:50:19.042623Z","iopub.status.idle":"2024-07-29T21:50:19.782201Z","shell.execute_reply.started":"2024-07-29T21:50:19.042599Z","shell.execute_reply":"2024-07-29T21:50:19.781141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing Data","metadata":{}},{"cell_type":"code","source":"def display_dicom_images(image_paths: List[str], num_images: int = 3):\n    \"\"\"Display a specified number of DICOM images.\"\"\"\n    fig, axes = plt.subplots(1, num_images, figsize=(5*num_images, 5))\n    for i, path in enumerate(image_paths[:num_images]):\n        ds = pydicom.dcmread(path)\n        ax = axes[i] if num_images > 1 else axes\n        ax.imshow(ds.pixel_array, cmap=plt.cm.bone)\n        ax.set_title(f\"Image {i+1}\")\n        ax.axis('off')\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:19.786507Z","iopub.execute_input":"2024-07-29T21:50:19.786864Z","iopub.status.idle":"2024-07-29T21:50:19.794858Z","shell.execute_reply.started":"2024-07-29T21:50:19.786833Z","shell.execute_reply":"2024-07-29T21:50:19.793966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_dicom_with_coordinates(image_paths: List[str], label_df: pd.DataFrame):\n    \"\"\"Display DICOM images along with coordinates.\"\"\"\n    fig, axes = plt.subplots(1, len(image_paths), figsize=(6*len(image_paths), 6))\n    axes = [axes] if len(image_paths) == 1 else axes\n    \n    # Group labels by study_id and series_id for faster lookup\n    label_dict = defaultdict(list)\n    for _, row in label_df.iterrows():\n        label_dict[(row['study_id'], row['series_id'])].append((row['x'], row['y']))\n    \n    for idx, path in enumerate(image_paths):\n        study_id = int(path.split('/')[-3])\n        series_id = int(path.split('/')[-2])\n        \n        ds = pydicom.dcmread(path)\n        axes[idx].imshow(ds.pixel_array, cmap='gray')\n        axes[idx].set_title(f\"Study ID: {study_id}, Series ID: {series_id}\")\n        axes[idx].axis('off')\n        \n        # Plot coordinates\n        for x, y in label_dict.get((study_id, series_id), []):\n            axes[idx].plot(x, y, 'ro', markersize=5)\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:19.796121Z","iopub.execute_input":"2024-07-29T21:50:19.796442Z","iopub.status.idle":"2024-07-29T21:50:19.807807Z","shell.execute_reply.started":"2024-07-29T21:50:19.796414Z","shell.execute_reply":"2024-07-29T21:50:19.806867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" # Constants\nTRAIN_PATH = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/'\nSTUDY_ID = \"100206310\"","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:19.809023Z","iopub.execute_input":"2024-07-29T21:50:19.809318Z","iopub.status.idle":"2024-07-29T21:50:19.816673Z","shell.execute_reply.started":"2024-07-29T21:50:19.809294Z","shell.execute_reply":"2024-07-29T21:50:19.815762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load label data\nlabel = pd.read_csv(os.path.join(TRAIN_PATH, 'train_label_coordinates.csv'))","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:19.817953Z","iopub.execute_input":"2024-07-29T21:50:19.818282Z","iopub.status.idle":"2024-07-29T21:50:19.898428Z","shell.execute_reply.started":"2024-07-29T21:50:19.818234Z","shell.execute_reply":"2024-07-29T21:50:19.897464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load DICOM images\nstudy_folder = os.path.join(TRAIN_PATH, 'train_images', STUDY_ID)\nimage_paths = []\nfor series_folder in os.listdir(study_folder):\n    series_folder_path = os.path.join(study_folder, series_folder)\n    dicom_files = load_dicom_files(series_folder_path)\n    if dicom_files:\n        image_paths.append(dicom_files[0])  # Add the first image from each series\n\n# Display DICOM images with coordinates\ndisplay_dicom_with_coordinates(image_paths, label)","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:19.899661Z","iopub.execute_input":"2024-07-29T21:50:19.899945Z","iopub.status.idle":"2024-07-29T21:50:23.986868Z","shell.execute_reply.started":"2024-07-29T21:50:19.899919Z","shell.execute_reply":"2024-07-29T21:50:23.985937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom typing import Dict, List","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:23.988155Z","iopub.execute_input":"2024-07-29T21:50:23.988476Z","iopub.status.idle":"2024-07-29T21:50:23.993561Z","shell.execute_reply.started":"2024-07-29T21:50:23.988446Z","shell.execute_reply":"2024-07-29T21:50:23.992385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Constants\nTRAIN_PATH = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/'","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:23.994854Z","iopub.execute_input":"2024-07-29T21:50:23.995187Z","iopub.status.idle":"2024-07-29T21:50:24.004815Z","shell.execute_reply.started":"2024-07-29T21:50:23.995160Z","shell.execute_reply":"2024-07-29T21:50:24.003819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data() -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame]:\n    \"\"\"Load and return all necessary dataframes.\"\"\"\n    train = pd.read_csv(os.path.join(TRAIN_PATH, 'train.csv'))\n    label = pd.read_csv(os.path.join(TRAIN_PATH, 'train_label_coordinates.csv'))\n    train_desc = pd.read_csv(os.path.join(TRAIN_PATH, 'train_series_descriptions.csv'))\n    test_desc = pd.read_csv(os.path.join(TRAIN_PATH, 'test_series_descriptions.csv'))\n    sub = pd.read_csv(os.path.join(TRAIN_PATH, 'sample_submission.csv'))\n    return train, label, train_desc, test_desc, sub","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:24.006103Z","iopub.execute_input":"2024-07-29T21:50:24.006485Z","iopub.status.idle":"2024-07-29T21:50:24.015336Z","shell.execute_reply.started":"2024-07-29T21:50:24.006457Z","shell.execute_reply":"2024-07-29T21:50:24.014369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reshape_row(row: pd.Series) -> pd.DataFrame:\n    \"\"\"Reshape a single row of the DataFrame into a new format.\"\"\"\n    data: Dict[str, List] = {'study_id': [], 'condition': [], 'level': [], 'severity': []}","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:24.016474Z","iopub.execute_input":"2024-07-29T21:50:24.016812Z","iopub.status.idle":"2024-07-29T21:50:24.026483Z","shell.execute_reply.started":"2024-07-29T21:50:24.016787Z","shell.execute_reply":"2024-07-29T21:50:24.025506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reshape_row(row: pd.Series) -> pd.DataFrame:\n    \"\"\"Reshape a single row of the DataFrame into a new format.\"\"\"\n    data: Dict[str, List] = {'study_id': [], 'condition': [], 'level': [], 'severity': []}\n    \n    for column, value in row.items():\n        if column not in ['study_id', 'series_id', 'instance_number', 'x', 'y', 'series_description']:\n            parts = column.split('_')\n            condition = ' '.join(word.capitalize() for word in parts[:-2])\n            level = f\"{parts[-2].capitalize()}/{parts[-1].capitalize()}\"\n            data['study_id'].append(row['study_id'])\n            data['condition'].append(condition)\n            data['level'].append(level)\n            data['severity'].append(value)\n    \n    return pd.DataFrame(data)","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:24.027962Z","iopub.execute_input":"2024-07-29T21:50:24.028687Z","iopub.status.idle":"2024-07-29T21:50:24.036917Z","shell.execute_reply.started":"2024-07-29T21:50:24.028652Z","shell.execute_reply":"2024-07-29T21:50:24.035935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_train_data(train: pd.DataFrame, label: pd.DataFrame, train_desc: pd.DataFrame) -> pd.DataFrame:\n    \"\"\"Preprocess and merge the training data.\"\"\"\n    new_train_df = pd.concat((reshape_row(row) for _, row in train.iterrows()), ignore_index=True)\n    merged_df = pd.merge(new_train_df, label, on=['study_id', 'condition', 'level'], how='inner')\n    final_merged_df = pd.merge(merged_df, train_desc, on=['series_id', 'study_id'], how='inner')\n    \n    final_merged_df['row_id'] = (\n        final_merged_df['study_id'].astype(str) + '_' +\n        final_merged_df['condition'].str.lower().str.replace(' ', '_') + '_' +\n        final_merged_df['level'].str.lower().str.replace('/', '_')\n    )\n    \n    final_merged_df['image_path'] = (\n        f'{TRAIN_PATH}/train_images/' + \n        final_merged_df['study_id'].astype(str) + '/' +\n        final_merged_df['series_id'].astype(str) + '/' +\n        final_merged_df['instance_number'].astype(str) + '.dcm'\n    )\n    \n    final_merged_df['severity'] = final_merged_df['severity'].map({'Normal/Mild': 'normal_mild', 'Moderate': 'moderate', 'Severe': 'severe'})\n    \n    return final_merged_df","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:24.038125Z","iopub.execute_input":"2024-07-29T21:50:24.038390Z","iopub.status.idle":"2024-07-29T21:50:24.052168Z","shell.execute_reply.started":"2024-07-29T21:50:24.038365Z","shell.execute_reply":"2024-07-29T21:50:24.051062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_image_paths(row: pd.Series) -> List[str]:\n    \"\"\"Get image paths for a series.\"\"\"\n    series_path = os.path.join(TRAIN_PATH, 'test_images', str(row['study_id']), str(row['series_id']))\n    if os.path.exists(series_path):\n        return [os.path.join(series_path, f) for f in os.listdir(series_path) if os.path.isfile(os.path.join(series_path, f))]\n    return []","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:24.053267Z","iopub.execute_input":"2024-07-29T21:50:24.053596Z","iopub.status.idle":"2024-07-29T21:50:24.061675Z","shell.execute_reply.started":"2024-07-29T21:50:24.053552Z","shell.execute_reply":"2024-07-29T21:50:24.060620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_test_data(test_desc: pd.DataFrame) -> pd.DataFrame:\n    \"\"\"Preprocess the test data.\"\"\"\n    condition_mapping = {\n        'Sagittal T1': {'left': 'left_neural_foraminal_narrowing', 'right': 'right_neural_foraminal_narrowing'},\n        'Axial T2': {'left': 'left_subarticular_stenosis', 'right': 'right_subarticular_stenosis'},\n        'Sagittal T2/STIR': 'spinal_canal_stenosis'\n    }\n\n    expanded_rows = []\n    for _, row in test_desc.iterrows():\n        image_paths = get_image_paths(row)\n        conditions = condition_mapping.get(row['series_description'], {})\n        if isinstance(conditions, str):\n            conditions = {'left': conditions, 'right': conditions}\n        for side, condition in conditions.items():\n            for image_path in image_paths:\n                expanded_rows.append({\n                    'study_id': row['study_id'],\n                    'series_id': row['series_id'],\n                    'series_description': row['series_description'],\n                    'image_path': image_path,\n                    'condition': condition,\n                    'row_id': f\"{row['study_id']}_{condition}\"\n                })\n\n    return pd.DataFrame(expanded_rows)","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:24.062871Z","iopub.execute_input":"2024-07-29T21:50:24.063147Z","iopub.status.idle":"2024-07-29T21:50:24.075409Z","shell.execute_reply.started":"2024-07-29T21:50:24.063115Z","shell.execute_reply":"2024-07-29T21:50:24.074527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def check_path_exists(path: str) -> bool:\n    \"\"\"Check if a path exists.\"\"\"\n    return os.path.exists(path)","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:24.076734Z","iopub.execute_input":"2024-07-29T21:50:24.077038Z","iopub.status.idle":"2024-07-29T21:50:24.085170Z","shell.execute_reply.started":"2024-07-29T21:50:24.077013Z","shell.execute_reply":"2024-07-29T21:50:24.084267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def validate_train_data(train_data: pd.DataFrame) -> pd.DataFrame:\n    \"\"\"Validate and filter the train data.\"\"\"\n    train_data['study_id_exists'] = train_data.apply(lambda row: check_path_exists(f'{TRAIN_PATH}/train_images/{row[\"study_id\"]}'), axis=1)\n    train_data['series_id_exists'] = train_data.apply(lambda row: check_path_exists(f'{TRAIN_PATH}/train_images/{row[\"study_id\"]}/{row[\"series_id\"]}'), axis=1)\n    train_data['image_exists'] = train_data['image_path'].apply(check_path_exists)\n    \n    return train_data[(train_data['study_id_exists']) & (train_data['series_id_exists']) & (train_data['image_exists'])]","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:24.086334Z","iopub.execute_input":"2024-07-29T21:50:24.086736Z","iopub.status.idle":"2024-07-29T21:50:24.096149Z","shell.execute_reply.started":"2024-07-29T21:50:24.086709Z","shell.execute_reply":"2024-07-29T21:50:24.095103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dicom(path: str) -> np.ndarray:\n    \"\"\"Load and preprocess a DICOM image.\"\"\"\n    dicom = pydicom.read_file(path)\n    data = dicom.pixel_array\n    data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:24.097654Z","iopub.execute_input":"2024-07-29T21:50:24.098040Z","iopub.status.idle":"2024-07-29T21:50:24.109732Z","shell.execute_reply.started":"2024-07-29T21:50:24.098008Z","shell.execute_reply":"2024-07-29T21:50:24.108949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_sample_images(train_data: pd.DataFrame, num_samples: int = 2):\n    \"\"\"Plot sample images from the training data.\"\"\"\n    selected_indices = random.sample(range(len(train_data)), num_samples)\n    images = [load_dicom(train_data['image_path'].iloc[i]) for i in selected_indices]\n    row_ids = [train_data['row_id'].iloc[i] for i in selected_indices]\n\n    fig, axes = plt.subplots(1, num_samples, figsize=(4*num_samples, 4))\n    for i, (image, row_id) in enumerate(zip(images, row_ids)):\n        ax = axes[i] if num_samples > 1 else axes\n        ax.imshow(image, cmap='gray')\n        ax.set_title(f'Row ID: {row_id}', fontsize=8)\n        ax.axis('off')\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:24.110639Z","iopub.execute_input":"2024-07-29T21:50:24.110979Z","iopub.status.idle":"2024-07-29T21:50:24.120059Z","shell.execute_reply.started":"2024-07-29T21:50:24.110951Z","shell.execute_reply":"2024-07-29T21:50:24.119054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load data\ntrain, label, train_desc, test_desc, sub = load_data()","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:24.121351Z","iopub.execute_input":"2024-07-29T21:50:24.121715Z","iopub.status.idle":"2024-07-29T21:50:24.234451Z","shell.execute_reply.started":"2024-07-29T21:50:24.121661Z","shell.execute_reply":"2024-07-29T21:50:24.233502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preprocess train data\ntrain_data = preprocess_train_data(train, label, train_desc)\ntrain_data = validate_train_data(train_data)","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:24.239544Z","iopub.execute_input":"2024-07-29T21:50:24.239919Z","iopub.status.idle":"2024-07-29T21:50:57.740687Z","shell.execute_reply.started":"2024-07-29T21:50:24.239891Z","shell.execute_reply":"2024-07-29T21:50:57.739855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preprocess test data\ntest_data = preprocess_test_data(test_desc)","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:57.741898Z","iopub.execute_input":"2024-07-29T21:50:57.742198Z","iopub.status.idle":"2024-07-29T21:50:57.886525Z","shell.execute_reply.started":"2024-07-29T21:50:57.742172Z","shell.execute_reply":"2024-07-29T21:50:57.885532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display some information\nprint(\"Train data shape:\", train_data.shape)\nprint(\"Test data shape:\", test_data.shape)","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:57.887809Z","iopub.execute_input":"2024-07-29T21:50:57.888105Z","iopub.status.idle":"2024-07-29T21:50:57.893256Z","shell.execute_reply.started":"2024-07-29T21:50:57.888078Z","shell.execute_reply":"2024-07-29T21:50:57.892276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot sample images\nplot_sample_images(train_data)","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:57.894887Z","iopub.execute_input":"2024-07-29T21:50:57.895243Z","iopub.status.idle":"2024-07-29T21:50:58.378980Z","shell.execute_reply.started":"2024-07-29T21:50:57.895209Z","shell.execute_reply":"2024-07-29T21:50:58.378038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomDataset(Dataset):\n    def __init__(self, dataframe, transform=None):\n        self.dataframe = dataframe\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.dataframe)\n\n    def __getitem__(self, index):\n        image_path = self.dataframe.iloc[index]['image_path']\n        image = load_dicom(image_path)\n        label = self.dataframe.iloc[index]['severity']\n        \n        if self.transform:\n            image = self.transform(image)\n\n        return image, label","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:58.380240Z","iopub.execute_input":"2024-07-29T21:50:58.380632Z","iopub.status.idle":"2024-07-29T21:50:58.388649Z","shell.execute_reply.started":"2024-07-29T21:50:58.380596Z","shell.execute_reply":"2024-07-29T21:50:58.387736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_datasets_and_loaders(df, series_description, transform, batch_size=8):\n    filtered_df = df[df['series_description'] == series_description]\n    \n    train_df, val_df = train_test_split(filtered_df, test_size=0.2, random_state=42)\n    train_df = train_df.reset_index(drop=True)\n    val_df = val_df.reset_index(drop=True)\n\n    train_dataset = CustomDataset(train_df, transform)\n    val_dataset = CustomDataset(val_df, transform)\n\n    trainloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n    valloader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\n    \n    return trainloader, valloader, len(train_df), len(val_df)","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:58.389806Z","iopub.execute_input":"2024-07-29T21:50:58.390101Z","iopub.status.idle":"2024-07-29T21:50:58.401019Z","shell.execute_reply.started":"2024-07-29T21:50:58.390073Z","shell.execute_reply":"2024-07-29T21:50:58.400086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the transforms\ntransform = transforms.Compose([\n    transforms.Lambda(lambda x: (x * 255).astype(np.uint8)),\n    transforms.ToPILImage(),\n    transforms.Resize((224, 224)),\n    transforms.Grayscale(num_output_channels=3),\n    transforms.ToTensor(),\n])","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:58.402130Z","iopub.execute_input":"2024-07-29T21:50:58.402403Z","iopub.status.idle":"2024-07-29T21:50:58.411212Z","shell.execute_reply.started":"2024-07-29T21:50:58.402378Z","shell.execute_reply":"2024-07-29T21:50:58.410216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create dataloaders for each series description\ndataloaders = {}\nlengths = {}","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:58.412348Z","iopub.execute_input":"2024-07-29T21:50:58.412645Z","iopub.status.idle":"2024-07-29T21:50:58.429764Z","shell.execute_reply.started":"2024-07-29T21:50:58.412619Z","shell.execute_reply":"2024-07-29T21:50:58.428837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for desc in ['Sagittal T1', 'Axial T2', 'Sagittal T2/STIR']:\n    trainloader, valloader, len_train, len_val = create_datasets_and_loaders(train_data, desc, transform)\n    dataloaders[desc] = (trainloader, valloader)\n    lengths[desc] = (len_train, len_val)","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:58.430847Z","iopub.execute_input":"2024-07-29T21:50:58.431168Z","iopub.status.idle":"2024-07-29T21:50:58.498589Z","shell.execute_reply.started":"2024-07-29T21:50:58.431141Z","shell.execute_reply":"2024-07-29T21:50:58.497547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize samples from each dataloader\nfor desc in dataloaders:\n    print(f\"Visualizing {desc} samples\")\n    visualize_batch(dataloaders[desc][0], f\"{desc} Samples\")","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:50:58.499789Z","iopub.execute_input":"2024-07-29T21:50:58.500102Z","iopub.status.idle":"2024-07-29T21:51:01.505918Z","shell.execute_reply.started":"2024-07-29T21:50:58.500074Z","shell.execute_reply":"2024-07-29T21:51:01.504991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dictionary mapping labels to indices\nlabel_map = {'normal_mild': 0, 'moderate': 1, 'severe': 2}","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:51:01.507111Z","iopub.execute_input":"2024-07-29T21:51:01.507382Z","iopub.status.idle":"2024-07-29T21:51:01.511413Z","shell.execute_reply.started":"2024-07-29T21:51:01.507357Z","shell.execute_reply":"2024-07-29T21:51:01.510539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display a single sample\ntrainloader = dataloaders['Axial T2'][0]\nimage, label = next(iter(trainloader))\nsample = image[0].permute(1, 2, 0)","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:51:01.512568Z","iopub.execute_input":"2024-07-29T21:51:01.512846Z","iopub.status.idle":"2024-07-29T21:51:01.730853Z","shell.execute_reply.started":"2024-07-29T21:51:01.512821Z","shell.execute_reply":"2024-07-29T21:51:01.729781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_batch(dataloader, title):\n    images, labels = next(iter(dataloader))\n    fig, axes = plt.subplots(1, len(images), figsize=(20, 5))\n    for i, (img, lbl) in enumerate(zip(images, labels)):\n        ax = axes[i]\n        img = img.permute(1, 2, 0).numpy()  # Convert to HWC for visualization\n        ax.imshow(img, cmap='gray')\n        ax.set_title(f\"Label: {lbl}\")\n        ax.axis('off')\n    plt.suptitle(title)\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:51:01.735178Z","iopub.execute_input":"2024-07-29T21:51:01.736193Z","iopub.status.idle":"2024-07-29T21:51:01.743320Z","shell.execute_reply.started":"2024-07-29T21:51:01.736152Z","shell.execute_reply":"2024-07-29T21:51:01.742381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8, 4))\nplt.imshow(sample, cmap='gray')\nplt.title(f\"Label: {label[0]}\")\nplt.axis('off')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:51:01.744830Z","iopub.execute_input":"2024-07-29T21:51:01.745779Z","iopub.status.idle":"2024-07-29T21:51:01.993646Z","shell.execute_reply.started":"2024-07-29T21:51:01.745744Z","shell.execute_reply":"2024-07-29T21:51:01.992777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import layers, models, optimizers","metadata":{"execution":{"iopub.status.busy":"2024-07-29T21:53:52.883015Z","iopub.execute_input":"2024-07-29T21:53:52.883390Z","iopub.status.idle":"2024-07-29T21:54:04.727894Z","shell.execute_reply.started":"2024-07-29T21:53:52.883361Z","shell.execute_reply":"2024-07-29T21:54:04.726311Z"},"trusted":true},"execution_count":null,"outputs":[]}]}