{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":30761,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## naming the new dataframes of the csv files\n\ntrainset1 = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv')\ntrainset2 = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_label_coordinates.csv')\ntrainset3 = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_series_descriptions.csv')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainset1.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainset2.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainset3.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_df = pd.merge(trainset2, trainset3, on='series_id', how='inner')\n# Assuming 'study_id' is identical in both DataFrames\nmerged_df['study_id'] = merged_df['study_id_x']  # or merged_df['study_id_y'], depending on preference\n\n# Drop the redundant 'study_id_x' and 'study_id_y' columns\nmerged_df = merged_df.drop(columns=['study_id_x', 'study_id_y'])\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_merged_df = pd.merge(merged_df, trainset1, on='study_id', how='inner')\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_merged_df.columns.tolist()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(final_merged_df)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pydicom","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n# import pydicom\n# import numpy as np\n# import matplotlib.pyplot as plt\n# import cv2\n\n# # Function to map the level to the corresponding stenosis column in the DataFrame\n# def get_stenosis_column(level, condition):\n#     \"\"\"\n#     Returns the appropriate stenosis column based on the level and condition.\n#     \"\"\"\n#     stenosis_map = {\n#         'L1/L2': {\n#             'Spinal Canal Stenosis': 'spinal_canal_stenosis_l1_l2',\n#             'Left Neural Foraminal Narrowing': 'left_neural_foraminal_narrowing_l1_l2',\n#             'Right Neural Foraminal Narrowing': 'right_neural_foraminal_narrowing_l1_l2',\n#             'Left Subarticular Stenosis': 'left_subarticular_stenosis_l1_l2',\n#             'Right Subarticular Stenosis': 'right_subarticular_stenosis_l1_l2',\n#         },\n#         'L2/L3': {\n#             'Spinal Canal Stenosis': 'spinal_canal_stenosis_l2_l3',\n#             'Left Neural Foraminal Narrowing': 'left_neural_foraminal_narrowing_l2_l3',\n#             'Right Neural Foraminal Narrowing': 'right_neural_foraminal_narrowing_l2_l3',\n#             'Left Subarticular Stenosis': 'left_subarticular_stenosis_l2_l3',\n#             'Right Subarticular Stenosis': 'right_subarticular_stenosis_l2_l3',\n#         },\n#         'L3/L4': {\n#             'Spinal Canal Stenosis': 'spinal_canal_stenosis_l3_l4',\n#             'Left Neural Foraminal Narrowing': 'left_neural_foraminal_narrowing_l3_l4',\n#             'Right Neural Foraminal Narrowing': 'right_neural_foraminal_narrowing_l3_l4',\n#             'Left Subarticular Stenosis': 'left_subarticular_stenosis_l3_l4',\n#             'Right Subarticular Stenosis': 'right_subarticular_stenosis_l3_l4',\n#         },\n#         'L4/L5': {\n#             'Spinal Canal Stenosis': 'spinal_canal_stenosis_l4_l5',\n#             'Left Neural Foraminal Narrowing': 'left_neural_foraminal_narrowing_l4_l5',\n#             'Right Neural Foraminal Narrowing': 'right_neural_foraminal_narrowing_l4_l5',\n#             'Left Subarticular Stenosis': 'left_subarticular_stenosis_l4_l5',\n#             'Right Subarticular Stenosis': 'right_subarticular_stenosis_l4_l5',\n#         },\n#         'L5/S1': {\n#             'Spinal Canal Stenosis': 'spinal_canal_stenosis_l5_s1',\n#             'Left Neural Foraminal Narrowing': 'left_neural_foraminal_narrowing_l5_s1',\n#             'Right Neural Foraminal Narrowing': 'right_neural_foraminal_narrowing_l5_s1',\n#             'Left Subarticular Stenosis': 'left_subarticular_stenosis_l5_s1',\n#             'Right Subarticular Stenosis': 'right_subarticular_stenosis_l5_s1',\n#         }\n#     }\n\n#     return stenosis_map.get(level, {}).get(condition, None)\n\n# # Function to apply CLAHE to the image\n# def apply_clahe(image):\n#     \"\"\"\n#     Apply CLAHE (Contrast Limited Adaptive Histogram Equalization) to the input image.\n\n#     Parameters:\n#     - image: The input image (assumed to be grayscale).\n\n#     Returns:\n#     - enhanced_image: The image after applying CLAHE.\n#     \"\"\"\n#     # Create a CLAHE object with a clip limit\n#     clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))\n#     enhanced_image = clahe.apply(image)\n#     return enhanced_image\n\n# # Function to normalize the image data to 8-bit format (uint8)\n# def normalize_to_uint8(image):\n#     \"\"\"\n#     Normalize the image data to 8-bit format for compatibility with OpenCV and matplotlib.\n#     \"\"\"\n#     # Normalize the image to the range [0, 255]\n#     image_normalized = cv2.normalize(image, None, 0, 255, cv2.NORM_MINMAX)\n#     # Convert to uint8\n#     return image_normalized.astype(np.uint8)\n\n# # Function to color code the severity and normalize it for matplotlib\n# def get_circle_color(severity):\n#     \"\"\"\n#     Returns the normalized color for the circle based on the severity of the condition.\n#     Normalizes color values to [0, 1] for matplotlib.\n#     \"\"\"\n#     severity_color_map = {\n#         'Normal/Mild': (0, 1.0, 0),    # Green (normalized)\n#         'Moderate': (1.0, 1.0, 0),     # Yellow (normalized)\n#         'Severe': (1.0, 0, 0)          # Red (normalized)\n#     }\n#     return severity_color_map.get(severity, (1.0, 1.0, 1.0))  # Default to white if unknown\n\n# # Function to display images grouped by description with hollow dots for severity\n# def display_images_grouped_by_description(images_by_description, study_id):\n#     \"\"\"\n#     Displays images grouped by description and includes hollow circles representing severity.\n\n#     Parameters:\n#     - images_by_description: Dictionary containing lists of images categorized by description.\n#     - study_id: The study ID being processed.\n#     \"\"\"\n#     for description, images in images_by_description.items():\n#         # Create a plot for each description\n#         num_images = len(images)\n#         fig, axs = plt.subplots(1, num_images, figsize=(20, 6))\n\n#         if num_images == 1:\n#             axs = [axs]  # To ensure axs is iterable even for a single image\n\n#         # Plot each image with a hollow circle for severity\n#         for i, (img, instance_number, x, y, severity, series_id) in enumerate(images):\n#             # Apply CLAHE to enhance the image contrast\n#             enhanced_img = apply_clahe(img)\n\n#             # Display the enhanced image with gray color map\n#             axs[i].imshow(enhanced_img, cmap='gray')\n#             axs[i].scatter([x], [y], facecolors='none', edgecolors=get_circle_color(severity), s=100, linewidths=2)\n#             axs[i].set_title(f\"{instance_number}.dcm\")\n#             axs[i].axis('off')\n\n#         plt.suptitle(f\"Study ID: {study_id} | Series ID: {series_id} | Description: {description}\", fontsize=16)\n#         plt.tight_layout()\n#         plt.show()\n\n# # Main function to process images and categorize them by description\n# def process_study_ids_by_description(df, root_dir):\n#     \"\"\"\n#     Loop through multiple study_ids and categorize the images by description, displaying them with hollow dots for severity.\n\n#     Parameters:\n#     - df: The DataFrame containing all the data.\n#     - root_dir: The root directory where the DICOM images are stored.\n#     \"\"\"\n#     # Loop through each unique study_id\n#     for study_id in df['study_id'].unique():\n#         # Filter the DataFrame for the current study_id\n#         study_df = df[df['study_id'] == study_id]\n\n#         # Initialize a dictionary to categorize images by description\n#         images_by_description = {}\n\n#         # Loop through each unique series_id within the current study_id\n#         for series_id in study_df['series_id'].unique():\n#             series_df = study_df[study_df['series_id'] == series_id]\n#             series_description = series_df['series_description'].iloc[0]  # Get description for the series\n\n#             # Path to the study and series folders\n#             study_folder = os.path.join(root_dir, str(study_id))\n#             series_folder = os.path.join(study_folder, str(series_id))\n\n#             # Check if the folder exists\n#             if not os.path.exists(series_folder):\n#                 print(f\"Series folder {series_folder} not found.\")\n#                 continue\n\n#             # Loop through the rows and categorize images by description\n#             for index, row in series_df.iterrows():\n#                 instance_number = int(row['instance_number'])  # Instance number is the DICOM file number\n#                 x = row['x']  # X coordinate\n#                 y = row['y']  # Y coordinate\n#                 level = row['level']  # Level (e.g., L1/L2)\n#                 condition = row['condition']  # Condition (e.g., Spinal Canal Stenosis)\n\n#                 # Get the appropriate stenosis column based on the level and condition\n#                 stenosis_column = get_stenosis_column(level, condition)\n#                 if stenosis_column is None:\n#                     print(f\"No corresponding stenosis column found for level {level} and condition {condition}.\")\n#                     continue\n\n#                 # Retrieve the condition status (e.g., \"Normal/Mild\", \"Moderate\", \"Severe\")\n#                 condition_status = row[stenosis_column]\n\n#                 # Locate the corresponding DICOM file (using the instance number)\n#                 dcm_file = f\"{instance_number}.dcm\"\n#                 dcm_path = os.path.join(series_folder, dcm_file)\n\n#                 if not os.path.exists(dcm_path):\n#                     print(f\"DICOM file {dcm_file} not found for Series ID: {series_id}.\")\n#                     continue\n\n#                 # Load the DICOM file\n#                 dicom_data = pydicom.dcmread(dcm_path)\n\n#                 # Extract pixel data and normalize to uint8\n#                 image = normalize_to_uint8(dicom_data.pixel_array)\n\n#                 # Add the image, instance number, and coordinates to the appropriate description category\n#                 if series_description not in images_by_description:\n#                     images_by_description[series_description] = []\n#                 images_by_description[series_description].append((image, instance_number, x, y, condition_status, series_id))\n\n#         # Display the images grouped by description\n#         display_images_grouped_by_description(images_by_description, study_id)\n\n# # Example usage:\n# root_dir = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images'\n# process_study_ids_by_description(final_merged_df, root_dir)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n# import pydicom\n# import numpy as np\n# import cv2\n# import pandas as pd\n# import tensorflow as tf\n# from sklearn.model_selection import train_test_split\n# from tensorflow.keras.models import Sequential\n# from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout, BatchNormalization\n# from tensorflow.keras.utils import Sequence\n\n# # Function to map the level to the corresponding stenosis column in the DataFrame\n# def get_stenosis_column(level, condition):\n#     \"\"\"\n#     Returns the appropriate stenosis column based on the level and condition.\n#     \"\"\"\n#     stenosis_map = {\n#         'L1/L2': {\n#             'Spinal Canal Stenosis': 'spinal_canal_stenosis_l1_l2',\n#             'Left Neural Foraminal Narrowing': 'left_neural_foraminal_narrowing_l1_l2',\n#             'Right Neural Foraminal Narrowing': 'right_neural_foraminal_narrowing_l1_l2',\n#             'Left Subarticular Stenosis': 'left_subarticular_stenosis_l1_l2',\n#             'Right Subarticular Stenosis': 'right_subarticular_stenosis_l1_l2',\n#         },\n#         'L2/L3': {\n#             'Spinal Canal Stenosis': 'spinal_canal_stenosis_l2_l3',\n#             'Left Neural Foraminal Narrowing': 'left_neural_foraminal_narrowing_l2_l3',\n#             'Right Neural Foraminal Narrowing': 'right_neural_foraminal_narrowing_l2_l3',\n#             'Left Subarticular Stenosis': 'left_subarticular_stenosis_l2_l3',\n#             'Right Subarticular Stenosis': 'right_subarticular_stenosis_l2_l3',\n#         },\n#         'L3/L4': {\n#             'Spinal Canal Stenosis': 'spinal_canal_stenosis_l3_l4',\n#             'Left Neural Foraminal Narrowing': 'left_neural_foraminal_narrowing_l3_l4',\n#             'Right Neural Foraminal Narrowing': 'right_neural_foraminal_narrowing_l3_l4',\n#             'Left Subarticular Stenosis': 'left_subarticular_stenosis_l3_l4',\n#             'Right Subarticular Stenosis': 'right_subarticular_stenosis_l3_l4',\n#         },\n#         'L4/L5': {\n#             'Spinal Canal Stenosis': 'spinal_canal_stenosis_l4_l5',\n#             'Left Neural Foraminal Narrowing': 'left_neural_foraminal_narrowing_l4_l5',\n#             'Right Neural Foraminal Narrowing': 'right_neural_foraminal_narrowing_l4_l5',\n#             'Left Subarticular Stenosis': 'left_subarticular_stenosis_l4_l5',\n#             'Right Subarticular Stenosis': 'right_subarticular_stenosis_l4_l5',\n#         },\n#         'L5/S1': {\n#             'Spinal Canal Stenosis': 'spinal_canal_stenosis_l5_s1',\n#             'Left Neural Foraminal Narrowing': 'left_neural_foraminal_narrowing_l5_s1',\n#             'Right Neural Foraminal Narrowing': 'right_neural_foraminal_narrowing_l5_s1',\n#             'Left Subarticular Stenosis': 'left_subarticular_stenosis_l5_s1',\n#             'Right Subarticular Stenosis': 'right_subarticular_stenosis_l5_s1',\n#         }\n#     }\n\n#     return stenosis_map.get(level, {}).get(condition, None)\n\n# # Function to map severity and populate the condition_status column\n# def get_condition_status(row):\n#     \"\"\"\n#     Dynamically retrieve the severity status from the correct column based on the condition and level.\n#     \"\"\"\n#     stenosis_column = get_stenosis_column(row['level'], row['condition'])\n#     if stenosis_column:\n#         return row[stenosis_column]\n#     return None\n\n# # Function to load and preprocess DICOM images\n# def load_dicom_image(study_id, series_id, instance_number, root_dir, target_size=(128, 128)):\n#     \"\"\"\n#     Load and preprocess the DICOM image, including resizing and normalization.\n#     \"\"\"\n#     study_folder = os.path.join(root_dir, str(study_id))\n#     series_folder = os.path.join(study_folder, str(series_id))\n\n#     # Loop through all subfolders within the series folder\n#     for folder in os.listdir(series_folder):\n#         subfolder_path = os.path.join(series_folder, folder)\n#         if os.path.isdir(subfolder_path):\n#             dcm_file = f\"{instance_number}.dcm\"\n#             dcm_path = os.path.join(subfolder_path, dcm_file)\n#             if os.path.exists(dcm_path):\n#                 dicom_data = pydicom.dcmread(dcm_path)\n#                 image = dicom_data.pixel_array\n\n#                 # Normalize the image to [0, 1] range\n#                 img_resized = cv2.resize(image, target_size)\n#                 img_normalized = img_resized / 255.0\n\n#                 # Expand dimensions to match model input (grayscale -> single channel)\n#                 if len(img_normalized.shape) == 2:\n#                     img_normalized = np.expand_dims(img_normalized, axis=-1)\n#                 return img_normalized\n\n#     return None\n\n# # Function to encode labels (severity and location)\n# def encode_labels(row):\n#     \"\"\"\n#     Encodes the severity (Normal/Mild, Moderate, Severe) and location (L1/L2, etc.).\n#     \"\"\"\n#     severity_map = {'Normal/Mild': 0, 'Moderate': 1, 'Severe': 2}\n#     location_map = {'L1/L2': 0, 'L2/L3': 1, 'L3/L4': 2, 'L4/L5': 3, 'L5/S1': 4}\n\n#     severity = severity_map.get(row['condition_status'], 0)  # Default to Normal/Mild\n#     location = location_map.get(row['level'], 0)  # Default to L1/L2\n#     return severity, location\n\n# # Data generator class for loading data in batches\n# class DataGenerator(Sequence):\n#     def __init__(self, df, root_dir, batch_size=32, target_size=(128, 128), shuffle=True):\n#         self.df = df.reset_index(drop=True)\n#         self.root_dir = root_dir\n#         self.batch_size = batch_size\n#         self.target_size = target_size\n#         self.shuffle = shuffle\n#         self.indexes = np.arange(len(self.df))\n#         self.on_epoch_end()\n\n#     def __len__(self):\n#         # Return the number of batches per epoch\n#         return int(np.floor(len(self.df) / self.batch_size))\n\n#     def __getitem__(self, index):\n#         # Generate indexes for the batch\n#         batch_indexes = self.indexes[index * self.batch_size:(index + 1) * self.batch_size]\n#         batch_df = self.df.iloc[batch_indexes]\n\n#         # Load and preprocess images and labels for the batch\n#         images, severities, locations = [], [], []\n#         for _, row in batch_df.iterrows():\n#             img = load_dicom_image(row['study_id'], row['series_id'], int(row['instance_number']), self.root_dir, self.target_size)\n#             if img is None:\n#                 continue\n\n#             severity, location = encode_labels(row)\n#             images.append(img)\n#             severities.append(severity)\n#             locations.append(location)\n\n#         # Convert lists to numpy arrays\n#         X = np.array(images)\n#         y_severity = np.array(severities)\n#         y_location = np.array(locations)\n\n#         return X, {'severity_output': y_severity, 'location_output': y_location}\n\n#     def on_epoch_end(self):\n#         if self.shuffle:\n#             np.random.shuffle(self.indexes)\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # CNN Model\n# def create_cnn_model(input_shape):\n#     \"\"\"\n#     Creates a CNN model for multi-output (severity and location).\n#     \"\"\"\n#     model = Sequential()\n\n#     model.add(Conv2D(32, (3, 3), activation='relu', input_shape=input_shape))\n#     model.add(MaxPooling2D((2, 2)))\n#     model.add(BatchNormalization())\n\n#     model.add(Conv2D(64, (3, 3), activation='relu'))\n#     model.add(MaxPooling2D((2, 2)))\n#     model.add(BatchNormalization())\n\n#     model.add(Conv2D(128, (3, 3), activation='relu'))\n#     model.add(MaxPooling2D((2, 2)))\n#     model.add(BatchNormalization())\n\n#     model.add(Flatten())\n\n#     # Shared fully connected layer\n#     model.add(Dense(256, activation='relu'))\n#     model.add(Dropout(0.5))\n\n#     # Output for severity classification\n#     severity_output = Dense(3, activation='softmax', name='severity_output')  # 3 classes (Normal/Mild, Moderate, Severe)\n\n#     # Output for location classification\n#     location_output = Dense(5, activation='softmax', name='location_output')  # 5 locations (L1/L2 to L5/S1)\n\n#     model = tf.keras.Model(inputs=model.input, outputs=[severity_output, location_output])\n\n#     # Compile the model\n#     model.compile(optimizer='adam',\n#                   loss={'severity_output': 'sparse_categorical_crossentropy',\n#                         'location_output': 'sparse_categorical_crossentropy'},\n#                   metrics={'severity_output': 'accuracy', 'location_output': 'accuracy'})\n\n#     return model\n\n# # Split the dataset into train, validation, and test sets\n# def split_dataset(df, test_size=0.15, validation_size=0.15):\n#     train_df, test_df = train_test_split(df, test_size=test_size, stratify=df['condition_status'], random_state=42)\n#     train_df, val_df = train_test_split(train_df, test_size=validation_size / (1 - test_size), stratify=train_df['condition_status'], random_state=42)\n#     return train_df, val_df, test_df\n\n# # Ensure the condition_status column exists in the DataFrame\n# final_merged_df['condition_status'] = final_merged_df.apply(get_condition_status, axis=1)\n\n# # Check if there are any missing values\n# print(final_merged_df['condition_status'].isnull().sum())  # Should return 0 if all rows have severity\n\n# # Split the data into train, validation, and test sets\n# train_df, val_df, test_df = split_dataset(final_merged_df)\n\n# # Create data generators\n# root_dir = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images'\n# train_generator = DataGenerator(train_df, root_dir, batch_size=32)\n# val_generator = DataGenerator(val_df, root_dir, batch_size=32)\n# test_generator = DataGenerator(test_df, root_dir, batch_size=32, shuffle=False)\n\n# # Create and compile the CNN model\n# input_shape = (128, 128, 1)  # Assuming grayscale images\n# model = create_cnn_model(input_shape)\n\n# # Train the model\n# history = model.fit(\n#     train_generator,\n#     validation_data=val_generator,\n#     epochs=10  # Adjust number of epochs based on your setup\n# )\n\n# # Evaluate the model on the test set\n# test_loss, test_severity_acc, test_location_acc = model.evaluate(test_generator)\n# print(f\"Test Loss: {test_loss:.4f}\")\n# print(f\"Test Severity Accuracy: {test_severity_acc:.4f}\")\n# print(f\"Test Location Accuracy: {test_location_acc:.4f}\")\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}