{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":31236,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install tensorflow","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:00:31.209472Z","iopub.execute_input":"2026-01-01T14:00:31.209800Z","iopub.status.idle":"2026-01-01T14:00:35.390970Z","shell.execute_reply.started":"2026-01-01T14:00:31.209768Z","shell.execute_reply":"2026-01-01T14:00:35.390154Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport pydicom\nimport cv2\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Input, Conv2D, MaxPooling2D, UpSampling2D, concatenate\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.applications import EfficientNetB0\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint","metadata":{"_cell_guid":"e4cc7d10-d5ba-4a3a-bc15-2602a8a9e9ad","_uuid":"769f3a6d-d073-488f-95f6-8c34c46fdcb5","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:00:40.072455Z","iopub.execute_input":"2026-01-01T14:00:40.073151Z","iopub.status.idle":"2026-01-01T14:00:53.894408Z","shell.execute_reply.started":"2026-01-01T14:00:40.073113Z","shell.execute_reply":"2026-01-01T14:00:53.893674Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train  = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv')\nlabel = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_label_coordinates.csv')\ntrain_desc = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_series_descriptions.csv')\ntest_desc = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_series_descriptions.csv')","metadata":{"_cell_guid":"81d8ce9c-160e-4af6-9209-17fa6874f3a2","_uuid":"45ae8b8e-2e99-42f9-98d6-da96318acbaf","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:01:05.198851Z","iopub.execute_input":"2026-01-01T14:01:05.199402Z","iopub.status.idle":"2026-01-01T14:01:05.386779Z","shell.execute_reply.started":"2026-01-01T14:01:05.199375Z","shell.execute_reply":"2026-01-01T14:01:05.386265Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Preview the data\ntrain.head()","metadata":{"_cell_guid":"6e45dd1e-718d-4a87-8f90-7a08bfc23f0f","_uuid":"d01ff453-8a71-4f8b-9818-44a398a9fd88","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:01:06.712783Z","iopub.execute_input":"2026-01-01T14:01:06.713514Z","iopub.status.idle":"2026-01-01T14:01:06.744032Z","shell.execute_reply.started":"2026-01-01T14:01:06.713485Z","shell.execute_reply":"2026-01-01T14:01:06.743508Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_desc.head()","metadata":{"_cell_guid":"40d691f4-e99e-48f7-a8a2-c6ece549a180","_uuid":"b95b96b1-4802-46b1-9fd7-b8913599180d","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:01:08.563425Z","iopub.execute_input":"2026-01-01T14:01:08.563761Z","iopub.status.idle":"2026-01-01T14:01:08.571183Z","shell.execute_reply.started":"2026-01-01T14:01:08.563733Z","shell.execute_reply":"2026-01-01T14:01:08.570594Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def reshape_dataframe(df):\n    # Create a list of columns to exclude\n    exclude_columns = ['study_id', 'series_id', 'instance_number', 'x', 'y', 'series_description']\n    \n    # Filter the columns to process\n    columns_to_process = [col for col in df.columns if col not in exclude_columns]\n    \n    # Split the columns into condition and level, extract severity, and concatenate to form the new DataFrame\n    reshaped_df = pd.DataFrame([\n        {\n            'study_id': row['study_id'],\n            'condition': ' '.join([word.capitalize() for word in col.split('_')[:-2]]),\n            'level': col.split('_')[-2].capitalize() + '/' + col.split('_')[-1].capitalize(),\n            'severity': row[col]\n        }\n        for _, row in df.iterrows()\n        for col in columns_to_process\n    ])\n    \n    return reshaped_df\n\n# Reshape the DataFrame\nnew_train_df = reshape_dataframe(train)\n\n# Display the first few rows of the reshaped DataFrame\nnew_train_df.head()","metadata":{"_cell_guid":"e903bd74-eb3d-4d47-81eb-9599e0147f23","_uuid":"a452b332-b05a-48e8-bee5-c5fad1b4c95f","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:01:10.178327Z","iopub.execute_input":"2026-01-01T14:01:10.178922Z","iopub.status.idle":"2026-01-01T14:01:10.545701Z","shell.execute_reply.started":"2026-01-01T14:01:10.178896Z","shell.execute_reply":"2026-01-01T14:01:10.544924Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Print columns in a neat way\nprint(\"\\nColumns in new_train_df:\")\nprint(\",\".join(new_train_df.columns))\n\nprint(\"\\nColumns in label:\")\nprint(\",\".join(label.columns))\n\nprint(\"\\nColumns in test_desc:\")\nprint(\",\".join(test_desc.columns))","metadata":{"_cell_guid":"55c084b2-334f-4753-9fdd-c31fff84dd45","_uuid":"cdd5e746-b060-47f8-afdf-fee183c657e2","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:02:34.244778Z","iopub.execute_input":"2026-01-01T14:02:34.245135Z","iopub.status.idle":"2026-01-01T14:02:34.249977Z","shell.execute_reply.started":"2026-01-01T14:02:34.245106Z","shell.execute_reply":"2026-01-01T14:02:34.249216Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Merge reshaped labels with coordinate labels\nmerged_df = pd.merge(\n    new_train_df,\n    label,\n    on=['study_id', 'condition', 'level'],\n    how='inner'\n)\n\n# Quick sanity check\nprint(\"Merged rows:\", len(merged_df))\nprint(merged_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:02:35.841300Z","iopub.execute_input":"2026-01-01T14:02:35.841589Z","iopub.status.idle":"2026-01-01T14:02:35.889408Z","shell.execute_reply.started":"2026-01-01T14:02:35.841565Z","shell.execute_reply":"2026-01-01T14:02:35.888701Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Merge the dataframes on the common column 'series_id'\nfinal_merged_df = pd.merge(merged_df, train_desc, on=['series_id','study_id'], how='inner')\n# Display the first few rows of the final merged dataframe\nfinal_merged_df.head()","metadata":{"_cell_guid":"456408f9-ce58-4cc8-bc12-b5d5b33d5315","_uuid":"a876f3a2-e86b-45f5-908f-49f7d58b8f02","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:02:37.835673Z","iopub.execute_input":"2026-01-01T14:02:37.836351Z","iopub.status.idle":"2026-01-01T14:02:37.858308Z","shell.execute_reply.started":"2026-01-01T14:02:37.836323Z","shell.execute_reply":"2026-01-01T14:02:37.857607Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"In medical imaging for spinal conditions, specific MRI sequences are often used to identify different types of spinal stenosis:\n\n- **Sagittal T1-weighted images** are primarily utilized to evaluate **Neural Foraminal Narrowing**. \n- **Axial T2-weighted images** are crucial for assessing **Subarticular Stenosis**. \n- **Sagittal T2-weighted or STIR (Short Tau Inversion Recovery) images** are typically employed to detect and analyze **Spinal Canal Stenosis**.\n\nThese imaging sequences are chosen for their ability to provide the most relevant anatomical and pathological information for each specific type of stenosis.","metadata":{"_cell_guid":"390329b2-ad78-4dce-9961-5f1230f92935","_uuid":"2ab51b9b-5811-4655-a7d3-982ce5eccfec","trusted":true}},{"cell_type":"code","source":"# Create the row_id column\nfinal_merged_df['row_id'] = (\n    final_merged_df['study_id'].astype(str) + '_' +\n    final_merged_df['condition'].str.lower().str.replace(' ', '_') + '_' +\n    final_merged_df['level'].str.lower().str.replace('/', '_')\n)\n\n# Create the image_path column\nfinal_merged_df['image_path'] = (\n    '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/' + \n    final_merged_df['study_id'].astype(str) + '/' +\n    final_merged_df['series_id'].astype(str) + '/' +\n    final_merged_df['instance_number'].astype(str) + '.dcm'\n)\n\n# Note: Check image path, since there's 1 instance id, for 1 image, but there's many more images other than the ones labelled in the instance ID. \n\n# Display the updated dataframe\nfinal_merged_df.head()","metadata":{"_cell_guid":"aafc3ed4-b4d1-4b1d-adb1-3bbb3d4a4306","_uuid":"337a3e61-c05b-4511-aa53-3624ae3ffe93","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:02:46.227693Z","iopub.execute_input":"2026-01-01T14:02:46.228171Z","iopub.status.idle":"2026-01-01T14:02:46.360497Z","shell.execute_reply.started":"2026-01-01T14:02:46.228146Z","shell.execute_reply":"2026-01-01T14:02:46.359804Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the base path for test images\nbase_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_images'\n\n# Function to get image paths for a series\ndef get_image_paths(row):\n    series_path = os.path.join(base_path, str(row['study_id']), str(row['series_id']))\n    if os.path.exists(series_path):\n        return [os.path.join(series_path, f) for f in os.listdir(series_path) if os.path.isfile(os.path.join(series_path, f))]\n    return []\n\n# Mapping of series_description to conditions\ncondition_mapping = {\n    'Sagittal T1': {'left': 'left_neural_foraminal_narrowing', 'right': 'right_neural_foraminal_narrowing'},\n    'Axial T2': {'left': 'left_subarticular_stenosis', 'right': 'right_subarticular_stenosis'},\n    'Sagittal T2/STIR': 'spinal_canal_stenosis'\n}\n\n# Create a list to store the expanded rows\nexpanded_rows = []\n\n# Expand the dataframe by adding new rows for each file path\nfor index, row in test_desc.iterrows():\n    image_paths = get_image_paths(row)\n    conditions = condition_mapping.get(row['series_description'], {})\n    if isinstance(conditions, str):  # Single condition\n        conditions = {'left': conditions, 'right': conditions}\n    for side, condition in conditions.items():\n        for image_path in image_paths:\n            expanded_rows.append({\n                'study_id': row['study_id'],\n                'series_id': row['series_id'],\n                'series_description': row['series_description'],\n                'image_path': image_path,\n                'condition': condition,\n                'row_id': f\"{row['study_id']}_{condition}\"\n            })\n\n# Create a new dataframe from the expanded rows\nexpanded_test_desc = pd.DataFrame(expanded_rows)\n\n# Display the resulting dataframe\nexpanded_test_desc.head(5)","metadata":{"_cell_guid":"7897c7f7-85d6-46cc-85db-cec4c363eee4","_uuid":"b03c8c55-40a7-4df1-a7c6-4c3c542eefc9","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:02:49.456945Z","iopub.execute_input":"2026-01-01T14:02:49.457806Z","iopub.status.idle":"2026-01-01T14:02:49.678900Z","shell.execute_reply.started":"2026-01-01T14:02:49.457777Z","shell.execute_reply":"2026-01-01T14:02:49.678322Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_merged_df['severity'] = final_merged_df['severity'].fillna('Normal/Mild')","metadata":{"_cell_guid":"d98e8037-d2d7-4025-834f-c4119ec2b08e","_uuid":"596d6d58-0014-4adb-8147-3e3b93786fd6","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:03:15.552082Z","iopub.execute_input":"2026-01-01T14:03:15.552370Z","iopub.status.idle":"2026-01-01T14:03:15.560404Z","shell.execute_reply.started":"2026-01-01T14:03:15.552345Z","shell.execute_reply":"2026-01-01T14:03:15.559867Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_data = expanded_test_desc\ntrain_data = final_merged_df","metadata":{"_cell_guid":"7e44f186-126c-4919-8f11-2761c8b06bf1","_uuid":"c7af545f-d5e5-4eb2-994d-cb97848b8546","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:03:16.800673Z","iopub.execute_input":"2026-01-01T14:03:16.801166Z","iopub.status.idle":"2026-01-01T14:03:16.804674Z","shell.execute_reply.started":"2026-01-01T14:03:16.801138Z","shell.execute_reply":"2026-01-01T14:03:16.803842Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.isnull().sum()","metadata":{"_cell_guid":"23dd5387-eac5-4dc4-afe1-6a41da1b85c2","_uuid":"05140088-d100-4f96-9b65-edb0eec7ae23","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:03:18.030105Z","iopub.execute_input":"2026-01-01T14:03:18.030762Z","iopub.status.idle":"2026-01-01T14:03:18.051732Z","shell.execute_reply.started":"2026-01-01T14:03:18.030732Z","shell.execute_reply":"2026-01-01T14:03:18.051121Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Checking whether there are any errors or outliers in the co-ordinates that are necessary to remove or not","metadata":{"_cell_guid":"53a3cb20-8556-402b-80f7-935023f7af39","_uuid":"cb3aadcc-9db1-4008-a386-7ed28cf483d0","trusted":true}},{"cell_type":"markdown","source":"# Data Visualizations","metadata":{"_cell_guid":"68ddfae7-7959-42c3-a1a6-edece55fdceb","_uuid":"d98a81e1-8d6f-4d7f-95a3-3b280d0cf1a7","trusted":true}},{"cell_type":"code","source":"# Display basic statistics for 'x' and 'y' columns\nx_stats = train_data['x'].describe()\ny_stats = train_data['y'].describe()\n\nprint(\"X Coordinate Statistics:\")\nprint(x_stats)\n\nprint(\"\\nY Coordinate Statistics:\")\nprint(y_stats)","metadata":{"_cell_guid":"74229ce7-9bea-48f6-8ee8-bd2e37371534","_uuid":"b9428fa6-02bf-45e2-97c0-365516a8faa6","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:03:21.631202Z","iopub.execute_input":"2026-01-01T14:03:21.631496Z","iopub.status.idle":"2026-01-01T14:03:21.645323Z","shell.execute_reply.started":"2026-01-01T14:03:21.631463Z","shell.execute_reply":"2026-01-01T14:03:21.644587Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\n\n# Create a histogram for 'x' values\nx_hist = go.Histogram(\n    x=train_data['x'],\n    nbinsx=30,\n    name='X Coordinates',\n    marker_color='blue',\n    opacity=0.7\n)\n\n# Create a histogram for 'y' values\ny_hist = go.Histogram(\n    x=train_data['y'],\n    nbinsx=30,\n    name='Y Coordinates',\n    marker_color='green',\n    opacity=0.7\n)\n\n# Create a figure with subplots\nfig = make_subplots(rows=1, cols=2, subplot_titles=('Distribution of X Coordinates', 'Distribution of Y Coordinates'))\n\n# Add the histograms to the figure\nfig.add_trace(x_hist, row=1, col=1)\nfig.add_trace(y_hist, row=1, col=2)\n\n# Update layout for a cleaner look\nfig.update_layout(\n    title_text=\"Distribution of X and Y Coordinates\",\n    showlegend=False,\n    xaxis_title=\"X Values\",\n    yaxis_title=\"Frequency\",\n    xaxis2_title=\"Y Values\",\n    yaxis2_title=\"Frequency\",\n    bargap=0.2,  # Gap between bars\n)\n\n# Show the plot\nfig.show()","metadata":{"_cell_guid":"d98c3da8-50e1-49f4-ae1e-8e710301c486","_uuid":"0e6adc83-237d-42ee-aeb6-7b5e6763762a","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:03:24.041985Z","iopub.execute_input":"2026-01-01T14:03:24.042794Z","iopub.status.idle":"2026-01-01T14:03:25.810350Z","shell.execute_reply.started":"2026-01-01T14:03:24.042756Z","shell.execute_reply":"2026-01-01T14:03:25.809401Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import plotly.express as px\n\n# Count the occurrences of each severity within each condition\nseverity_condition_counts = train_data.groupby(['condition', 'severity']).size().reset_index(name='count')\n\n# Create a grouped bar chart\nfig = px.bar(\n    severity_condition_counts,\n    x='condition',\n    y='count',\n    color='severity',\n    barmode='group',\n    title='Distribution of Severities for Each Condition',\n    labels={'condition': 'Condition', 'count': 'Number of Cases', 'severity': 'Severity'},\n    color_discrete_sequence=px.colors.qualitative.Set1  # Custom color sequence\n)\n\n# Update the layout for better presentation\nfig.update_layout(\n    xaxis_title='Condition',\n    yaxis_title='Number of Cases',\n    legend_title='Severity',\n    bargap=0.15,\n    bargroupgap=0.1\n)\n\nfig.show()","metadata":{"_cell_guid":"547bdb5d-032c-45e3-b9cc-9ff07770f012","_uuid":"76a2f293-ffe9-4344-bf76-659ded8f4e74","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:03:30.041023Z","iopub.execute_input":"2026-01-01T14:03:30.041730Z","iopub.status.idle":"2026-01-01T14:03:31.737138Z","shell.execute_reply.started":"2026-01-01T14:03:30.041701Z","shell.execute_reply":"2026-01-01T14:03:31.736558Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Count the occurrences of each severity within each condition\nseverity_condition_counts = train_data.groupby(['condition', 'series_description']).size().reset_index(name='count')\n\n# Create a grouped bar chart\nfig = px.bar(\n    severity_condition_counts,\n    x='condition',\n    y='count',\n    color='series_description',\n    barmode='group',\n    title='Distribution of Condition for Respective Angle',\n    labels={'condition': 'Condition', 'count': 'Number of Cases', 'series_description': 'Angle of MR Image'},\n    color_discrete_sequence=px.colors.qualitative.Set1  # Custom color sequence\n)\n\n# Update the layout for better presentation\nfig.update_layout(\n    xaxis_title='Condition',\n    yaxis_title='Number of Cases',\n    legend_title='Angle',\n    bargap=0.15,\n    bargroupgap=0.1\n)\n\nfig.show()","metadata":{"_cell_guid":"2d053601-ff9f-4855-9d48-08c894603e99","_uuid":"7c5b6cf6-b860-4cc0-a882-00b144b40e96","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:03:35.676726Z","iopub.execute_input":"2026-01-01T14:03:35.677477Z","iopub.status.idle":"2026-01-01T14:03:35.731174Z","shell.execute_reply.started":"2026-01-01T14:03:35.677440Z","shell.execute_reply":"2026-01-01T14:03:35.730477Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"In medical imaging for spinal conditions, specific MRI sequences are often used to identify different types of spinal stenosis:\n\n- **Sagittal T1-weighted images** are primarily utilized to evaluate **Neural Foraminal Narrowing**. \n- **Axial T2-weighted images** are crucial for assessing **Subarticular Stenosis**. \n- **Sagittal T2-weighted or STIR (Short Tau Inversion Recovery) images** are typically employed to detect and analyze **Spinal Canal Stenosis**.\n\nThese imaging sequences are chosen for their ability to provide the most relevant anatomical and pathological information for each specific type of stenosis.","metadata":{"_cell_guid":"5a0331a5-d395-4fea-8f6f-24e2b872cad7","_uuid":"dc2582c7-942e-49e5-a10e-f3f0d044727e","trusted":true}},{"cell_type":"code","source":"# Group by 'level' and 'condition' and count the occurrences\nlevel_condition_counts = train_data.groupby(['condition', 'level']).size().reset_index(name='count')\n\n# Create a grouped bar chart\nfig = px.bar(\n    level_condition_counts,\n    x='condition',\n    y='count',\n    color='level',\n    barmode='group',\n    title='Distribution of Levels for Each Condition',\n    labels={'condition': 'Condition', 'count': 'Number of Cases', 'level': 'Level'},\n    color_discrete_sequence=px.colors.qualitative.Set1  # Custom color sequence\n)\n\n# Update the layout for better presentation\nfig.update_layout(\n    xaxis_title='Condition',\n    yaxis_title='Number of Cases',\n    legend_title='Level',\n    bargap=0.15,\n    bargroupgap=0.1\n)\n\nfig.show()","metadata":{"_cell_guid":"fa01ff2a-2ad3-48db-ab44-5b6e0b98326a","_uuid":"694c40e4-9768-4dd9-b3fc-8670f0d5edb1","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:03:40.822434Z","iopub.execute_input":"2026-01-01T14:03:40.822774Z","iopub.status.idle":"2026-01-01T14:03:40.878826Z","shell.execute_reply.started":"2026-01-01T14:03:40.822748Z","shell.execute_reply":"2026-01-01T14:03:40.878241Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Count the occurrences of each level within each condition\nlevel_condition_counts = train_data.groupby(['level', 'condition']).size().reset_index(name='count')\n\n# Create a pivot table to structure the data for the heatmap\nheatmap_data = level_condition_counts.pivot(index='level', columns='condition', values='count')\n\n# Create the heatmap\nfig = px.imshow(\n    heatmap_data,\n    labels={'x': 'Condition', 'y': 'Level', 'color': 'Count'},\n    title='Heatmap of Levels by Condition',\n    color_continuous_scale='Viridis'\n)\n\nfig.show()","metadata":{"_cell_guid":"3ed6c023-4da8-47cb-a8ad-534a61157724","_uuid":"1ad5c50f-ca9c-4f64-ba8b-4887bf63efb0","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:03:44.678094Z","iopub.execute_input":"2026-01-01T14:03:44.678615Z","iopub.status.idle":"2026-01-01T14:03:44.757764Z","shell.execute_reply.started":"2026-01-01T14:03:44.678587Z","shell.execute_reply":"2026-01-01T14:03:44.757191Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Export the DataFrame to a CSV file\nfinal_merged_df.to_csv('train_processed.csv', index=False)\ntest_data.to_csv('test_processed.csv', index=False)","metadata":{"_cell_guid":"003c6c07-be21-418b-96a0-7d4c8016526d","_uuid":"a62a3e46-b1bb-463e-a524-35320211c967","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:03:47.660150Z","iopub.execute_input":"2026-01-01T14:03:47.660828Z","iopub.status.idle":"2026-01-01T14:03:48.138773Z","shell.execute_reply.started":"2026-01-01T14:03:47.660800Z","shell.execute_reply":"2026-01-01T14:03:48.138186Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data Preprocessing and SMOTE Resampling","metadata":{}},{"cell_type":"code","source":"from imblearn.over_sampling import SMOTE\nfrom sklearn.preprocessing import LabelEncoder, OneHotEncoder\nimport pandas as pd\n\n# Mapping for the 'level' column\nlevel_mapping = {'L1/L2': 0, 'L2/L3': 1, 'L3/L4': 2, 'L4/L5': 3, 'L5/S1': 4}\nfinal_merged_df['level_encoded'] = final_merged_df['level'].map(level_mapping)\n\n# Mapping for the 'series_description' column\nseries_description_mapping = {'Sagittal T1': 0, 'Sagittal T2/STIR': 1, 'Axial T2': 2}\nfinal_merged_df['series_description_encoded'] = final_merged_df['series_description'].map(series_description_mapping)\n\n# Convert categorical features to numerical using LabelEncoder for 'condition' and 'severity'\nle_condition = LabelEncoder()\nle_severity = LabelEncoder()\n\nfinal_merged_df['condition_encoded'] = le_condition.fit_transform(final_merged_df['condition'])\nfinal_merged_df['severity_encoded'] = le_severity.fit_transform(final_merged_df['severity'])\n\n# Concatenate condition and severity labels\nfinal_merged_df['combined_label'] = final_merged_df['condition_encoded'].astype(str) + \"_\" + final_merged_df['severity_encoded'].astype(str)\n\n# Remove non-numeric columns and use only relevant features\nX = final_merged_df.drop(['condition', 'severity', 'row_id', 'image_path', 'level', 'series_description', 'condition_encoded', 'severity_encoded', 'combined_label'], axis=1)\nX['level'] = final_merged_df['level_encoded']  # Add the newly encoded 'level' column\nX['series_description'] = final_merged_df['series_description_encoded']  # Add the newly encoded 'series_description' column\n\n# SMOTE: Apply SMOTE to the concatenated labels (condition + severity)\ny_combined = final_merged_df['combined_label']\n\nsmote = SMOTE(random_state=42)\nX_smote, y_combined_smote = smote.fit_resample(X, y_combined)\n\n# Split back the combined label into separate condition and severity labels\ny_condition_smote = y_combined_smote.str.split(\"_\", expand=True)[0].astype(int)  # Condition part\ny_severity_smote = y_combined_smote.str.split(\"_\", expand=True)[1].astype(int)  # Severity part\n\n# One-hot encode the target labels for multi-class classification\nonehot = OneHotEncoder()\n\ny_condition_smote_oh = onehot.fit_transform(y_condition_smote.values.reshape(-1, 1)).toarray()\ny_severity_smote_oh = onehot.fit_transform(y_severity_smote.values.reshape(-1, 1)).toarray()\n\n# Print the shapes to verify correctness\nprint(f\"Shape of X_smote: {X_smote.shape}\")\nprint(f\"Shape of y_condition_smote_oh: {y_condition_smote_oh.shape}\")\nprint(f\"Shape of y_severity_smote_oh: {y_severity_smote_oh.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:03:52.950669Z","iopub.execute_input":"2026-01-01T14:03:52.951322Z","iopub.status.idle":"2026-01-01T14:03:54.966970Z","shell.execute_reply.started":"2026-01-01T14:03:52.951294Z","shell.execute_reply":"2026-01-01T14:03:54.966304Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"In this step, categorical features like level and series_description are mapped to numeric values. The target variables condition and severity are label-encoded and combined. SMOTE (Synthetic Minority Oversampling Technique) is applied to balance the dataset by generating synthetic samples for minority classes. After resampling, the combined labels are split back into separate condition and severity labels, which are then one-hot encoded for multi-class classification. The resulting dataset is now balanced and ready for model training.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\n# Step 1: Retain 'study_id', 'series_id', 'instance_number', 'x', 'y', and 'level' from the original dataset\nX_combined = final_merged_df[['study_id', 'series_id', 'instance_number', 'x', 'y', 'level']].copy()\n\n# Mapping for the 'level' column as specified\nlevel_mapping = {'L1/L2': 1, 'L2/L3': 2, 'L3/L4': 3, 'L4/L5': 4, 'L5/S1': 5}\nX_combined['level'] = X_combined['level'].map(level_mapping)\n\n# Step 2: Initialize y_condition_smote_df and apply condition mapping as specified\ncondition_mapping = {\n    'Spinal Canal Stenosis': 1,\n    'Left Neural Foraminal Narrowing': 2,\n    'Right Neural Foraminal Narrowing': 3,\n    'Left Subarticular Stenosis': 4,\n    'Right Subarticular Stenosis': 5\n}\n\n# Assuming y_condition_smote_oh is available from SMOTE\ny_condition_smote_df = pd.DataFrame(y_condition_smote_oh, columns=[f\"condition_{i}\" for i in range(y_condition_smote_oh.shape[1])])\n\n# Apply the condition mapping\ny_condition_smote_df = y_condition_smote_df.replace(condition_mapping)\n\n# Step 3: Initialize y_severity_smote_df from SMOTE results\n# Assuming y_severity_smote_oh is available from SMOTE\ny_severity_smote_df = pd.DataFrame(y_severity_smote_oh, columns=[f\"severity_{i}\" for i in range(y_severity_smote_oh.shape[1])])\n\n# Step 4: Convert the integer-related columns in X_combined to actual integers\nX_combined['study_id'] = X_combined['study_id'].astype(int)\nX_combined['series_id'] = X_combined['series_id'].astype(int)\nX_combined['instance_number'] = X_combined['instance_number'].astype(int)\nX_combined['level'] = X_combined['level'].astype(int)\n\n# Keep 'x' and 'y' as float\nX_combined['x'] = X_combined['x'].astype(float)\nX_combined['y'] = X_combined['y'].astype(float)\n\n# Step 5: Initialize X_smote_df from SMOTE result\nX_smote_df = pd.DataFrame(X_smote, columns=[f\"feature_{i}\" for i in range(X_smote.shape[1])])\n\n# Step 6: Drop columns 'feature_0' to 'feature_8'\nX_smote_df.drop(columns=[f\"feature_{i}\" for i in range(9)], inplace=True)\n\n# Step 7: Combine SMOTE features with 'study_id', 'series_id', 'instance_number', 'x', 'y', and 'level'\nX_smote_combined_df = pd.concat([X_combined, X_smote_df], axis=1)\n\n# Step 8: Drop rows where all values are NaN/null\nX_smote_combined_df.dropna(how='all', inplace=True)\n\n# Step 9: Prevent scientific notation for integer columns and save as CSV\npd.set_option('display.float_format', '{:.0f}'.format)  # Disable scientific notation for integers","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:03:58.452399Z","iopub.execute_input":"2026-01-01T14:03:58.453538Z","iopub.status.idle":"2026-01-01T14:03:58.490252Z","shell.execute_reply.started":"2026-01-01T14:03:58.453500Z","shell.execute_reply":"2026-01-01T14:03:58.489708Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"`condition_0`: **Spinal Canal Stenosis**\n\n`condition_1`: **Left Neural Foraminal Narrowing**\n\n`condition_2`: **Right Neural Foraminal Narrowing**\n\n`condition_3`: **Left Subarticular Stenosis**\n\n`condition_4`: **Right Subarticular Stenosis**","metadata":{}},{"cell_type":"markdown","source":"`severity_0`: **Normal/Mild**\n\n`severity_1`: **Moderate**\n\n`severity_2`: **Severe**","metadata":{}},{"cell_type":"code","source":"\n# Check the number of samples\nprint(f\"Filtered X_smote_combined_df size: {len(X_smote_combined_df)}\")  # Should be 48,692\n\n# Ensure the labels are of the same size as X_smote_combined_df by filtering the corresponding labels\n# Make sure this aligns with how rows were dropped earlier in X_smote_combined_df\ny_condition_smote_df_filtered = y_condition_smote_df.iloc[:len(X_smote_combined_df)]\ny_severity_smote_df_filtered = y_severity_smote_df.iloc[:len(X_smote_combined_df)]\n\n# Now check the sizes\nprint(f\"y_condition_smote_df size after filtering: {len(y_condition_smote_df_filtered)}\")  # Should match 48,692\nprint(f\"y_severity_smote_df size after filtering: {len(y_severity_smote_df_filtered)}\")    # Should match 48,692","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:04:02.138449Z","iopub.execute_input":"2026-01-01T14:04:02.138844Z","iopub.status.idle":"2026-01-01T14:04:02.144924Z","shell.execute_reply.started":"2026-01-01T14:04:02.138818Z","shell.execute_reply":"2026-01-01T14:04:02.144243Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save the combined DataFrame and labels to CSV files\nX_smote_combined_df.to_csv('/kaggle/working/X_smote_combined.csv', index=False)\ny_condition_smote_df.to_csv('/kaggle/working/y_condition_smote_oh.csv', index=False)\ny_severity_smote_df.to_csv('/kaggle/working/y_severity_smote_oh.csv', index=False)\n\n# Confirm the export\nprint(\"/kaggle/working/X_smote_combined.csv, /kaggle/working/y_condition_smote_oh.csv, /kaggle/working/y_severity_smote_oh.csv exported successfully\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:04:05.122787Z","iopub.execute_input":"2026-01-01T14:04:05.123078Z","iopub.status.idle":"2026-01-01T14:04:05.833680Z","shell.execute_reply.started":"2026-01-01T14:04:05.123054Z","shell.execute_reply":"2026-01-01T14:04:05.833017Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Pre-processing & Loading Images","metadata":{}},{"cell_type":"code","source":"import os\nimport pydicom\nimport cv2\nimport numpy as np\nfrom tensorflow.keras.utils import Sequence\n\n# Data generator class\nclass DataGenerator(Sequence):\n    def __init__(self, df, image_folder, y_condition, y_severity, batch_size=32, img_size=(224, 224)):\n        self.df = df\n        self.image_folder = image_folder\n        self.y_condition = y_condition\n        self.y_severity = y_severity\n        self.batch_size = batch_size\n        self.img_size = img_size\n        self.indices = np.arange(len(df))\n    \n    def __len__(self):\n        return int(np.ceil(len(self.df) / self.batch_size))\n    \n    def __getitem__(self, index):\n        # Generate batch indices\n        batch_indices = self.indices[index * self.batch_size:(index + 1) * self.batch_size]\n        \n        # Get the batch data\n        batch_df = self.df.iloc[batch_indices]\n        batch_y_condition = self.y_condition[batch_indices]\n        batch_y_severity = self.y_severity[batch_indices]\n        \n        # Load the images for the batch\n        images = []\n        for _, row in batch_df.iterrows():\n            study_id = int(row['study_id'])  # Ensure integer format\n            series_id = int(row['series_id'])  # Ensure integer format\n            instance_number = int(row['instance_number'])  # Ensure integer format\n            \n            # Construct image path using integer values (convert to string)\n            img_path = os.path.join(self.image_folder, str(study_id), str(series_id), f\"{instance_number}.dcm\")\n            \n            # Load and preprocess the DICOM image\n            img = self.load_dicom_image(img_path)\n            images.append(img)\n        \n        return np.array(images), {'condition_output': batch_y_condition, 'severity_output': batch_y_severity}\n    \n    def load_dicom_image(self, image_path):\n        try:\n            dicom = pydicom.dcmread(image_path)\n            if hasattr(dicom, 'pixel_array'):\n                img = dicom.pixel_array\n                img = cv2.resize(img, self.img_size)  # Resize to 224x224\n                img = np.stack((img,) * 3, axis=-1)  # Convert to RGB (3 channels)\n                img = img / 255.0  # Normalize\n                return img\n            else:\n                print(f\"Warning: No pixel data in DICOM file: {image_path}\")\n                return np.zeros((*self.img_size, 3))  # Return a blank image if no pixel data\n        except Exception as e:\n            print(f\"Error reading DICOM file: {image_path}. Error: {e}\")\n            return np.zeros((*self.img_size, 3))  # Return a blank image if any error occurs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:04:08.712860Z","iopub.execute_input":"2026-01-01T14:04:08.713347Z","iopub.status.idle":"2026-01-01T14:04:08.725009Z","shell.execute_reply.started":"2026-01-01T14:04:08.713318Z","shell.execute_reply":"2026-01-01T14:04:08.724396Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Build the EfficientNetB0 Model","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.applications import EfficientNetB0\nfrom tensorflow.keras.layers import Dense, Flatten, Input\nfrom tensorflow.keras.models import Model\n\n# Function to build the EfficientNetB0 model\ndef build_efficientnet_model(num_classes_condition, num_classes_severity, input_shape=(224, 224, 3)):\n    inputs = Input(shape=input_shape)\n    base_model = EfficientNetB0(include_top=False, weights='imagenet', input_tensor=inputs)\n    \n    # Add global pooling and dense layers for both condition and severity\n    x = Flatten()(base_model.output)\n    \n    # Output for condition classification\n    condition_output = Dense(num_classes_condition, activation='softmax', name='condition_output')(x)\n    \n    # Output for severity classification\n    severity_output = Dense(num_classes_severity, activation='softmax', name='severity_output')(x)\n    \n    # Define the model\n    model = Model(inputs, outputs=[condition_output, severity_output])\n    \n    # Compile the model\n    model.compile(optimizer='adam',\n                  loss={'condition_output': 'categorical_crossentropy', 'severity_output': 'categorical_crossentropy'},\n                  metrics={'condition_output': 'accuracy', 'severity_output': 'accuracy'})\n    \n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:04:12.158178Z","iopub.execute_input":"2026-01-01T14:04:12.158880Z","iopub.status.idle":"2026-01-01T14:04:12.164451Z","shell.execute_reply.started":"2026-01-01T14:04:12.158850Z","shell.execute_reply":"2026-01-01T14:04:12.163884Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Build RNN Model with LSTM for Sequential Medical Image Analysis","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Input, Conv2D, MaxPooling2D, BatchNormalization, LSTM, Bidirectional, Dropout, TimeDistributed\nfrom tensorflow.keras.models import Model\ndef build_rnn_model(num_classes_condition, num_classes_severity, input_shape=(224, 224, 3), timesteps=5):\n    \"\"\"\n    IMPROVED RNN/LSTM Model\n    - Deeper CNN (3 blocks)\n    - Global Average Pooling (drastically reduces parameters/memory)\n    - Bidirectional LSTM (better context)\n    \"\"\"\n    inputs = Input(shape=(timesteps, *input_shape))\n    \n    # --- 1. Feature Extraction (Improved) ---\n    # Block 1\n    x = TimeDistributed(Conv2D(16, (3, 3), activation='relu', padding='same'))(inputs)\n    x = TimeDistributed(BatchNormalization())(x)\n    x = TimeDistributed(MaxPooling2D((2, 2)))(x)\n    \n    # Block 2\n    x = TimeDistributed(Conv2D(32, (3, 3), activation='relu', padding='same'))(x)\n    x = TimeDistributed(BatchNormalization())(x)\n    x = TimeDistributed(MaxPooling2D((2, 2)))(x)\n    \n    # Block 3 (New separate block for deeper features)\n    x = TimeDistributed(Conv2D(64, (3, 3), activation='relu', padding='same'))(x)\n    x = TimeDistributed(BatchNormalization())(x)\n    x = TimeDistributed(MaxPooling2D((2, 2)))(x)\n    \n    # Global Pooling instead of Flatten (Crucial Modification)\n    # Reduces feature map to vector without massive Dense layers\n    x = TimeDistributed(GlobalAveragePooling2D())(x)\n    \n    # --- 2. Sequence Analysis ---\n    # Bidirectional LSTM to capture patterns in both directions (up/down spine)\n    x = Bidirectional(LSTM(64, return_sequences=False, dropout=0.3))(x)\n    \n    # --- 3. Classification Heads ---\n    x = Dense(64, activation='relu')(x)\n    x = Dropout(0.3)(x)\n    \n    condition_output = Dense(num_classes_condition, activation='softmax', name='condition_output')(x)\n    severity_output = Dense(num_classes_severity, activation='softmax', name='severity_output')(x)\n    \n    model = Model(inputs=inputs, outputs=[condition_output, severity_output])\n    \n    model.compile(\n        optimizer='adam',\n        loss={'condition_output': 'categorical_crossentropy', 'severity_output': 'categorical_crossentropy'},\n        metrics={'condition_output': 'accuracy', 'severity_output': 'accuracy'}\n    )\n    \n    return model\nprint(\"Improved build_rnn_model defined successfully\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:04:16.678080Z","iopub.execute_input":"2026-01-01T14:04:16.678613Z","iopub.status.idle":"2026-01-01T14:04:16.687665Z","shell.execute_reply.started":"2026-01-01T14:04:16.678584Z","shell.execute_reply":"2026-01-01T14:04:16.686881Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ALTERNATIVE: MEMORY-EFFICIENT GRU model (Use this if LSTM still causes OOM)\nfrom tensorflow.keras.layers import GRU, BatchNormalization\n\ndef build_gru_model_lightweight(num_classes_condition, num_classes_severity, input_shape=(224, 224, 3), timesteps=3):\n    \"\"\"\n    MINIMAL GRU-based model - ULTRA LIGHTWEIGHT for severe GPU memory constraints\n    Uses single Conv2D + GRU instead of TimeDistributed\n    \"\"\"\n    inputs = Input(shape=(timesteps, *input_shape))\n    \n    # Single lightweight Conv layer\n    x = TimeDistributed(Conv2D(8, (3, 3), activation='relu'))(inputs)\n    x = TimeDistributed(MaxPooling2D((2, 2)))(x)\n    x = TimeDistributed(Flatten())(x)\n    \n    # Single GRU layer (minimal units)\n    x = GRU(16, return_sequences=False, dropout=0.2)(x)\n    \n    # Minimal Dense layer\n    x = Dense(16, activation='relu')(x)\n    x = Dropout(0.2)(x)\n    \n    # Output layers\n    condition_output = Dense(num_classes_condition, activation='softmax', name='condition_output')(x)\n    severity_output = Dense(num_classes_severity, activation='softmax', name='severity_output')(x)\n    \n    model = Model(inputs=inputs, outputs=[condition_output, severity_output])\n    \n    model.compile(\n        optimizer='adam',\n        loss={'condition_output': 'categorical_crossentropy', 'severity_output': 'categorical_crossentropy'},\n        metrics={'condition_output': 'accuracy', 'severity_output': 'accuracy'}\n    )\n    \n    return model\n\nprint(\"MEMORY-EFFICIENT GRU Model available as backup\")\nprint(\"Use this if LSTM still causes ResourceExhaustedError:\")\nprint(\"  rnn_model = build_gru_model_lightweight(5, 3, timesteps=3)\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:04:23.096869Z","iopub.execute_input":"2026-01-01T14:04:23.097400Z","iopub.status.idle":"2026-01-01T14:04:23.104017Z","shell.execute_reply.started":"2026-01-01T14:04:23.097371Z","shell.execute_reply":"2026-01-01T14:04:23.103319Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## RNN Data Generator for Sequential Images\n\nSince lumbar spine MRI scans contain multiple slices (sequential images), we can leverage RNN/LSTM to analyze these sequences. This generator loads multiple consecutive DICOM slices as a sequence.","metadata":{}},{"cell_type":"code","source":"class SequentialDataGenerator(Sequence):\n    \"\"\"\n    Data generator for loading sequential DICOM images for RNN/LSTM models\n    Loads multiple consecutive slices from the same series\n    \"\"\"\n    def __init__(self, df, image_folder, y_condition, y_severity, batch_size=16, img_size=(224, 224), sequence_length=10):\n        # RESET INDEX to ensure it aligns with the 0-based numpy arrays (y_condition, y_severity)\n        self.df = df.reset_index(drop=True)\n        \n        self.image_folder = image_folder\n        self.y_condition = y_condition\n        self.y_severity = y_severity\n        self.batch_size = batch_size\n        self.img_size = img_size\n        self.sequence_length = sequence_length\n        \n        # Group by study_id and series_id to get sequences\n        self.grouped = self.df.groupby(['study_id', 'series_id'])\n        self.group_keys = list(self.grouped.groups.keys())\n        self.indices = np.arange(len(self.group_keys))\n    \n    def __len__(self):\n        return int(np.ceil(len(self.group_keys) / self.batch_size))\n    \n    def __getitem__(self, index):\n        batch_indices = self.indices[index * self.batch_size:(index + 1) * self.batch_size]\n        batch_keys = [self.group_keys[i] for i in batch_indices]\n        \n        sequences = []\n        batch_y_condition = []\n        batch_y_severity = []\n        \n        for study_id, series_id in batch_keys:\n            group_df = self.grouped.get_group((study_id, series_id))\n            \n            # Sort by instance number to maintain sequence\n            group_df = group_df.sort_values('instance_number')\n            \n            # Load sequence of images\n            sequence = []\n            for _, row in group_df.head(self.sequence_length).iterrows():\n                img_path = os.path.join(\n                    self.image_folder, \n                    str(int(row['study_id'])), \n                    str(int(row['series_id'])), \n                    f\"{int(row['instance_number'])}.dcm\"\n                )\n                img = self.load_dicom_image(img_path)\n                sequence.append(img)\n            \n            # Pad sequence if shorter than sequence_length\n            while len(sequence) < self.sequence_length:\n                sequence.append(np.zeros((*self.img_size, 3)))\n            \n            sequences.append(sequence)\n            \n            # Get labels (use first row of the group)\n            # Since we reset index, first_idx will now be a valid index for y_condition/y_severity\n            first_idx = group_df.index[0]\n            batch_y_condition.append(self.y_condition[first_idx])\n            batch_y_severity.append(self.y_severity[first_idx])\n        \n        return (\n            np.array(sequences), \n            {\n                'condition_output': np.array(batch_y_condition), \n                'severity_output': np.array(batch_y_severity)\n            }\n        )\n    \n    def load_dicom_image(self, image_path):\n        try:\n            dicom = pydicom.dcmread(image_path)\n            if hasattr(dicom, 'pixel_array'):\n                img = dicom.pixel_array\n                img = cv2.resize(img, self.img_size)\n                img = np.stack((img,) * 3, axis=-1)\n                img = img / 255.0\n                return img\n            else:\n                return np.zeros((*self.img_size, 3))\n        except Exception as e:\n            # print(f\"Error reading DICOM file: {image_path}. Error: {e}\")\n            return np.zeros((*self.img_size, 3))\n\nprint(\"Sequential Data Generator updated with index fix.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:04:26.039594Z","iopub.execute_input":"2026-01-01T14:04:26.040310Z","iopub.status.idle":"2026-01-01T14:04:26.051048Z","shell.execute_reply.started":"2026-01-01T14:04:26.040282Z","shell.execute_reply":"2026-01-01T14:04:26.050362Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Train RNN/LSTM Model\n\nTraining the RNN model with sequential DICOM images to capture temporal patterns across multiple slices.","metadata":{}},{"cell_type":"code","source":"# Define image folder for training/validation generators\n# Update this path if your data lives elsewhere\nimage_folder = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images'\nprint('Using image folder:', image_folder)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:04:30.243790Z","iopub.execute_input":"2026-01-01T14:04:30.244281Z","iopub.status.idle":"2026-01-01T14:04:30.248412Z","shell.execute_reply.started":"2026-01-01T14:04:30.244256Z","shell.execute_reply":"2026-01-01T14:04:30.247685Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Prepare data for RNN training\n# Split data maintaining sequential structure\nX_train_rnn, X_val_rnn, y_train_cond_rnn, y_val_cond_rnn, y_train_sev_rnn, y_val_sev_rnn = train_test_split(\n    X_smote_combined_df, \n    y_condition_smote_df_filtered.values, \n    y_severity_smote_df_filtered.values, \n    test_size=0.2, \n    random_state=42\n)\n\nprint(f\"RNN Training set: {len(X_train_rnn)} samples\")\nprint(f\"RNN Validation set: {len(X_val_rnn)} samples\")\n\n# Create sequential data generators with ULTRA-REDUCED batch size and sequence length\ntrain_seq_generator = SequentialDataGenerator(\n    X_train_rnn, \n    image_folder, \n    y_train_cond_rnn, \n    y_train_sev_rnn, \n    batch_size=4,  # ULTRA-REDUCED from 8 to 4\n    sequence_length=3  # REDUCED from 5 to 3\n)\n\nval_seq_generator = SequentialDataGenerator(\n    X_val_rnn, \n    image_folder, \n    y_val_cond_rnn, \n    y_val_sev_rnn, \n    batch_size=4,  # ULTRA-REDUCED from 8 to 4\n    sequence_length=3  # REDUCED from 5 to 3\n)\n\n# Build ULTRA-LIGHTWEIGHT RNN model with reduced complexity\nrnn_model = build_rnn_model(num_classes_condition=5, num_classes_severity=3, timesteps=3)\n\n# Display model summary\nrnn_model.summary()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:04:32.266588Z","iopub.execute_input":"2026-01-01T14:04:32.267134Z","iopub.status.idle":"2026-01-01T14:04:34.422701Z","shell.execute_reply.started":"2026-01-01T14:04:32.267107Z","shell.execute_reply":"2026-01-01T14:04:34.422149Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tensorflow as tf\nimport gc\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n\n# 1. Clear session\ntf.keras.backend.clear_session()\ngc.collect()\n\n# 2. INCREASE BATCH SIZE for multi-GPU efficiency\nGLOBAL_BATCH_SIZE = 16 \n\nprint(f\"Re-creating generators with optimized batch size: {GLOBAL_BATCH_SIZE}\")\n\n# Re-initialize generators with larger batch size\ntrain_seq_generator = SequentialDataGenerator(\n    X_train_rnn, \n    image_folder, \n    y_train_cond_rnn, \n    y_train_sev_rnn, \n    batch_size=GLOBAL_BATCH_SIZE,\n    sequence_length=3\n)\n\nval_seq_generator = SequentialDataGenerator(\n    X_val_rnn, \n    image_folder, \n    y_val_cond_rnn, \n    y_val_sev_rnn, \n    batch_size=GLOBAL_BATCH_SIZE,\n    sequence_length=3\n)\n\n# 3. Define Strategy\nstrategy = tf.distribute.MirroredStrategy()\nprint(f\"Number of devices: {strategy.num_replicas_in_sync}\")\n\n# 4. Build & Compile INSIDE scope\nwith strategy.scope():\n    rnn_model = build_rnn_model(num_classes_condition=5, num_classes_severity=3, timesteps=3)\n    \n    # Optional: Slightly higher learning rate for larger batches\n    opt = tf.keras.optimizers.Adam(learning_rate=0.001) \n    \n    rnn_model.compile(\n        optimizer=opt,\n        loss={'condition_output': 'categorical_crossentropy', 'severity_output': 'categorical_crossentropy'},\n        metrics={'condition_output': 'accuracy', 'severity_output': 'accuracy'}\n    )\n\n# 5. Define Callbacks\nrnn_earlystop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True, verbose=1)\nrnn_checkpoint = ModelCheckpoint('rnn_lstm_model.keras', save_best_only=True, verbose=1)\nreduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=2, min_lr=1e-7, verbose=1)\n\n# 6. Train with workers REMOVED to fix TypeError\nprint(\"Starting Optimized Distributed RNN Training...\")\nrnn_history = rnn_model.fit(\n    train_seq_generator,\n    validation_data=val_seq_generator,\n    epochs=10,\n    callbacks=[rnn_earlystop, rnn_checkpoint, reduce_lr],\n    verbose=1\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T14:04:41.283201Z","iopub.execute_input":"2026-01-01T14:04:41.283797Z","iopub.status.idle":"2026-01-01T14:45:14.534904Z","shell.execute_reply.started":"2026-01-01T14:04:41.283767Z","shell.execute_reply":"2026-01-01T14:45:14.534235Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## RNN Model Evaluation and Visualization","metadata":{}},{"cell_type":"code","source":"# Plot RNN training history\nfig, axes = plt.subplots(2, 2, figsize=(15, 10))\n\n# Condition accuracy\naxes[0, 0].plot(rnn_history.history['condition_output_accuracy'], label='Train')\naxes[0, 0].plot(rnn_history.history['val_condition_output_accuracy'], label='Validation')\naxes[0, 0].set_title('RNN Condition Classification Accuracy')\naxes[0, 0].set_xlabel('Epoch')\naxes[0, 0].set_ylabel('Accuracy')\naxes[0, 0].legend()\naxes[0, 0].grid(True)\n\n# Condition loss\naxes[0, 1].plot(rnn_history.history['condition_output_loss'], label='Train')\naxes[0, 1].plot(rnn_history.history['val_condition_output_loss'], label='Validation')\naxes[0, 1].set_title('RNN Condition Classification Loss')\naxes[0, 1].set_xlabel('Epoch')\naxes[0, 1].set_ylabel('Loss')\naxes[0, 1].legend()\naxes[0, 1].grid(True)\n\n# Severity accuracy\naxes[1, 0].plot(rnn_history.history['severity_output_accuracy'], label='Train')\naxes[1, 0].plot(rnn_history.history['val_severity_output_accuracy'], label='Validation')\naxes[1, 0].set_title('RNN Severity Classification Accuracy')\naxes[1, 0].set_xlabel('Epoch')\naxes[1, 0].set_ylabel('Accuracy')\naxes[1, 0].legend()\naxes[1, 0].grid(True)\n\n# Severity loss\naxes[1, 1].plot(rnn_history.history['severity_output_loss'], label='Train')\naxes[1, 1].plot(rnn_history.history['val_severity_output_loss'], label='Validation')\naxes[1, 1].set_title('RNN Severity Classification Loss')\naxes[1, 1].set_xlabel('Epoch')\naxes[1, 1].set_ylabel('Loss')\naxes[1, 1].legend()\naxes[1, 1].grid(True)\n\nplt.tight_layout()\nplt.show()\n\nprint(\"RNN training visualization complete\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T15:00:03.643569Z","iopub.execute_input":"2026-01-01T15:00:03.644398Z","iopub.status.idle":"2026-01-01T15:00:04.273131Z","shell.execute_reply.started":"2026-01-01T15:00:03.644367Z","shell.execute_reply":"2026-01-01T15:00:04.272449Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.metrics import classification_report\n\n# 1. EVALUATE & PREDICT (Crucial Step: Generate predictions first)\nprint(\"Evaluating RNN model on validation set...\")\nrnn_results = rnn_model.evaluate(val_seq_generator, verbose=1)\n\nprint(\"\\nGenerating predictions...\")\nrnn_pred_condition, rnn_pred_severity = rnn_model.predict(val_seq_generator)\n\n# Convert predictions to class labels\nrnn_pred_condition_classes = rnn_pred_condition.argmax(axis=1)\nrnn_pred_severity_classes = rnn_pred_severity.argmax(axis=1)\n\n# 2. Extract the TRUE labels from the generator (in the correct order)\nprint(\"Extracting true labels from generator...\")\ny_val_cond_true_seq = []\ny_val_sev_true_seq = []\n\n# Iterate through the generator to get the batch labels\nfor i in range(len(val_seq_generator)):\n    # Generator returns (inputs, targets)\n    _, y_batch = val_seq_generator[i]\n    \n    # targets is a dict: {'condition_output': ..., 'severity_output': ...}\n    y_val_cond_true_seq.extend(y_batch['condition_output'])\n    y_val_sev_true_seq.extend(y_batch['severity_output'])\n\n# Convert to numpy arrays\ny_val_cond_true_seq = np.array(y_val_cond_true_seq)\ny_val_sev_true_seq = np.array(y_val_sev_true_seq)\n\n# 3. Convert One-Hot Encoded labels to Class Integers (0, 1, 2...)\ny_val_cond_classes_seq = y_val_cond_true_seq.argmax(axis=1)\ny_val_sev_classes_seq = y_val_sev_true_seq.argmax(axis=1)\n\n# 4. Generate the report using the SEQUENCE-aligned labels\nprint(\"\\n=== RNN Classification Reports (Corrected) ===\")\n\nprint(\"\\n--- Condition Classification ---\")\nprint(classification_report(y_val_cond_classes_seq, rnn_pred_condition_classes))\n\nprint(\"\\n--- Severity Classification ---\")\nprint(classification_report(y_val_sev_classes_seq, rnn_pred_severity_classes))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T15:00:08.219293Z","iopub.execute_input":"2026-01-01T15:00:08.219910Z","iopub.status.idle":"2026-01-01T15:04:42.136015Z","shell.execute_reply.started":"2026-01-01T15:00:08.219879Z","shell.execute_reply":"2026-01-01T15:04:42.135297Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Plot confusion matrices for RNN model\nfrom sklearn.metrics import confusion_matrix\nimport seaborn as sns\n\nfig, axes = plt.subplots(1, 2, figsize=(16, 6))\n\n# Condition confusion matrix\n# Use y_val_cond_classes_seq (4979 samples) instead of y_val_cond_classes (9739 samples)\ncondition_cm_rnn = confusion_matrix(y_val_cond_classes_seq, rnn_pred_condition_classes)\nsns.heatmap(condition_cm_rnn, annot=True, fmt='d', cmap='Blues', ax=axes[0])\naxes[0].set_title('RNN Condition Classification Confusion Matrix')\naxes[0].set_xlabel('Predicted')\naxes[0].set_ylabel('Actual')\naxes[0].set_xticklabels(['SCS', 'LNFN', 'RNFN', 'LSS', 'RSS'], rotation=45)\naxes[0].set_yticklabels(['SCS', 'LNFN', 'RNFN', 'LSS', 'RSS'], rotation=0)\n\n# Severity confusion matrix\n# Use y_val_sev_classes_seq instead of y_val_sev_classes\nseverity_cm_rnn = confusion_matrix(y_val_sev_classes_seq, rnn_pred_severity_classes)\nsns.heatmap(severity_cm_rnn, annot=True, fmt='d', cmap='Greens', ax=axes[1])\naxes[1].set_title('RNN Severity Classification Confusion Matrix')\naxes[1].set_xlabel('Predicted')\naxes[1].set_ylabel('Actual')\naxes[1].set_xticklabels(['Normal/Mild', 'Moderate', 'Severe'], rotation=45)\naxes[1].set_yticklabels(['Normal/Mild', 'Moderate', 'Severe'], rotation=0)\n\nplt.tight_layout()\nplt.show()\n\nprint(\"\\nConfusion matrices generated for RNN model\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T15:07:34.313373Z","iopub.execute_input":"2026-01-01T15:07:34.314154Z","iopub.status.idle":"2026-01-01T15:07:35.092285Z","shell.execute_reply.started":"2026-01-01T15:07:34.314114Z","shell.execute_reply":"2026-01-01T15:07:35.091674Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save the RNN model\nrnn_model_save_path = os.path.join('/kaggle/working/', 'rnn_lstm_model.h5')\nrnn_model.save(rnn_model_save_path)\nprint(f\"RNN/LSTM model saved at {rnn_model_save_path}\")\n\n# Model comparison summary\nprint(\"\\n\" + \"=\"*60)\nprint(\"MODEL COMPARISON SUMMARY\")\nprint(\"=\"*60)\nprint(\"\\n1. EfficientNetB0 Model:\")\nprint(\"   - Architecture: Pre-trained CNN with transfer learning\")\nprint(\"   - Input: Single image analysis\")\nprint(\"   - Strengths: Strong feature extraction, good baseline performance\")\nprint(\"\\n2. RNN/LSTM Model:\")\nprint(\"   - Architecture: Sequential model with Bidirectional LSTM\")\nprint(\"   - Input: Sequence of images (temporal analysis)\")\nprint(\"   - Strengths: Captures patterns across multiple slices\")\nprint(\"   - Use case: Better for analyzing complete MRI series\")\nprint(\"\\n3. Recommendation:\")\nprint(\"   - Use EfficientNetB0 for: Single slice classification, faster inference\")\nprint(\"   - Use RNN/LSTM for: Complete series analysis, temporal patterns\")\nprint(\"   - Consider ensemble: Combine both models for best results\")\nprint(\"=\"*60)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T15:07:51.000223Z","iopub.execute_input":"2026-01-01T15:07:51.001098Z","iopub.status.idle":"2026-01-01T15:07:51.469541Z","shell.execute_reply.started":"2026-01-01T15:07:51.001068Z","shell.execute_reply":"2026-01-01T15:07:51.468907Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Train the Model in Batches with Training and Validation Split","metadata":{}},{"cell_type":"markdown","source":"Batch-wise splitting and yielding is used to efficiently handle large datasets like DICOM images without exceeding memory limits. By processing data in smaller batches, we reduce memory usage, preventing system crashes or notebook restarts. This approach allows for incremental loading, splitting, and model training while keeping only a portion of the data in memory at any time. It ensures scalability and stability when working with high-dimensional image data, optimizing both resource utilization and performance during training and validation.","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n\n# 1. Prepare Data Split (Restored from previous step)\nX_train_df, X_val_df, y_train_condition, y_val_condition, y_train_severity, y_val_severity = train_test_split(\n    X_smote_combined_df, \n    y_condition_smote_df_filtered.values, \n    y_severity_smote_df_filtered.values, \n    test_size=0.2, \n    random_state=42\n)\n\n# Confirm sizes\nprint(f\"X_train_df size: {len(X_train_df)}, X_val_df size: {len(X_val_df)}\")\n\n# 2. Define Image Folder and Generators (Restored)\nimage_folder = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/'\n\ntrain_generator = DataGenerator(X_train_df, image_folder, y_train_condition, y_train_severity, batch_size=32)\nval_generator = DataGenerator(X_val_df, image_folder, y_val_condition, y_val_severity, batch_size=32)\n\n# 3. Setup Callbacks (Restored)\nearlystop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)\ncheckpoint = ModelCheckpoint('efficientnetb0_model.keras', save_best_only=True)\n\n# 4. Distributed Training Setup (The new optimization)\ntf.keras.backend.clear_session()\nstrategy = tf.distribute.MirroredStrategy()\nprint(f\"Number of devices: {strategy.num_replicas_in_sync}\")\n\nwith strategy.scope():\n    # Build model inside scope\n    model = build_efficientnet_model(num_classes_condition=5, num_classes_severity=3)\n\nprint(\"Starting Distributed Training for EfficientNet...\")\nmodel.fit(\n    train_generator,\n    validation_data=val_generator,\n    epochs=10,\n    callbacks=[earlystop, checkpoint],\n    verbose=1\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T15:07:59.625955Z","iopub.execute_input":"2026-01-01T15:07:59.626247Z","iopub.status.idle":"2026-01-01T16:08:46.193208Z","shell.execute_reply.started":"2026-01-01T15:07:59.626215Z","shell.execute_reply":"2026-01-01T16:08:46.192608Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Saving the Trained Model to the Output Folder","metadata":{}},{"cell_type":"code","source":"# Save the trained model to the output directory\noutput_directory = '/kaggle/working/'  # Output folder in Kaggle\nmodel_save_path = os.path.join(output_directory, 'efficientnetb0_model.h5')\n\n# Save the model\nmodel.save(model_save_path)\n\nprint(f\"Model saved at {model_save_path}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T16:09:08.727606Z","iopub.execute_input":"2026-01-01T16:09:08.728278Z","iopub.status.idle":"2026-01-01T16:09:09.413260Z","shell.execute_reply.started":"2026-01-01T16:09:08.728249Z","shell.execute_reply":"2026-01-01T16:09:09.412614Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Evaluating the Model","metadata":{}},{"cell_type":"code","source":"# Define batch size\nbatch_size = 32  # Set your desired batch size here\n\n# Evaluate the model on validation data\nval_generator = DataGenerator(X_val_df, image_folder, y_val_condition, y_val_severity, batch_size=batch_size)\n\n# Perform evaluation, expecting 3 values: overall loss, condition accuracy, severity accuracy\nresults = model.evaluate(val_generator)\nprint(results)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T16:09:13.120212Z","iopub.execute_input":"2026-01-01T16:09:13.120735Z","iopub.status.idle":"2026-01-01T16:10:54.557222Z","shell.execute_reply.started":"2026-01-01T16:09:13.120706Z","shell.execute_reply":"2026-01-01T16:10:54.556619Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ROC Curve for Condition and Severity Classification","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import roc_curve, auc\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import label_binarize\n\n# Function to plot ROC curve\ndef plot_roc_curve(y_true, y_pred, n_classes, title):\n    # Binarize the true labels for ROC\n    y_true_binarized = label_binarize(y_true, classes=range(n_classes))\n\n    # Compute ROC curve and AUC for each class\n    fpr = {}\n    tpr = {}\n    roc_auc = {}\n\n    for i in range(n_classes):\n        fpr[i], tpr[i], _ = roc_curve(y_true_binarized[:, i], y_pred[:, i])\n        roc_auc[i] = auc(fpr[i], tpr[i])\n\n    # Plot ROC curves\n    plt.figure()\n    for i in range(n_classes):\n        plt.plot(fpr[i], tpr[i], label=f\"Class {i} (AUC = {roc_auc[i]:.2f})\")\n\n    plt.plot([0, 1], [0, 1], 'k--')  # Diagonal line for random guessing\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.05])\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title(title)\n    plt.legend(loc=\"lower right\")\n    plt.show()\n\n# Get the model predictions using the validation data generator\ny_pred_condition, y_pred_severity = model.predict(val_generator)\n\n# Plot ROC curves for condition classification\nplot_roc_curve(y_val_condition.argmax(axis=1), y_pred_condition, 5, \"ROC Curve for Condition Classification\")\n\n# Plot ROC curves for severity classification\nplot_roc_curve(y_val_severity.argmax(axis=1), y_pred_severity, 3, \"ROC Curve for Severity Classification\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T16:11:38.195517Z","iopub.execute_input":"2026-01-01T16:11:38.195829Z","iopub.status.idle":"2026-01-01T16:13:13.156922Z","shell.execute_reply.started":"2026-01-01T16:11:38.195802Z","shell.execute_reply":"2026-01-01T16:13:13.156301Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Confusion Matrix and Classification Report","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, classification_report, accuracy_score\n\n# Convert predictions to class labels\ny_pred_condition_classes = y_pred_condition.argmax(axis=1)\ny_pred_severity_classes = y_pred_severity.argmax(axis=1)\n\ny_val_condition_classes = y_val_condition.argmax(axis=1)\ny_val_severity_classes = y_val_severity.argmax(axis=1)\n\n# Confusion Matrix for Condition Classification\ncondition_cm = confusion_matrix(y_val_condition_classes, y_pred_condition_classes)\nprint(\"Confusion Matrix for Condition Classification:\")\nprint(condition_cm)\n\n# Confusion Matrix for Severity Classification\nseverity_cm = confusion_matrix(y_val_severity_classes, y_pred_severity_classes)\nprint(\"Confusion Matrix for Severity Classification:\")\nprint(severity_cm)\n\n# Classification Report for Condition Classification\ncondition_report = classification_report(y_val_condition_classes, y_pred_condition_classes)\nprint(\"Classification Report for Condition Classification:\")\nprint(condition_report)\n\n# Classification Report for Severity Classification\nseverity_report = classification_report(y_val_severity_classes, y_pred_severity_classes)\nprint(\"Classification Report for Severity Classification:\")\nprint(severity_report)\n\n# Overall Accuracy for both tasks\ncondition_accuracy = accuracy_score(y_val_condition_classes, y_pred_condition_classes)\nseverity_accuracy = accuracy_score(y_val_severity_classes, y_pred_severity_classes)\n\nprint(f\"Condition Classification Accuracy: {condition_accuracy:.2f}\")\nprint(f\"Severity Classification Accuracy: {severity_accuracy:.2f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T16:13:22.399802Z","iopub.execute_input":"2026-01-01T16:13:22.400298Z","iopub.status.idle":"2026-01-01T16:13:22.425972Z","shell.execute_reply.started":"2026-01-01T16:13:22.400267Z","shell.execute_reply":"2026-01-01T16:13:22.425443Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Testing","metadata":{}},{"cell_type":"code","source":"# Modified Data Generator class for test data using study_id and series_id from test_desc\nclass TestDataGenerator(Sequence):\n    def __init__(self, df, image_folder, batch_size=32, img_size=(224, 224)):\n        self.df = df  # Use the test_desc DataFrame\n        self.image_folder = image_folder\n        self.batch_size = batch_size\n        self.img_size = img_size\n        self.indices = np.arange(len(df))\n    \n    def __len__(self):\n        return int(np.ceil(len(self.df) / self.batch_size))\n    \n    def __getitem__(self, index):\n        # Generate batch indices\n        batch_indices = self.indices[index * self.batch_size:(index + 1) * self.batch_size]\n        \n        # Get the batch data\n        batch_df = self.df.iloc[batch_indices]\n        \n        # Load the images for the batch\n        images = []\n        for _, row in batch_df.iterrows():\n            study_id = int(row['study_id'])  # Ensure integer format\n            series_id = int(row['series_id'])  # Ensure integer format\n            \n            # Construct image path using integer values\n            series_path = os.path.join(self.image_folder, str(study_id), str(series_id))\n            dicom_files = [f for f in os.listdir(series_path) if f.endswith('.dcm')]\n            \n            # Load the first image in the series (adjust if needed)\n            if dicom_files:\n                img_path = os.path.join(series_path, dicom_files[0])\n                img = self.load_dicom_image(img_path)\n                images.append(img)\n            else:\n                print(f\"Warning: No DICOM files found for series {series_id} in study {study_id}\")\n                images.append(np.zeros((*self.img_size, 3)))  # Blank image if no DICOM\n            \n        return np.array(images)\n    \n    def load_dicom_image(self, image_path):\n        try:\n            dicom = pydicom.dcmread(image_path)\n            if hasattr(dicom, 'pixel_array'):\n                img = dicom.pixel_array\n                img = cv2.resize(img, self.img_size)  # Resize to 224x224\n                img = np.stack((img,) * 3, axis=-1)  # Convert to RGB (3 channels)\n                img = img / 255.0  # Normalize\n                return img\n            else:\n                print(f\"Warning: No pixel data in DICOM file: {image_path}\")\n                return np.zeros((*self.img_size, 3))  # Return a blank image if no pixel data\n        except Exception as e:\n            print(f\"Error reading DICOM file: {image_path}. Error: {e}\")\n            return np.zeros((*self.img_size, 3))  # Return a blank image if any error occurs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T16:13:59.877942Z","iopub.execute_input":"2026-01-01T16:13:59.878229Z","iopub.status.idle":"2026-01-01T16:13:59.887764Z","shell.execute_reply.started":"2026-01-01T16:13:59.878205Z","shell.execute_reply":"2026-01-01T16:13:59.887129Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the test image folder\ntest_image_folder = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_images/'\n\n# Create the test data generator using the test_desc DataFrame\ntest_generator = TestDataGenerator(test_desc, test_image_folder, batch_size=32)\n\n# Get model predictions for the test set\ny_pred_condition, y_pred_severity = model.predict(test_generator)\n\n# Since there are no true labels, just output predictions\nprint(f\"Predicted Conditions: {y_pred_condition}\")\nprint(f\"Predicted Severity: {y_pred_severity}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T16:14:09.330226Z","iopub.execute_input":"2026-01-01T16:14:09.330524Z","iopub.status.idle":"2026-01-01T16:14:11.649406Z","shell.execute_reply.started":"2026-01-01T16:14:09.330491Z","shell.execute_reply":"2026-01-01T16:14:11.648730Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Breakdown of Predicted Conditions and Severities:\n\nThe model predicts both a condition class and a severity level for each sample. Here, we interpret the predicted probabilities:","metadata":{}},{"cell_type":"markdown","source":"**Predicted Conditions**:\n\nEach row corresponds to the model's prediction for a test sample, with probabilities assigned to each of the five condition classes (`condition_0` to `condition_4`), which represent specific lumbar spine conditions. The class with the highest probability is considered the model's prediction.\n\nSample 1:\n`[45.9%, 0.16%, 8.15%, 0.11%, 45.7%]` \nThe model is almost equally confident between **Spinal Canal Stenosis** (45\\.9%) and **Right Subarticular Stenosis** (45\\.7%).\n\nSample 2:\n`[0.009%, 67.08%, 0.008%, 32.9%, 0.005%]`\nThe model strongly predicts **Left Neural Foraminal Narrowing** (67.08%), but also assigns a considerable probability to **Left Subarticular Stenosis** (32.9%). Despite the presence of the second-highest score, Left Neural Foraminal Narrowing is clearly favored.\n\nSample 3:\n`[0.0000008%, 0.00000029%, 0.00000031%, 0.000000086%, 100%]`\nThe model is **100%** confident in **Right Subarticular Stenosis**. This is an extremely confident prediction, with no uncertainty about the correct condition class.","metadata":{}},{"cell_type":"markdown","source":"**Predicted Severities**:\n\nEach row also corresponds to the model's prediction of severity for the given condition, with probabilities assigned to three severity levels (severity_0 to severity_2). The class with the highest probability is considered the predicted severity level.\n\nSample 1: \n`[19.68%, 43.98%, 36.35%]`\nThe model is moderately confident in **Normal/Mild** (44%), but it also assigns a significant probability to **Moderate** (36.35%). This indicates some uncertainty in the severity classification.\n\n\nSample 2:\n\n`[4.6%, 94.2%, 1.2%]`\nThe model is highly confident that this sample corresponds to **Normal/Mild** (94.2%), with almost no chance of it being any other severity.\n\nSample 3:\n\n`[0.92%, 3.02%, 96.05%]`\nThe model is highly confident in predicting **Moderate** (~96%), with minimal doubt about the severity level.","metadata":{}},{"cell_type":"code","source":"ls","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T07:32:50.723756Z","iopub.execute_input":"2025-12-31T07:32:50.723974Z","iopub.status.idle":"2025-12-31T07:32:50.848999Z","shell.execute_reply.started":"2025-12-31T07:32:50.723941Z","shell.execute_reply":"2025-12-31T07:32:50.848327Z"}},"outputs":[],"execution_count":null}]}