{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":30761,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nfrom glob import glob\nfrom pathlib import Path\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pydicom\nimport warnings\nimport json\nimport cv2\nfrom skimage import measure\nfrom mpl_toolkits.mplot3d.art3d import Poly3DCollection\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport random\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport tensorflow as tf\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPooling2D, Input\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.utils import to_categorical\n\n\ncompetition_dataset_directory = Path('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-08-28T15:06:28.458022Z","iopub.execute_input":"2024-08-28T15:06:28.458715Z","iopub.status.idle":"2024-08-28T15:06:44.898775Z","shell.execute_reply.started":"2024-08-28T15:06:28.458647Z","shell.execute_reply":"2024-08-28T15:06:44.897666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load datasets\ndf_train_series_descriptions = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_series_descriptions.csv')\ndf_test_series_descriptions = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_series_descriptions.csv')\ndf_sample_submission = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/sample_submission.csv')\ndf_train = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv')\ndf_train_label_coordinates = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_label_coordinates.csv')\n\n# Display dataframes\nprint(\"Train Series Descriptions:\")\nprint(df_train_series_descriptions.head()) \n\nprint(\"\\nTest Series Descriptions:\")\nprint(df_test_series_descriptions.head())\n\nprint(\"\\nSample Submission:\")\nprint(df_sample_submission.head())\n\nprint(\"\\nTraining Data:\")\nprint(df_train.head())\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:44.900650Z","iopub.execute_input":"2024-08-28T15:06:44.901329Z","iopub.status.idle":"2024-08-28T15:06:45.080717Z","shell.execute_reply.started":"2024-08-28T15:06:44.901289Z","shell.execute_reply":"2024-08-28T15:06:45.079538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display dataframes as tables\nprint(\"Train Series Descriptions:\")\ndisplay(df_train_series_descriptions)\n\nprint(\"\\nTest Series Descriptions:\")\ndisplay(df_test_series_descriptions)\n\nprint(\"\\nSample Submission:\")\ndisplay(df_sample_submission)\n\nprint(\"\\nTraining Data:\")\ndisplay(df_train)\n\nprint(\"\\nTraining label coordinates:\")\ndisplay(df_train_label_coordinates)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:45.082164Z","iopub.execute_input":"2024-08-28T15:06:45.082541Z","iopub.status.idle":"2024-08-28T15:06:45.153572Z","shell.execute_reply.started":"2024-08-28T15:06:45.082504Z","shell.execute_reply":"2024-08-28T15:06:45.152538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_condition_counts(df, title, path=None):\n    \n    \"\"\"\n    Visualize condition counts on training set\n    \n    Parameters\n    ----------\n    df: pandas.DataFrame\n        Counts and percentages of conditions\n        \n    title: str\n        Title of the plot\n        \n    path: str, pathlib.Path or None\n        Path of the output file (if path is None, plot is displayed with selected backend)\n    \"\"\"\n    \n    fig, ax = plt.subplots(figsize=(24, 16))\n\n    ax.barh(\n        y=np.arange(df.shape[0] // 3) - 0.2,\n        width=df['count'].values[0::3],\n        height=0.2,\n        align='center',\n        label='Normal/Mild'\n    )\n    ax.barh(\n        y=np.arange(df.shape[0] // 3),\n        width=df['count'].values[1::3],\n        height=0.2,\n        align='center',\n        label='Moderate'\n    )\n    ax.barh(\n        y=np.arange(df.shape[0] // 3) + 0.2,\n        width=df['count'].values[2::3],\n        height=0.2,\n        align='center',\n        label='Severe'\n    )\n\n    ax.set_yticks(np.arange(df.shape[0] // 3))\n    ax.set_yticklabels([\n        f'{level}\\nNormal Count: {normal_count} ({normal_percentage:.2f}%)\\nModerate Count: {moderate_count} ({moderate_percentage:.2f}%)\\nSevere Count: {severe_count} ({severe_percentage:.2f}%)' for level, normal_count, normal_percentage, moderate_count, moderate_percentage, severe_count, severe_percentage, in zip(\n            df['level'].values[0::3],\n            df['count'].values[0::3],\n            df['percentage'].values[0::3],\n            df['count'].values[1::3],\n            df['percentage'].values[1::3],\n            df['count'].values[2::3],\n            df['percentage'].values[2::3],\n        )\n    ])\n    ax.set_xlabel('')\n    ax.tick_params(axis='x', labelsize=17.5, pad=10)\n    ax.tick_params(axis='y', labelsize=17.5, pad=10)\n    ax.set_title(title, size=20, pad=15)\n    ax.legend(loc='best', prop={'size': 18})\n    plt.gca().invert_yaxis()\n\n    plt.show()\n\n    if path is None:\n        plt.show()\n    else:\n        plt.savefig(path)\n        plt.close(fig)","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:45.155978Z","iopub.execute_input":"2024-08-28T15:06:45.156356Z","iopub.status.idle":"2024-08-28T15:06:45.169483Z","shell.execute_reply.started":"2024-08-28T15:06:45.156317Z","shell.execute_reply":"2024-08-28T15:06:45.168238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spinal_canal_stenosis_columns = [column for column in df_train.columns if column.startswith('spinal_canal_stenosis')]\ndf_train_spinal_canal_stenosis = []\n\nfor column in spinal_canal_stenosis_columns:\n    df = df_train[[column]].copy(deep=True)\n    df['level'] = '_'.join(column.split('_')[-2:])\n    df = df.rename(columns={column: 'spinal_canal_stenosis'})\n    df_train_spinal_canal_stenosis.append(df)\n    \ndf_train_spinal_canal_stenosis = pd.concat(df_train_spinal_canal_stenosis, axis=0).reset_index(drop=True)\n\ndf_train_spinal_canal_stenosis_counts = df_train_spinal_canal_stenosis.groupby('level').value_counts().reset_index()\ndf_train_spinal_canal_stenosis_counts['severity'] = df_train_spinal_canal_stenosis_counts['spinal_canal_stenosis'].map({\n    'Normal/Mild': 0,\n    'Moderate': 1,\n    'Severe': 2\n})\ndf_train_spinal_canal_stenosis_counts = df_train_spinal_canal_stenosis_counts.sort_values(by=['level', 'severity'], ascending=True)\ndf_train_spinal_canal_stenosis_counts['percentage'] = df_train_spinal_canal_stenosis_counts['count'] / df_train_spinal_canal_stenosis_counts.groupby('level')['count'].transform('sum') * 100","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:45.170940Z","iopub.execute_input":"2024-08-28T15:06:45.171409Z","iopub.status.idle":"2024-08-28T15:06:45.218783Z","shell.execute_reply.started":"2024-08-28T15:06:45.171344Z","shell.execute_reply":"2024-08-28T15:06:45.217655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_condition_counts(\n    df=df_train_spinal_canal_stenosis_counts,\n    title='Spinal Canal Stenosis Counts On Levels'\n)","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:45.220442Z","iopub.execute_input":"2024-08-28T15:06:45.220782Z","iopub.status.idle":"2024-08-28T15:06:45.772640Z","shell.execute_reply.started":"2024-08-28T15:06:45.220750Z","shell.execute_reply":"2024-08-28T15:06:45.771459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"neural_foraminal_narrowing_columns = [column for column in df_train.columns if 'neural_foraminal_narrowing' in column]\ndf_train_neural_foraminal_narrowing = []\n\nfor column in neural_foraminal_narrowing_columns:\n    df = df_train[[column]].copy(deep=True)\n    df['level'] = '_'.join(column.split('_')[-2:])\n    df = df.rename(columns={column: 'neural_foraminal_narrowing'})\n    df_train_neural_foraminal_narrowing.append(df)\n    \ndf_train_neural_foraminal_narrowing = pd.concat(df_train_neural_foraminal_narrowing, axis=0).reset_index(drop=True)\n\ndf_train_neural_foraminal_narrowing_counts = df_train_neural_foraminal_narrowing.groupby('level').value_counts().reset_index()\ndf_train_neural_foraminal_narrowing_counts['severity'] = df_train_neural_foraminal_narrowing_counts['neural_foraminal_narrowing'].map({\n    'Normal/Mild': 0,\n    'Moderate': 1,\n    'Severe': 2\n})\ndf_train_neural_foraminal_narrowing_counts = df_train_neural_foraminal_narrowing_counts.sort_values(by=['level', 'severity'], ascending=True)\ndf_train_neural_foraminal_narrowing_counts['percentage'] = df_train_neural_foraminal_narrowing_counts['count'] / df_train_neural_foraminal_narrowing_counts.groupby('level')['count'].transform('sum') * 100\n\nleft_neural_foraminal_narrowing_columns = [column for column in df_train.columns if column.startswith('left_neural_foraminal_narrowing')]\ndf_train_left_neural_foraminal_narrowing = []\n\nfor column in left_neural_foraminal_narrowing_columns:\n    df = df_train[[column]].copy(deep=True)\n    df['level'] = '_'.join(column.split('_')[-2:])\n    df = df.rename(columns={column: 'left_neural_foraminal_narrowing'})\n    df_train_left_neural_foraminal_narrowing.append(df)\n    \ndf_train_left_neural_foraminal_narrowing = pd.concat(df_train_left_neural_foraminal_narrowing, axis=0).reset_index(drop=True)\n\ndf_train_left_neural_foraminal_narrowing_counts = df_train_left_neural_foraminal_narrowing.groupby('level').value_counts().reset_index()\ndf_train_left_neural_foraminal_narrowing_counts['severity'] = df_train_left_neural_foraminal_narrowing_counts['left_neural_foraminal_narrowing'].map({\n    'Normal/Mild': 0,\n    'Moderate': 1,\n    'Severe': 2\n})\ndf_train_left_neural_foraminal_narrowing_counts = df_train_left_neural_foraminal_narrowing_counts.sort_values(by=['level', 'severity'], ascending=True)\ndf_train_left_neural_foraminal_narrowing_counts['percentage'] = df_train_left_neural_foraminal_narrowing_counts['count'] / df_train_left_neural_foraminal_narrowing_counts.groupby('level')['count'].transform('sum') * 100\n\nright_neural_foraminal_narrowing_columns = [column for column in df_train.columns if column.startswith('right_neural_foraminal_narrowing')]\ndf_train_right_neural_foraminal_narrowing = []\n\nfor column in right_neural_foraminal_narrowing_columns:\n    df = df_train[[column]].copy(deep=True)\n    df['level'] = '_'.join(column.split('_')[-2:])\n    df = df.rename(columns={column: 'right_neural_foraminal_narrowing'})\n    df_train_right_neural_foraminal_narrowing.append(df)\n    \ndf_train_right_neural_foraminal_narrowing = pd.concat(df_train_right_neural_foraminal_narrowing, axis=0).reset_index(drop=True)\n\ndf_train_right_neural_foraminal_narrowing_counts = df_train_right_neural_foraminal_narrowing.groupby('level').value_counts().reset_index()\ndf_train_right_neural_foraminal_narrowing_counts['severity'] = df_train_right_neural_foraminal_narrowing_counts['right_neural_foraminal_narrowing'].map({\n    'Normal/Mild': 0,\n    'Moderate': 1,\n    'Severe': 2\n})\ndf_train_right_neural_foraminal_narrowing_counts = df_train_right_neural_foraminal_narrowing_counts.sort_values(by=['level', 'severity'], ascending=True)\ndf_train_right_neural_foraminal_narrowing_counts['percentage'] = df_train_right_neural_foraminal_narrowing_counts['count'] / df_train_right_neural_foraminal_narrowing_counts.groupby('level')['count'].transform('sum') * 100","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:45.774654Z","iopub.execute_input":"2024-08-28T15:06:45.775109Z","iopub.status.idle":"2024-08-28T15:06:45.868330Z","shell.execute_reply.started":"2024-08-28T15:06:45.775058Z","shell.execute_reply":"2024-08-28T15:06:45.867399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_condition_counts(\n    df=df_train_neural_foraminal_narrowing_counts,\n    title='Neural Foraminal Narrowing Counts On Levels'\n)\n\nvisualize_condition_counts(\n    df=df_train_left_neural_foraminal_narrowing_counts,\n    title='Left Neural Foraminal Narrowing Counts On Levels'\n)\n\nvisualize_condition_counts(\n    df=df_train_right_neural_foraminal_narrowing_counts,\n    title='Right Neural Foraminal Narrowing Counts On Levels'\n)","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:45.869540Z","iopub.execute_input":"2024-08-28T15:06:45.869874Z","iopub.status.idle":"2024-08-28T15:06:47.772427Z","shell.execute_reply.started":"2024-08-28T15:06:45.869839Z","shell.execute_reply":"2024-08-28T15:06:47.769790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Extract columns related to neural foraminal narrowing\nneural_foraminal_narrowing_columns = [col for col in df_train.columns if 'neural_foraminal_narrowing' in col]\n\n# Separate left and right columns\nleft_columns = neural_foraminal_narrowing_columns[:5]\nright_columns = neural_foraminal_narrowing_columns[5:]\n\n# Create DataFrame for left and right neural foraminal narrowing\ndf_train_left = pd.melt(\n    df_train[left_columns],\n    var_name='level',\n    value_name='left_neural_foraminal_narrowing'\n)\ndf_train_left['level'] = df_train_left['level'].apply(lambda x: '_'.join(x.split('_')[-2:]))\n\ndf_train_right = pd.melt(\n    df_train[right_columns],\n    var_name='level',\n    value_name='right_neural_foraminal_narrowing'\n)\ndf_train_right['level'] = df_train_right['level'].apply(lambda x: '_'.join(x.split('_')[-2:]))\n\n# Concatenate left and right data\ndf_train_left_right_neural_foraminal_narrowing = pd.concat([df_train_left['left_neural_foraminal_narrowing'], \n                                                            df_train_right[['right_neural_foraminal_narrowing', 'level']]], axis=1)\n\n# Map severity levels\nseverity_map = {'Normal/Mild': 0, 'Moderate': 1, 'Severe': 2}\ndf_train_left_right_neural_foraminal_narrowing['left_severity'] = df_train_left_right_neural_foraminal_narrowing['left_neural_foraminal_narrowing'].map(severity_map)\ndf_train_left_right_neural_foraminal_narrowing['right_severity'] = df_train_left_right_neural_foraminal_narrowing['right_neural_foraminal_narrowing'].map(severity_map)\n\n# Map level to numerical values\nlevel_map = {'l1_l2': 0, 'l2_l3': 1, 'l3_l4': 2, 'l4_l5': 3, 'l5_s1': 4}\ndf_train_left_right_neural_foraminal_narrowing['level_'] = df_train_left_right_neural_foraminal_narrowing['level'].map(level_map)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:47.774322Z","iopub.execute_input":"2024-08-28T15:06:47.774778Z","iopub.status.idle":"2024-08-28T15:06:47.828728Z","shell.execute_reply.started":"2024-08-28T15:06:47.774730Z","shell.execute_reply":"2024-08-28T15:06:47.827356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate and display correlations\ndf_neural_foraming_narrowing_correlations = df_train_left_right_neural_foraminal_narrowing.loc[:, ['level_', 'left_severity', 'right_severity']].corr()\ndf_neural_foraming_narrowing_correlations.style.background_gradient(cmap='coolwarm')\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:47.833043Z","iopub.execute_input":"2024-08-28T15:06:47.833570Z","iopub.status.idle":"2024-08-28T15:06:47.929967Z","shell.execute_reply.started":"2024-08-28T15:06:47.833516Z","shell.execute_reply":"2024-08-28T15:06:47.928860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_subarticular_stenosis(side=None):\n    if side:\n        columns = [col for col in df_train.columns if col.startswith(f'{side}_subarticular_stenosis')]\n        col_name = f'{side}_subarticular_stenosis'\n    else:\n        columns = [col for col in df_train.columns if 'subarticular_stenosis' in col]\n        col_name = 'subarticular_stenosis'\n\n    df_subarticular_stenosis = pd.concat(\n        [df_train[[col]].rename(columns={col: col_name}).assign(level='_'.join(col.split('_')[-2:])) for col in columns],\n        axis=0\n    ).reset_index(drop=True)\n\n    df_counts = df_subarticular_stenosis.groupby('level')[col_name].value_counts().reset_index(name='count')\n    df_counts['severity'] = df_counts[col_name].map({'Normal/Mild': 0, 'Moderate': 1, 'Severe': 2})\n    df_counts = df_counts.sort_values(by=['level', 'severity']).reset_index(drop=True)\n    df_counts['percentage'] = df_counts['count'] / df_counts.groupby('level')['count'].transform('sum') * 100\n\n    return df_counts\n\n# Process and store the counts for overall, left, and right subarticular stenosis\ndf_train_subarticular_stenosis_counts = process_subarticular_stenosis()\ndf_train_left_subarticular_stenosis_counts = process_subarticular_stenosis('left')\ndf_train_right_subarticular_stenosis_counts = process_subarticular_stenosis('right')\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:47.931396Z","iopub.execute_input":"2024-08-28T15:06:47.932098Z","iopub.status.idle":"2024-08-28T15:06:48.000843Z","shell.execute_reply.started":"2024-08-28T15:06:47.932059Z","shell.execute_reply":"2024-08-28T15:06:47.999583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"titles = ['Subarticular Stenosis Counts On Levels', \n          'Left Subarticular Stenosis Counts On Levels', \n          'Right Subarticular Stenosis Counts On Levels']\n\ndfs = [df_train_subarticular_stenosis_counts, \n       df_train_left_subarticular_stenosis_counts, \n       df_train_right_subarticular_stenosis_counts]\n\nfor df, title in zip(dfs, titles):\n    visualize_condition_counts(df=df, title=title)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:48.002208Z","iopub.execute_input":"2024-08-28T15:06:48.002660Z","iopub.status.idle":"2024-08-28T15:06:49.471564Z","shell.execute_reply.started":"2024-08-28T15:06:48.002618Z","shell.execute_reply":"2024-08-28T15:06:49.470542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subarticular_stenosis_columns = [column for column in df_train.columns if 'subarticular_stenosis' in column]\ndf_train_left_right_subarticular_stenosis = []\n\nfor column in subarticular_stenosis_columns[:5]:\n    df = df_train[[column]].copy(deep=True)\n    df['level'] = '_'.join(column.split('_')[-2:])\n    df = df.rename(columns={column: 'left_subarticular_stenosis'})\n    df_train_left_right_subarticular_stenosis.append(df)\n\nfor column in subarticular_stenosis_columns[5:]:\n    df = df_train[[column]].copy(deep=True)\n    df['level'] = '_'.join(column.split('_')[-2:])\n    df = df.rename(columns={column: 'right_subarticular_stenosis'})\n    df_train_left_right_subarticular_stenosis.append(df)\n    \ndf_train_left_right_subarticular_stenosis = pd.concat((\n    pd.concat(df_train_left_right_subarticular_stenosis[:5], axis=0).reset_index(drop=True).iloc[:, :1],\n    pd.concat(df_train_left_right_subarticular_stenosis[5:], axis=0).reset_index(drop=True)\n), axis=1, ignore_index=False)\n\ndf_train_left_right_subarticular_stenosis['left_severity'] = df_train_left_right_subarticular_stenosis['left_subarticular_stenosis'].map({\n    'Normal/Mild': 0,\n    'Moderate': 1,\n    'Severe': 2\n})\ndf_train_left_right_subarticular_stenosis['right_severity'] = df_train_left_right_subarticular_stenosis['right_subarticular_stenosis'].map({\n    'Normal/Mild': 0,\n    'Moderate': 1,\n    'Severe': 2\n})\ndf_train_left_right_subarticular_stenosis['level_'] = df_train_left_right_subarticular_stenosis['level'].map({\n    'l1_l2': 0,\n    'l2_l3': 1,\n    'l3_l4': 2,\n    'l4_l5': 3,\n    'l5_s1': 4\n})","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:49.472956Z","iopub.execute_input":"2024-08-28T15:06:49.473299Z","iopub.status.idle":"2024-08-28T15:06:49.510517Z","shell.execute_reply.started":"2024-08-28T15:06:49.473265Z","shell.execute_reply":"2024-08-28T15:06:49.509425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate correlations and display with a gradient background\ndf_subarticular_stenosis_correlations = df_train_left_right_subarticular_stenosis[['level_', 'left_severity', 'right_severity']].corr()\ndf_subarticular_stenosis_correlations.style.background_gradient(cmap='coolwarm')\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:49.512175Z","iopub.execute_input":"2024-08-28T15:06:49.512740Z","iopub.status.idle":"2024-08-28T15:06:49.529982Z","shell.execute_reply.started":"2024-08-28T15:06:49.512686Z","shell.execute_reply":"2024-08-28T15:06:49.528951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# structure\ndf_train.info()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:49.531449Z","iopub.execute_input":"2024-08-28T15:06:49.531915Z","iopub.status.idle":"2024-08-28T15:06:49.548121Z","shell.execute_reply.started":"2024-08-28T15:06:49.531829Z","shell.execute_reply":"2024-08-28T15:06:49.546987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Example: Distribution of 'spinal_canal_stenosis_l1_l2'\nplt.figure(figsize=(10, 6))\nsns.countplot(data=df_train, x='spinal_canal_stenosis_l1_l2')\nplt.title('Distribution of Spinal Canal Stenosis L1/L2')\nplt.xlabel('Spinal Canal Stenosis L1/L2')\nplt.ylabel('Count')\nplt.xticks(rotation=45)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:49.549804Z","iopub.execute_input":"2024-08-28T15:06:49.550645Z","iopub.status.idle":"2024-08-28T15:06:49.810192Z","shell.execute_reply.started":"2024-08-28T15:06:49.550595Z","shell.execute_reply":"2024-08-28T15:06:49.809049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preview\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:49.811926Z","iopub.execute_input":"2024-08-28T15:06:49.812442Z","iopub.status.idle":"2024-08-28T15:06:49.836769Z","shell.execute_reply.started":"2024-08-28T15:06:49.812367Z","shell.execute_reply":"2024-08-28T15:06:49.835368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# structure\ndf_train_label_coordinates.info()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:49.838442Z","iopub.execute_input":"2024-08-28T15:06:49.838834Z","iopub.status.idle":"2024-08-28T15:06:49.855895Z","shell.execute_reply.started":"2024-08-28T15:06:49.838789Z","shell.execute_reply":"2024-08-28T15:06:49.854564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preview\ndf_train_label_coordinates.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:49.857333Z","iopub.execute_input":"2024-08-28T15:06:49.857717Z","iopub.status.idle":"2024-08-28T15:06:49.871455Z","shell.execute_reply.started":"2024-08-28T15:06:49.857679Z","shell.execute_reply":"2024-08-28T15:06:49.870271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# look at categories\nfor f in ['instance_number','condition','level']:\n    print(df_train_label_coordinates[f].value_counts())\n    print()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:49.873031Z","iopub.execute_input":"2024-08-28T15:06:49.873943Z","iopub.status.idle":"2024-08-28T15:06:49.891077Z","shell.execute_reply.started":"2024-08-28T15:06:49.873890Z","shell.execute_reply":"2024-08-28T15:06:49.889947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# numerical plot for instance number\nplt.figure(figsize=(7,1))\ndf_train_label_coordinates.instance_number.plot(kind='box', vert=False)\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:49.892725Z","iopub.execute_input":"2024-08-28T15:06:49.893228Z","iopub.status.idle":"2024-08-28T15:06:50.080094Z","shell.execute_reply.started":"2024-08-28T15:06:49.893186Z","shell.execute_reply":"2024-08-28T15:06:50.078842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# look at few rows with the very high instance numbers\ndf_train_label_coordinates[df_train_label_coordinates.instance_number>150]","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:50.081702Z","iopub.execute_input":"2024-08-28T15:06:50.082090Z","iopub.status.idle":"2024-08-28T15:06:50.098011Z","shell.execute_reply.started":"2024-08-28T15:06:50.082047Z","shell.execute_reply":"2024-08-28T15:06:50.096753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# condition vs level\npd.crosstab(df_train_label_coordinates.condition, df_train_label_coordinates.level)","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:50.099884Z","iopub.execute_input":"2024-08-28T15:06:50.100737Z","iopub.status.idle":"2024-08-28T15:06:50.137433Z","shell.execute_reply.started":"2024-08-28T15:06:50.100686Z","shell.execute_reply":"2024-08-28T15:06:50.136191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# structure\ndf_train_series_descriptions.info()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:50.138954Z","iopub.execute_input":"2024-08-28T15:06:50.139445Z","iopub.status.idle":"2024-08-28T15:06:50.151410Z","shell.execute_reply.started":"2024-08-28T15:06:50.139378Z","shell.execute_reply":"2024-08-28T15:06:50.150094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preview\ndf_train_series_descriptions.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:50.152782Z","iopub.execute_input":"2024-08-28T15:06:50.153136Z","iopub.status.idle":"2024-08-28T15:06:50.165538Z","shell.execute_reply.started":"2024-08-28T15:06:50.153091Z","shell.execute_reply":"2024-08-28T15:06:50.164450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# look at categories\ncounts = df_train_series_descriptions.series_description.value_counts()\nprint(counts)\nplt.figure(figsize=(7,3))\ncounts.plot(kind='bar', color='blue')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:50.167065Z","iopub.execute_input":"2024-08-28T15:06:50.167579Z","iopub.status.idle":"2024-08-28T15:06:50.325197Z","shell.execute_reply.started":"2024-08-28T15:06:50.167530Z","shell.execute_reply":"2024-08-28T15:06:50.324070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Perform a left join on 'study_id' between df_train_label and df_train_main\ndf_train_step_1 = df_train_label_coordinates.merge(df_train, how='left', on='study_id').reset_index(drop=True)\ndf_train_step_1.head()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:50.326488Z","iopub.execute_input":"2024-08-28T15:06:50.327298Z","iopub.status.idle":"2024-08-28T15:06:50.439526Z","shell.execute_reply.started":"2024-08-28T15:06:50.327251Z","shell.execute_reply":"2024-08-28T15:06:50.438274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# join with third table\ndf_train = pd.merge(left=df_train_step_1, right=df_train_series_descriptions, how='left', on=['study_id', 'series_id']).reset_index(drop=True)\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:50.448324Z","iopub.execute_input":"2024-08-28T15:06:50.448748Z","iopub.status.idle":"2024-08-28T15:06:50.551950Z","shell.execute_reply.started":"2024-08-28T15:06:50.448710Z","shell.execute_reply":"2024-08-28T15:06:50.550784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert ids to categorical\ndf_train.study_id = df_train.study_id.astype('category')\ndf_train.series_id = df_train.series_id.astype('category')","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:50.553688Z","iopub.execute_input":"2024-08-28T15:06:50.554186Z","iopub.status.idle":"2024-08-28T15:06:50.565771Z","shell.execute_reply.started":"2024-08-28T15:06:50.554135Z","shell.execute_reply":"2024-08-28T15:06:50.564377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# combined table - basic stats\ndf_train.describe(include='all').T","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:50.567317Z","iopub.execute_input":"2024-08-28T15:06:50.567723Z","iopub.status.idle":"2024-08-28T15:06:50.779289Z","shell.execute_reply.started":"2024-08-28T15:06:50.567678Z","shell.execute_reply":"2024-08-28T15:06:50.778146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **EDA**","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Example: Distribution of 'spinal_canal_stenosis_l1_l2'\nplt.figure(figsize=(10, 6))\nsns.countplot(data=df_train, x='spinal_canal_stenosis_l1_l2')\nplt.title('Distribution of Spinal Canal Stenosis L1/L2')\nplt.xlabel('Spinal Canal Stenosis L1/L2')\nplt.ylabel('Count')\nplt.xticks(rotation=45)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:50.780605Z","iopub.execute_input":"2024-08-28T15:06:50.780916Z","iopub.status.idle":"2024-08-28T15:06:50.992221Z","shell.execute_reply.started":"2024-08-28T15:06:50.780883Z","shell.execute_reply":"2024-08-28T15:06:50.991124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nsns.countplot(data=df_train, x='left_neural_foraminal_narrowing_l1_l2', hue='spinal_canal_stenosis_l1_l2')\nplt.title('Comparison of Left Neural Foraminal Narrowing L1/L2 with Spinal Canal Stenosis L1/L2')\nplt.xlabel('Left Neural Foraminal Narrowing L1/L2')\nplt.ylabel('Count')\nplt.xticks(rotation=45)\nplt.legend(title='Spinal Canal Stenosis L1/L2')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:50.993828Z","iopub.execute_input":"2024-08-28T15:06:50.994292Z","iopub.status.idle":"2024-08-28T15:06:51.295596Z","shell.execute_reply.started":"2024-08-28T15:06:50.994241Z","shell.execute_reply":"2024-08-28T15:06:51.294430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nsns.countplot(data=df_train, x='right_subarticular_stenosis_l1_l2', hue='spinal_canal_stenosis_l1_l2')\nplt.title('Comparison of Right Subarticular Stenosis L1/L2 with Spinal Canal Stenosis L1/L2')\nplt.xlabel('Right Subarticular Stenosis L1/L2')\nplt.ylabel('Count')\nplt.xticks(rotation=45)\nplt.legend(title='Spinal Canal Stenosis L1/L2')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:51.296946Z","iopub.execute_input":"2024-08-28T15:06:51.297326Z","iopub.status.idle":"2024-08-28T15:06:51.625845Z","shell.execute_reply.started":"2024-08-28T15:06:51.297288Z","shell.execute_reply":"2024-08-28T15:06:51.624676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# combined table - basic stats\ndf_train.describe(include='all').T","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:51.627511Z","iopub.execute_input":"2024-08-28T15:06:51.628459Z","iopub.status.idle":"2024-08-28T15:06:51.831536Z","shell.execute_reply.started":"2024-08-28T15:06:51.628408Z","shell.execute_reply":"2024-08-28T15:06:51.830406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Reason for Suppressing FutureWarning\n\nIn the code below, we use the `warnings` module to suppress a specific `FutureWarning` related to an internal Pandas option (`use_inf_as_na`). This warning indicates that the option is deprecated and will be removed in a future version of Pandas.\n\n**Reason for Suppressing:**\n\nThis warning does not impact the functionality of our visualization and is primarily a notice for future changes in the Pandas library. To keep our output clean and focused on the key results, we suppress this warning. Additionally, suppressing this warning prevents it from cluttering our output and distracting from the main analysis.\n\n```python\nimport warnings\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nwarnings.filterwarnings(\"ignore\", category=FutureWarning, module=\"seaborn\")\n\nsns.jointplot(data=df_train, x='x', y='y', \n              color='darkblue', alpha=0.25)\nplt.grid()\nplt.show()\n\n","metadata":{}},{"cell_type":"code","source":"# Suppress the specific warning\nwarnings.filterwarnings(\"ignore\", category=FutureWarning, module=\"seaborn\")\n\n# Plot coordinates\nsns.jointplot(data=df_train, x='x', y='y', \n              color='darkblue', alpha=0.25)\nplt.grid()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:51.833009Z","iopub.execute_input":"2024-08-28T15:06:51.833479Z","iopub.status.idle":"2024-08-28T15:06:52.978970Z","shell.execute_reply.started":"2024-08-28T15:06:51.833428Z","shell.execute_reply":"2024-08-28T15:06:52.977909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Suppress the specific warning\nwarnings.filterwarnings(\"ignore\", category=FutureWarning, module=\"seaborn\")\n\n# plot coordinates - colored by condition\nax = sns.jointplot(data=df_train, x='x', y='y', \n                   hue='condition', alpha=0.25)\nplt.legend(bbox_to_anchor=(1.2,1), loc=2)\nplt.grid()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:52.980583Z","iopub.execute_input":"2024-08-28T15:06:52.980982Z","iopub.status.idle":"2024-08-28T15:06:55.751149Z","shell.execute_reply.started":"2024-08-28T15:06:52.980934Z","shell.execute_reply":"2024-08-28T15:06:55.749718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Suppress the specific warning\nwarnings.filterwarnings(\"ignore\", category=FutureWarning, module=\"seaborn\")\n\n# plot coordinates - colored by level\nsns.jointplot(data=df_train, x='x', y='y', \n              hue='level', alpha=0.25)\nplt.legend(bbox_to_anchor=(1.2,1), loc=2)\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:55.752699Z","iopub.execute_input":"2024-08-28T15:06:55.753064Z","iopub.status.idle":"2024-08-28T15:06:58.511589Z","shell.execute_reply.started":"2024-08-28T15:06:55.753020Z","shell.execute_reply":"2024-08-28T15:06:58.510217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = df_train.columns.drop('study_id').tolist()\nprint(labels)","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:58.513513Z","iopub.execute_input":"2024-08-28T15:06:58.513977Z","iopub.status.idle":"2024-08-28T15:06:58.520797Z","shell.execute_reply.started":"2024-08-28T15:06:58.513916Z","shell.execute_reply":"2024-08-28T15:06:58.519435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check for duplicate rows in each dataframe\nprint(\"Duplicate rows in Train Series Descriptions:\", df_train_series_descriptions.duplicated().sum())\nprint(\"Duplicate rows in Test Series Descriptions:\", df_test_series_descriptions.duplicated().sum())\nprint(\"Duplicate rows in Sample Submission:\", df_sample_submission.duplicated().sum())\nprint(\"Duplicate rows in Training Data:\", df_train.duplicated().sum())\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:58.522416Z","iopub.execute_input":"2024-08-28T15:06:58.522788Z","iopub.status.idle":"2024-08-28T15:06:58.657115Z","shell.execute_reply.started":"2024-08-28T15:06:58.522742Z","shell.execute_reply":"2024-08-28T15:06:58.655902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n# Check for missing values\nprint(\"Missing values in Train Series Descriptions:\")\nprint(df_train_series_descriptions.isnull().sum())\n\nprint(\"\\nMissing values in Test Series Descriptions:\")\nprint(df_test_series_descriptions.isnull().sum())\n\nprint(\"\\nMissing values in Sample Submission:\")\nprint(df_sample_submission.isnull().sum())\n\nprint(\"\\nMissing values in Training Data:\")\nprint(df_train.isnull().sum())\n\nprint(\"\\nMissing values in Training Label Coordinates:\")\nprint(df_train_label_coordinates.isnull().sum())\n\n# Check data types\nprint(\"\\nData Types in Train Series Descriptions:\")\nprint(df_train_series_descriptions.dtypes)\n\nprint(\"\\nData Types in Test Series Descriptions:\")\nprint(df_test_series_descriptions.dtypes)\n\nprint(\"\\nData Types in Sample Submission:\")\nprint(df_sample_submission.dtypes)\n\nprint(\"\\nData Types in Training Data:\")\nprint(df_train.dtypes)\n\nprint(\"\\nData Types in Training Label Coordinates:\")\nprint(df_train_label_coordinates.dtypes)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:58.658590Z","iopub.execute_input":"2024-08-28T15:06:58.658947Z","iopub.status.idle":"2024-08-28T15:06:58.747113Z","shell.execute_reply.started":"2024-08-28T15:06:58.658911Z","shell.execute_reply":"2024-08-28T15:06:58.746067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Missing Values**\n\n1. No missing values in the \"Train Series Descriptions,\" \"Test Series Descriptions,\" \"Sample Submission,\" or \"Training Label Coordinates\" datasets.\n\n1. The \"Training Data\" dataset has several missing values across different columns. Notably:\n\n*     The right_neural_foraminal_narrowing_l1_l2 to right_neural_foraminal_narrowing_l5_s1 columns have 8 missing values each.\n*     left_subarticular_stenosis_l1_l2 and right_subarticular_stenosis_l1_l2 have a significant number of missing values (164 and 161, respectively).","metadata":{}},{"cell_type":"markdown","source":"****Code to Handle Missing Values****","metadata":{}},{"cell_type":"code","source":"# percentage of missing values\n(df_train.isnull().sum()/(len(df_train)))*100","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:58.748602Z","iopub.execute_input":"2024-08-28T15:06:58.748957Z","iopub.status.idle":"2024-08-28T15:06:58.824676Z","shell.execute_reply.started":"2024-08-28T15:06:58.748922Z","shell.execute_reply":"2024-08-28T15:06:58.823366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define a threshold for columns and rows (e.g., 5%)\ncolumn_threshold = 5  # Percentage\nrow_threshold = 5     # Percentage\n\n# Calculate percentage of missing values for columns\nmissing_percent_columns = df_train.isnull().mean() * 100\n\n# Drop columns with missing values above the threshold\ncols_to_drop = missing_percent_columns[missing_percent_columns > column_threshold].index\ndf_train_reduced = df_train.drop(columns=cols_to_drop)\n\n# Calculate percentage of missing values for rows\nmissing_percent_rows = df_train_reduced.isnull().mean(axis=1) * 100\n\n# Drop rows with missing values above the threshold\nrows_to_drop = missing_percent_rows[missing_percent_rows > row_threshold].index\ndf_train_reduced = df_train_reduced.drop(index=rows_to_drop)\n\ndisplay(df_train_reduced)\n# Output the number of rows and columns in the reduced dataset\nprint(f\"Reduced dataset shape: {df_train_reduced.shape}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:58.826129Z","iopub.execute_input":"2024-08-28T15:06:58.826548Z","iopub.status.idle":"2024-08-28T15:06:59.051515Z","shell.execute_reply.started":"2024-08-28T15:06:58.826502Z","shell.execute_reply":"2024-08-28T15:06:59.050461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport missingno as msno\n\n# Plotting missing data matrix\nmsno.matrix(df_train, figsize=(32, 24))\n\n# Save the plot to a file\nplt.savefig('missing_data_matrix.png', bbox_inches='tight')\n\n# Show the plot\nplt.show()\n\n# Optionally close the plot to free up resources\nplt.close()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:06:59.052986Z","iopub.execute_input":"2024-08-28T15:06:59.053456Z","iopub.status.idle":"2024-08-28T15:07:02.975544Z","shell.execute_reply.started":"2024-08-28T15:06:59.053407Z","shell.execute_reply":"2024-08-28T15:07:02.974375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a mask of missing values\nmissing_data_mask = df_train.isnull()\n\n# Plot the missing values as a scatter plot\nplt.figure(figsize=(10, 6))\nfor col in df_train.columns:\n    missing_rows = missing_data_mask[col]\n    plt.scatter(missing_rows[missing_rows].index, [col] * missing_rows.sum(), label=col, alpha=0.6)\n\nplt.xlabel('Index')\nplt.ylabel('Columns with Missing Values')\nplt.title('Scatter Plot of Missing Values in Training Data')\nplt.xticks(rotation=45)\n#Place the legend below the plot\nplt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.15), ncol=3)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:02.976872Z","iopub.execute_input":"2024-08-28T15:07:02.977222Z","iopub.status.idle":"2024-08-28T15:07:04.295241Z","shell.execute_reply.started":"2024-08-28T15:07:02.977185Z","shell.execute_reply":"2024-08-28T15:07:04.294071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a DataFrame to hold the missing value information\nmissing_data_df = pd.DataFrame({\n    'Column': [col for col in df_train.columns for _ in range(df_train[col].isnull().sum())],\n    'Missing': [1] * df_train.isnull().sum().sum(),\n    'Index': [i for col in df_train.columns for i in range(df_train[col].isnull().sum())]\n})\n\n# Create a FacetGrid to plot missing values\ng = sns.FacetGrid(missing_data_df, col='Column', col_wrap=5, height=4, sharey=False)\ng.map_dataframe(sns.histplot, x='Index', hue='Missing', multiple='stack', palette='viridis')\ng.set_axis_labels('Index', 'Count')\ng.set_titles(col_template=\"{col_name}\")\ng.add_legend(title='Missing Value')\n\nplt.suptitle('Histogram of Missing Values per Column', y=1.02)\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:04.296856Z","iopub.execute_input":"2024-08-28T15:07:04.297328Z","iopub.status.idle":"2024-08-28T15:07:13.810100Z","shell.execute_reply.started":"2024-08-28T15:07:04.297277Z","shell.execute_reply":"2024-08-28T15:07:13.808878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a DataFrame to hold the missing value information\nmissing_data_df = pd.DataFrame({\n    'Column': [col for col in df_train.columns for _ in range(df_train[col].isnull().sum())],\n    'Missing': [1] * df_train.isnull().sum().sum()\n})\n\n# Plot the count of missing values using countplot\nplt.figure(figsize=(12, 6))\nsns.countplot(data=missing_data_df, x='Column', palette='viridis', order=df_train.columns)\n\nplt.xlabel('Columns')\nplt.ylabel('Number of Missing Values')\nplt.title('Count Plot of Missing Values per Column')\nplt.xticks(rotation=45)\nplt.grid(axis='y')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:13.811557Z","iopub.execute_input":"2024-08-28T15:07:13.811913Z","iopub.status.idle":"2024-08-28T15:07:14.479989Z","shell.execute_reply.started":"2024-08-28T15:07:13.811875Z","shell.execute_reply":"2024-08-28T15:07:14.478701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Calculate the number of missing values per column\nmissing_counts = df_train.isnull().sum()\n\n# Create a DataFrame from the missing counts\nmissing_counts_df = pd.DataFrame({\n    'Column': missing_counts.index,\n    'Missing Values': missing_counts.values\n})\n\n# Plot the line chart of missing values\nplt.figure(figsize=(12, 6))\nplt.plot(missing_counts_df['Column'], missing_counts_df['Missing Values'], marker='o', linestyle='-', color='b')\n\nplt.xlabel('Columns')\nplt.ylabel('Number of Missing Values')\nplt.title('Line Chart of Missing Values per Column')\nplt.xticks(rotation=45)\nplt.grid(True)\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:14.481729Z","iopub.execute_input":"2024-08-28T15:07:14.482735Z","iopub.status.idle":"2024-08-28T15:07:15.074755Z","shell.execute_reply.started":"2024-08-28T15:07:14.482679Z","shell.execute_reply":"2024-08-28T15:07:15.073619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Calculate the number of missing values per column\nmissing_counts = df_train.isnull().sum()\n\n# Create a DataFrame from the missing counts\nmissing_counts_df = pd.DataFrame({\n    'Column': missing_counts.index,\n    'Missing Values': missing_counts.values\n})\n\n# Plot the density plot of missing values\nplt.figure(figsize=(12, 6))\nsns.kdeplot(data=missing_counts_df, x='Missing Values', fill=True, color='darkblue', alpha=0.6)\n\nplt.xlabel('Number of Missing Values')\nplt.ylabel('Density')\nplt.title('Density Plot of Missing Values per Column')\nplt.grid(True)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:15.076699Z","iopub.execute_input":"2024-08-28T15:07:15.077125Z","iopub.status.idle":"2024-08-28T15:07:15.479782Z","shell.execute_reply.started":"2024-08-28T15:07:15.077078Z","shell.execute_reply":"2024-08-28T15:07:15.478566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport plotly.graph_objects as go\n\n# Calculate missing values by column\nmissing_counts = df_train.isnull().sum()\n\n# Create lists for the Sankey diagram\nlabels = missing_counts.index.tolist() + ['Missing Data']\nmissing_values = missing_counts.values.tolist() + [0]\n\n# Define the source and target for the Sankey diagram\nsources = list(range(len(missing_counts))) + [len(missing_counts)] * len(missing_counts)\ntargets = [len(missing_counts)] * len(missing_counts) + list(range(len(missing_counts)))\nvalues = missing_counts.values.tolist() + missing_counts.values.tolist()\n\n# Create the Sankey diagram\nfig = go.Figure(data=[go.Sankey(\n    node=dict(\n        pad=15,\n        thickness=20,\n        line=dict(color='black', width=0.5),\n        label=labels\n    ),\n    link=dict(\n        source=sources,\n        target=targets,\n        value=values\n    )\n)])\n\nfig.update_layout(title_text=\"Sankey Diagram of Missing Values\", font_size=10)\nfig.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:15.481396Z","iopub.execute_input":"2024-08-28T15:07:15.481767Z","iopub.status.idle":"2024-08-28T15:07:15.851121Z","shell.execute_reply.started":"2024-08-28T15:07:15.481728Z","shell.execute_reply":"2024-08-28T15:07:15.849837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport plotly.express as px\n\n# Calculate missing values by column\nmissing_counts = df_train.isnull().sum()\n\n# Create a DataFrame for the treemap\ntreemap_df = pd.DataFrame({\n    'Category': missing_counts.index,\n    'Missing Values': missing_counts.values\n})\n\n# Create the treemap\nfig = px.treemap(treemap_df, path=['Category'], values='Missing Values',\n                 title=\"Treemap of Missing Values in Training Data\")\n\nfig.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:15.852549Z","iopub.execute_input":"2024-08-28T15:07:15.853004Z","iopub.status.idle":"2024-08-28T15:07:17.474560Z","shell.execute_reply.started":"2024-08-28T15:07:15.852964Z","shell.execute_reply":"2024-08-28T15:07:17.473398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate the percentage of missing values for each column\nmissing_values = df_train.isnull().sum()\npercentage_missing = (missing_values / len(df_train)) * 100\n\n# Filter out columns with missing values\nmissing_percentage_df = percentage_missing[percentage_missing > 0]\n\n# Plot pie chart\nplt.figure(figsize=(10, 8))\nsizes = missing_percentage_df\nlabels = missing_percentage_df.index\ncolors = plt.cm.Paired(range(len(labels)))\n\n# Create the pie chart\nplt.pie(sizes, labels=labels, autopct='%1.1f%%', colors=colors)\n\n# Add a legend\nplt.legend(labels, title='Columns', loc='best', bbox_to_anchor=(1, 0.5))\n\nplt.title('Distribution of Missing Values by Column')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:17.476085Z","iopub.execute_input":"2024-08-28T15:07:17.476539Z","iopub.status.idle":"2024-08-28T15:07:18.185373Z","shell.execute_reply.started":"2024-08-28T15:07:17.476501Z","shell.execute_reply":"2024-08-28T15:07:18.184242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a DataFrame from the missing counts\nmissing_counts_df = pd.DataFrame({\n    'Column': missing_counts.index,\n    'Missing Values': missing_counts.values\n})\n\n# Create a new column for the 'Present' values\nmissing_counts_df['Present Values'] = df_train.shape[0] - missing_counts_df['Missing Values']\n\n# Set the column names to plot\ncolumns_to_plot = ['Missing Values', 'Present Values']\n\n# Plot the stacked bar plot\nplt.figure(figsize=(12, 8))\nmissing_counts_df.set_index('Column')[columns_to_plot].plot(kind='bar', stacked=True, color=['#FF6F61', '#6B5B95'])\n\nplt.xlabel('Columns')\nplt.ylabel('Count')\nplt.title('Stacked Bar Plot of Missing and Present Values per Column')\nplt.xticks(rotation=45)\nplt.legend(title='Data Type', labels=['Missing Values', 'Present Values'])\nplt.grid(axis='y')\n\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:18.187158Z","iopub.execute_input":"2024-08-28T15:07:18.187649Z","iopub.status.idle":"2024-08-28T15:07:18.790466Z","shell.execute_reply.started":"2024-08-28T15:07:18.187595Z","shell.execute_reply":"2024-08-28T15:07:18.788947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define a threshold for columns and rows (e.g., 5%)\ncolumn_threshold = 5  # Percentage\nrow_threshold = 5     # Percentage\n\n# Calculate percentage of missing values for columns\nmissing_percent_columns = df_train.isnull().mean() * 100\n\n# Drop columns with missing values above the threshold\ncols_to_drop = missing_percent_columns[missing_percent_columns > column_threshold].index\ndf_train_reduced = df_train.drop(columns=cols_to_drop)\n\n# Calculate percentage of missing values for rows\nmissing_percent_rows = df_train_reduced.isnull().mean(axis=1) * 100\n\n# Drop rows with missing values above the threshold\nrows_to_drop = missing_percent_rows[missing_percent_rows > row_threshold].index\ndf_train_reduced = df_train_reduced.drop(index=rows_to_drop)\n\ndisplay(df_train_reduced)\n# Output the number of rows and columns in the reduced dataset\nprint(f\"Reduced dataset shape: {df_train_reduced.shape}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:18.792467Z","iopub.execute_input":"2024-08-28T15:07:18.792963Z","iopub.status.idle":"2024-08-28T15:07:19.024500Z","shell.execute_reply.started":"2024-08-28T15:07:18.792911Z","shell.execute_reply":"2024-08-28T15:07:19.023435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Identify non-numeric columns\nnon_numeric_cols = df_train.select_dtypes(include=['object']).columns\nprint(\"Non-numeric columns:\", non_numeric_cols)","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:19.025768Z","iopub.execute_input":"2024-08-28T15:07:19.026118Z","iopub.status.idle":"2024-08-28T15:07:19.044857Z","shell.execute_reply.started":"2024-08-28T15:07:19.026074Z","shell.execute_reply":"2024-08-28T15:07:19.043670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv')\n\n# Summary statistics for numerical columns\ndf_train.describe()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:19.046340Z","iopub.execute_input":"2024-08-28T15:07:19.046718Z","iopub.status.idle":"2024-08-28T15:07:19.074904Z","shell.execute_reply.started":"2024-08-28T15:07:19.046673Z","shell.execute_reply":"2024-08-28T15:07:19.073777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Copy the dataframe\ndf_encoded = df_train.copy()\n\n# Convert categorical columns to numerical codes\ndf_encoded = df_encoded.apply(lambda col: col.astype('category').cat.codes if col.dtype == 'object' else col)\n\n# Calculate the correlation matrix\ncorrelation_matrix = df_encoded.corr()\n\n# Create a heatmap for the correlation matrix\nplt.figure(figsize=(12, 10))\nsns.heatmap(data=correlation_matrix, annot=True, cmap='coolwarm', fmt='.2f')\nplt.title('Correlation Matrix')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:19.076508Z","iopub.execute_input":"2024-08-28T15:07:19.076964Z","iopub.status.idle":"2024-08-28T15:07:21.438126Z","shell.execute_reply.started":"2024-08-28T15:07:19.076914Z","shell.execute_reply":"2024-08-28T15:07:21.436870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"here we have multiple columns with missing values, you can handle them based on their data types and the nature of the data","metadata":{}},{"cell_type":"code","source":"# List of categorical columns with missing values\ncategorical_columns = [\n    'left_neural_foraminal_narrowing_l1_l2',\n    'left_neural_foraminal_narrowing_l2_l3',\n    'left_neural_foraminal_narrowing_l3_l4',\n    'left_neural_foraminal_narrowing_l4_l5',\n    'left_neural_foraminal_narrowing_l5_s1',\n    'right_neural_foraminal_narrowing_l1_l2',\n    'right_neural_foraminal_narrowing_l2_l3',\n    'right_neural_foraminal_narrowing_l3_l4',\n    'right_neural_foraminal_narrowing_l4_l5',\n    'right_neural_foraminal_narrowing_l5_s1',\n    'left_subarticular_stenosis_l1_l2',\n    'left_subarticular_stenosis_l2_l3',\n    'left_subarticular_stenosis_l3_l4',\n    'left_subarticular_stenosis_l4_l5',\n    'left_subarticular_stenosis_l5_s1',\n    'right_subarticular_stenosis_l1_l2',\n    'right_subarticular_stenosis_l2_l3',\n    'right_subarticular_stenosis_l3_l4',\n    'right_subarticular_stenosis_l4_l5',\n    'right_subarticular_stenosis_l5_s1'\n]\n\nfor col in categorical_columns:\n    mode_value = df_train[col].mode()[0]  # Find the mode\n    df_train[col] = df_train[col].fillna(mode_value)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:21.439970Z","iopub.execute_input":"2024-08-28T15:07:21.440393Z","iopub.status.idle":"2024-08-28T15:07:21.470276Z","shell.execute_reply.started":"2024-08-28T15:07:21.440338Z","shell.execute_reply":"2024-08-28T15:07:21.468761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pairplot for selected columns\nselected_columns = df_train.columns[1:10]  # Select a subset for a clearer visualization\nsns.pairplot(df_train[selected_columns].astype('category').apply(lambda x: x.cat.codes))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:21.471780Z","iopub.execute_input":"2024-08-28T15:07:21.472333Z","iopub.status.idle":"2024-08-28T15:07:45.247737Z","shell.execute_reply.started":"2024-08-28T15:07:21.472279Z","shell.execute_reply":"2024-08-28T15:07:45.246288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select a subset of columns for clearer visualization\nselected_columns = df_train.columns[1:10]  # Adjust the range as needed\n\n# Copy the dataframe and encode categorical columns\ndf_selected = df_train[selected_columns].copy()\ndf_selected_encoded = df_selected.apply(lambda col: col.astype('category').cat.codes if col.dtype == 'object' else col)\n\n# Melt the encoded DataFrame to long-form\ndf_melted = df_selected_encoded.reset_index().melt(id_vars='index')\ndf_melted.columns = ['Index', 'Variable', 'Value']\n\n# Plot violin plots for the selected columns\nplt.figure(figsize=(14, 8))\nsns.violinplot(data=df_melted, x='Variable', y='Value', palette='coolwarm')\nplt.title('Violin Plots of Selected Columns')\nplt.xticks(rotation=45)\nplt.grid(True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:45.249790Z","iopub.execute_input":"2024-08-28T15:07:45.250226Z","iopub.status.idle":"2024-08-28T15:07:45.861118Z","shell.execute_reply.started":"2024-08-28T15:07:45.250178Z","shell.execute_reply":"2024-08-28T15:07:45.859814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select a subset of columns for clearer visualization\nselected_columns = df_train.columns[1:10]  # Adjust the range as needed\n\n# Copy the dataframe and encode categorical columns\ndf_selected = df_train[selected_columns].copy()\ndf_selected_encoded = df_selected.apply(lambda col: col.astype('category').cat.codes if col.dtype == 'object' else col)\n\n# Plot histograms for the selected columns\nplt.figure(figsize=(16, 12))\nfor i, column in enumerate(df_selected_encoded.columns, 1):\n    plt.subplot(len(df_selected_encoded.columns)//3 + 1, 3, i)\n    sns.histplot(df_selected_encoded[column], kde=False, color='skyblue')\n    plt.title(column)\n    plt.xlabel('Value')\n    plt.ylabel('Frequency')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:45.862696Z","iopub.execute_input":"2024-08-28T15:07:45.863594Z","iopub.status.idle":"2024-08-28T15:07:48.268944Z","shell.execute_reply.started":"2024-08-28T15:07:45.863551Z","shell.execute_reply":"2024-08-28T15:07:48.267695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Copy the dataframe and encode categorical columns\ndf_selected = df_train[selected_columns].copy()\ndf_selected_encoded = df_selected.apply(lambda col: col.astype('category').cat.codes if col.dtype == 'object' else col)\n\n# Compute the correlation matrix\ncorrelation_matrix = df_selected_encoded.corr()\n\n# Plot the heatmap of the correlation matrix\nplt.figure(figsize=(12, 10))\nsns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', fmt='.2f', vmin=-1, vmax=1)\nplt.title('Correlation Matrix Heatmap for Selected Columns')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:48.270310Z","iopub.execute_input":"2024-08-28T15:07:48.270670Z","iopub.status.idle":"2024-08-28T15:07:49.409533Z","shell.execute_reply.started":"2024-08-28T15:07:48.270634Z","shell.execute_reply":"2024-08-28T15:07:49.408421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_label_coordinates.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:49.410828Z","iopub.execute_input":"2024-08-28T15:07:49.411172Z","iopub.status.idle":"2024-08-28T15:07:49.425238Z","shell.execute_reply.started":"2024-08-28T15:07:49.411136Z","shell.execute_reply":"2024-08-28T15:07:49.424003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_label_coordinates.shape","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:49.426855Z","iopub.execute_input":"2024-08-28T15:07:49.427257Z","iopub.status.idle":"2024-08-28T15:07:49.441406Z","shell.execute_reply.started":"2024-08-28T15:07:49.427219Z","shell.execute_reply":"2024-08-28T15:07:49.440258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folder_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/1012284084'\ndicom_files = [f for f in os.listdir(folder_path) if f.endswith('.dcm')]","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:49.442758Z","iopub.execute_input":"2024-08-28T15:07:49.443102Z","iopub.status.idle":"2024-08-28T15:07:49.462678Z","shell.execute_reply.started":"2024-08-28T15:07:49.443069Z","shell.execute_reply":"2024-08-28T15:07:49.461538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folder_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/1012284084'\ndicom_files = [f for f in os.listdir(folder_path) if f.endswith('.dcm')]\nlabel_coordinates_df = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_label_coordinates.csv')\n\nstudy_id = folder_path.split('/')[-2]\nstudy_label_coordinates = label_coordinates_df[label_coordinates_df['study_id'] == int(study_id)]\nfiltered_dicom_files = []\nfiltered_label_coordinates = []\n\nfor dicom_file in dicom_files:\n    instance_number = int(dicom_file.split('.')[0])\n    corresponding_coordinates = study_label_coordinates[study_label_coordinates['instance_number'] == instance_number]\n    if not corresponding_coordinates.empty:\n        filtered_dicom_files.append(dicom_file)\n        filtered_label_coordinates.append(corresponding_coordinates)\nfig, axs = plt.subplots(1, 4, figsize=(20, 5))\nsecond_row_index = 1\nsecond_row_images = filtered_dicom_files[second_row_index : second_row_index + 4]\nsecond_row_coordinates = filtered_label_coordinates[second_row_index : second_row_index + 4]\n\nfor i, (dicom_file, label_coordinates) in enumerate(zip(second_row_images, second_row_coordinates)):\n    dicom_file_path = os.path.join(folder_path, dicom_file)\n    dicom_data = pydicom.dcmread(dicom_file_path)\n    image = dicom_data.pixel_array   \n    axs[i].imshow(image, cmap='gray')\n    axs[i].set_title(f'DICOM Image - {dicom_file}')\n    axs[i].axis('off')   \n    for _, row in label_coordinates.iterrows():\n        axs[i].plot(row['x'], row['y'], 'ro', markersize=5) \n        \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:49.464072Z","iopub.execute_input":"2024-08-28T15:07:49.464452Z","iopub.status.idle":"2024-08-28T15:07:50.466468Z","shell.execute_reply.started":"2024-08-28T15:07:49.464415Z","shell.execute_reply":"2024-08-28T15:07:50.465329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dicom(path):\n    dicom = pydicom.read_file(path)\n    data = dicom.pixel_array\n    data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:50.467695Z","iopub.execute_input":"2024-08-28T15:07:50.468017Z","iopub.status.idle":"2024-08-28T15:07:50.473772Z","shell.execute_reply.started":"2024-08-28T15:07:50.467984Z","shell.execute_reply":"2024-08-28T15:07:50.472761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nfrom matplotlib import animation, rc\n\nrc('animation', html='jshtml')\n\ndef load_dicom(filename):\n    ds = pydicom.dcmread(filename)\n    return ds.pixel_array\n\ndef load_dicom_line(path):\n    t_paths = sorted(\n        glob.glob(os.path.join(path, \"*\")), \n        key=lambda x: int(os.path.splitext(os.path.basename(x))[0].split(\"-\")[-1]),\n    )\n    images = []\n    for filename in t_paths:\n        data = load_dicom(filename)\n        if data.max() == 0:\n            continue\n        images.append(data)\n    return images\n\ndef create_animation(ims):\n    fig = plt.figure(figsize=(6, 6))\n    plt.axis('off')\n    im = plt.imshow(ims[0], cmap=\"gray\")\n\n    def animate_func(i):\n        im.set_array(ims[i])\n        return [im]\n\n    return animation.FuncAnimation(fig, animate_func, frames=len(ims), interval=1000//24)\n\npath_to_folder = \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/1792451510\"\nimages = load_dicom_line(path_to_folder)\nanim = create_animation(images)\nanim","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:50.475204Z","iopub.execute_input":"2024-08-28T15:07:50.475657Z","iopub.status.idle":"2024-08-28T15:07:53.150986Z","shell.execute_reply.started":"2024-08-28T15:07:50.475618Z","shell.execute_reply":"2024-08-28T15:07:53.149781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_dicom_files(path_to_folder, num_files=5):\n    files_glob = os.path.join(path_to_folder, \"*.dcm\")\n    dicom_files = sorted(glob.glob(files_glob), key=lambda f: int(os.path.splitext(os.path.basename(f))[0].split('-')[-1]))\n    return [pydicom.dcmread(f) for f in dicom_files[:num_files]]\n\ndef calculate_level(mean, std):\n    return mean + 1.7 * std\n\ndef stats_image(image):\n    noncero_pixels = image[np.nonzero(image)]\n    if noncero_pixels.size == 0:\n        mean = 0\n        std = 0\n    else:\n        mean = np.mean(noncero_pixels)\n        std = np.std(noncero_pixels)\n    return mean, std\n\ndef image_orientation(dicom):\n    rt = 'unknown'\n    x1, y1, _, x2, y2, _ = [round(v) for v in dicom.ImageOrientationPatient]\n    if (x1, y1, x2, y2) == (1, 0, 0, 0):\n        rt = 'coronal'\n    elif (x1, y1, x2, y2) == (1, 0, 0, 1):\n        rt = 'axial'\n    elif (x1, y1, x2, y2) == (0, 1, 0, 0):\n        rt = 'sagittal'\n    if rt == 'unknown':\n        raise ValueError(f'unknown ImageOrientationPatient: {dicom.ImageOrientationPatient}')\n    return rt\n\ndef plot_image_hist(image):\n    mean, std = stats_image(image)\n    pixels = image.ravel()\n    noncero_pixels = pixels[np.nonzero(pixels)]\n    noncero_pixels = (noncero_pixels - mean) / std\n    over_threshold = np.count_nonzero(noncero_pixels > calculate_level(mean, std))\n    \n    fig, (axi, axh) = plt.subplots(1, 2, figsize=(20, 3), gridspec_kw={'width_ratios': [1, 4]})\n    fig.suptitle(f'scan # ({over_threshold})')\n    \n    axh.hist(noncero_pixels, 200, range=(-5, 5))  \n    axh.set_xlim(-5, 5)\n\n    ax_limits = axh.get_ylim()\n    axh.vlines(mean, ymin=ax_limits[0], ymax=ax_limits[1], colors='r', label='Mean')\n    axh.vlines(mean + std, ymin=ax_limits[0], ymax=ax_limits[1], colors='g', linestyles='dotted', label='Mean + Std')\n    axh.vlines(calculate_level(mean, std), ymin=ax_limits[0], ymax=ax_limits[1], colors='b', linestyles='dashed', label='Threshold')\n\n    axi.imshow(image, cmap=plt.cm.gray)\n    axi.grid(False)\n    axi.axis('off')\n    axh.legend()\n    plt.show()\n\npath_to_folder = \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/1002894806/866293114\"\ndicom_files = read_dicom_files(path_to_folder)\n\nif dicom_files:\n    first_image = dicom_files[0].pixel_array\n    plot_image_hist(first_image)\nelse:\n    print(\"No DICOM files found.\")","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:53.152929Z","iopub.execute_input":"2024-08-28T15:07:53.153528Z","iopub.status.idle":"2024-08-28T15:07:53.901503Z","shell.execute_reply.started":"2024-08-28T15:07:53.153469Z","shell.execute_reply":"2024-08-28T15:07:53.900371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dicom_files[1] ","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:53.902995Z","iopub.execute_input":"2024-08-28T15:07:53.903437Z","iopub.status.idle":"2024-08-28T15:07:53.913252Z","shell.execute_reply.started":"2024-08-28T15:07:53.903370Z","shell.execute_reply":"2024-08-28T15:07:53.912238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dicom_images(folder_path, num_images=5):\n    dicom_files = sorted([os.path.join(folder_path, f) for f in os.listdir(folder_path) if f.endswith('.dcm')])[:num_images]\n    images = [pydicom.dcmread(f).pixel_array for f in dicom_files]\n    return np.stack(images, axis=-1)\n\ndef plot_3d(image, threshold=-300):\n\n    verts, faces, _, _ = measure.marching_cubes(image, level=threshold)  \n    \n    fig = plt.figure(figsize=(10, 10))\n    ax = fig.add_subplot(111, projection='3d')    \n    \n    mesh = Poly3DCollection(verts[faces], alpha=0.1)\n    face_color = [0.5, 0.5, 1]  # light blue\n    mesh.set_facecolor(face_color)\n    ax.add_collection3d(mesh)\n    \n    ax.set_xlim(0, image.shape[0])\n    ax.set_ylim(0, image.shape[1])\n    ax.set_zlim(0, image.shape[2])\n    \n    plt.show()\n\n\nfolder_path = \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/1012284084\"\ndicom_images = load_dicom_images(folder_path, num_images=5)\nplot_3d(dicom_images, threshold=300)","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:07:53.914529Z","iopub.execute_input":"2024-08-28T15:07:53.914899Z","iopub.status.idle":"2024-08-28T15:08:07.607634Z","shell.execute_reply.started":"2024-08-28T15:07:53.914865Z","shell.execute_reply":"2024-08-28T15:08:07.605718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dicom_images(folder_path, num_images=5):\n    dicom_files = sorted([os.path.join(folder_path, f) for f in os.listdir(folder_path) if f.endswith('.dcm')])[:num_images]\n    images = [pydicom.dcmread(f).pixel_array for f in dicom_files]\n    return np.stack(images, axis=-1)\n\ndef plot_3d_interactive(image, threshold=-300):    \n    verts, faces, _, _ = measure.marching_cubes(image, level=threshold)   \n    \n    fig = go.Figure(data=[\n        go.Mesh3d(\n            x=verts[:, 0],\n            y=verts[:, 1],\n            z=verts[:, 2],\n            i=faces[:, 0],\n            j=faces[:, 1],\n            k=faces[:, 2],\n            color='blue',\n            opacity=0.1\n        )\n    ])\n    \n    fig.update_layout(\n        scene=dict(\n            xaxis=dict(visible=True),\n            yaxis=dict(visible=True),\n            zaxis=dict(visible=True)\n        ),\n        width=800,\n        height=800,\n        title=\"Interactive 3D DICOM Image Visualization\"\n    )\n    \n    fig.show()\n\n\nfolder_path = \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/1002894806/866293114\"\ndicom_images = load_dicom_images(folder_path, num_images=5)\nplot_3d_interactive(dicom_images, threshold=100)","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:08:07.609903Z","iopub.execute_input":"2024-08-28T15:08:07.610792Z","iopub.status.idle":"2024-08-28T15:08:07.893722Z","shell.execute_reply.started":"2024-08-28T15:08:07.610739Z","shell.execute_reply":"2024-08-28T15:08:07.892433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_image(image_path):\n    dicom = pydicom.dcmread(image_path)\n    pixel_array = dicom.pixel_array\n    image = (pixel_array - np.min(pixel_array)) / (np.max(pixel_array) - np.min(pixel_array)) * 255\n    image = image.astype(np.uint8)\n    return image\n\ndef plot_3d_image(image):   \n    fig = plt.figure(figsize=(10, 8))\n    ax = fig.add_subplot(111, projection='3d')\n    rows, cols = image.shape\n    x, y = np.meshgrid(np.arange(cols), np.arange(rows))\n    ax.plot_surface(x, y, image, cmap='viridis', edgecolor='none')\n    ax.set_xlabel('X')\n    ax.set_ylabel('Y')\n    ax.set_zlabel('Intensity')\n    ax.set_title('3D Plot of Image')\n    plt.show()\n\n# Correct path to a DICOM image\nexample_image_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/1012284084/1.dcm'\nif os.path.exists(example_image_path):\n    image = preprocess_image(example_image_path)\n    plot_3d_image(image)\nelse:\n    print(f\"File not found: {example_image_path}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:08:07.895268Z","iopub.execute_input":"2024-08-28T15:08:07.895680Z","iopub.status.idle":"2024-08-28T15:08:08.543560Z","shell.execute_reply.started":"2024-08-28T15:08:07.895641Z","shell.execute_reply":"2024-08-28T15:08:08.542353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Model Training**","metadata":{}},{"cell_type":"markdown","source":"> **Model Training Using Convolutional Neural Networks (CNNs)**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport os\nimport pydicom\nimport tensorflow as tf\nimport numpy as np\n\n# Paths to the image directory\nimage_dir = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/'\n\n# Load the label coordinates CSV\ndf_train_label_coordinates = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_label_coordinates.csv')\ndf_train_label_coordinates['study_id'] = df_train_label_coordinates['study_id'].astype(int)  # Ensure study_id is int\n\n# Get the first 100 unique study_ids\nstudy_ids = df_train_label_coordinates['study_id'].unique()[:100]  # Get first 100 unique study_ids\n\n# Filter the DataFrame for these study_ids\nfiltered_df = df_train_label_coordinates[df_train_label_coordinates['study_id'].isin(study_ids)]\n\n# Load the training data CSV\ndf_train = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv')\n\n# Initialize lists to store the data\nimages = []\nseries_ids = []\nstudy_ids_labels = []\nconditions = []\nlevels = []\n\ntotal_images = len(filtered_df)\nprocessed_images = 0\n\nprint(\"Loading images...\")\n\nfor idx, row in filtered_df.iterrows():\n    study_id = str(row['study_id'])\n    series_id = str(row['series_id'])\n    instance_number = str(row['instance_number'])\n    condition = row['condition']\n    level = row['level']\n    \n    series_path = os.path.join(image_dir, study_id, series_id)\n    img_file = f\"{instance_number}.dcm\"\n    img_path = os.path.join(series_path, img_file)\n    \n    if os.path.exists(img_path):\n        # Load DICOM image\n        ds = pydicom.dcmread(img_path)\n        img = ds.pixel_array\n        if len(img.shape) == 2:\n            img = np.expand_dims(img, axis=-1)  # Add channel dimension\n        img = tf.image.resize(img, (128, 128))  # Resize image\n        img = img.numpy()  # Convert Tensor to numpy array\n        images.append(img)\n        series_ids.append(series_id)\n        study_ids_labels.append(study_id)\n        conditions.append(condition)\n        levels.append(level)\n        \n    processed_images += 1\n    \n    # Print progress every 100 images\n    if processed_images % 100 == 0:\n        print(f\"Processed {processed_images}/{total_images} images.\")\n\n# Convert lists to numpy arrays\nimages = np.array(images)\nseries_ids = np.array(series_ids)\nstudy_ids_labels = np.array(study_ids_labels)\nconditions = np.array(conditions)\nlevels = np.array(levels)\n\nprint(f\"Loaded {len(images)} images from {len(np.unique(study_ids_labels))} study IDs.\")\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:08:08.545236Z","iopub.execute_input":"2024-08-28T15:08:08.545715Z","iopub.status.idle":"2024-08-28T15:08:51.952752Z","shell.execute_reply.started":"2024-08-28T15:08:08.545665Z","shell.execute_reply":"2024-08-28T15:08:51.951369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nfrom tensorflow.keras.utils import to_categorical\n\n# Initialize label encoders\nle_series = LabelEncoder()\nle_study = LabelEncoder()\nle_condition = LabelEncoder()\nle_level = LabelEncoder()\n\n# Fit and transform labels\nseries_ids_encoded = le_series.fit_transform(series_ids)\nstudy_ids_encoded = le_study.fit_transform(study_ids_labels)\nconditions_encoded = le_condition.fit_transform(conditions)\nlevels_encoded = le_level.fit_transform(levels)\n\n# Convert to categorical\nseries_ids_categorical = to_categorical(series_ids_encoded)\nstudy_ids_categorical = to_categorical(study_ids_encoded)\nconditions_categorical = to_categorical(conditions_encoded)\nlevels_categorical = to_categorical(levels_encoded)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:08:51.954060Z","iopub.execute_input":"2024-08-28T15:08:51.954506Z","iopub.status.idle":"2024-08-28T15:08:51.966792Z","shell.execute_reply.started":"2024-08-28T15:08:51.954467Z","shell.execute_reply":"2024-08-28T15:08:51.965622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n# Split the data into training and validation sets\nX_train, X_val, y_series_train, y_series_val, y_study_train, y_study_val, y_condition_train, y_condition_val, y_level_train, y_level_val = train_test_split(\n    images, series_ids_categorical, study_ids_categorical, conditions_categorical, levels_categorical, test_size=0.2, random_state=42)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:08:51.968190Z","iopub.execute_input":"2024-08-28T15:08:51.968579Z","iopub.status.idle":"2024-08-28T15:08:52.030651Z","shell.execute_reply.started":"2024-08-28T15:08:51.968540Z","shell.execute_reply":"2024-08-28T15:08:52.029471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input, Conv2D, MaxPooling2D, Flatten, Dense\n\ndef build_cnn_model(input_shape, num_series_classes, num_study_classes, num_condition_classes, num_level_classes):\n    inputs = Input(shape=input_shape)\n    x = Conv2D(32, (3, 3), activation='relu')(inputs)\n    x = MaxPooling2D((2, 2))(x)\n    x = Conv2D(64, (3, 3), activation='relu')(x)\n    x = MaxPooling2D((2, 2))(x)\n    x = Flatten()(x)\n    x = Dense(128, activation='relu')(x)\n    \n    series_output = Dense(num_series_classes, activation='softmax', name='series_id')(x)\n    study_output = Dense(num_study_classes, activation='softmax', name='study_id')(x)\n    condition_output = Dense(num_condition_classes, activation='softmax', name='condition')(x)\n    level_output = Dense(num_level_classes, activation='softmax', name='level')(x)\n    \n    model = Model(inputs=inputs, outputs=[series_output, study_output, condition_output, level_output])\n    return model\n\n# Define the model\ninput_shape = (128, 128, 1)  # Image dimensions and channels\nnum_series_classes = len(le_series.classes_)\nnum_study_classes = len(le_study.classes_)\nnum_condition_classes = len(le_condition.classes_)\nnum_level_classes = len(le_level.classes_)\n\nmodel = build_cnn_model(input_shape, num_series_classes, num_study_classes, num_condition_classes, num_level_classes)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:08:52.032125Z","iopub.execute_input":"2024-08-28T15:08:52.032503Z","iopub.status.idle":"2024-08-28T15:08:52.175168Z","shell.execute_reply.started":"2024-08-28T15:08:52.032464Z","shell.execute_reply":"2024-08-28T15:08:52.174035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compile the model\nmodel.compile(\n    optimizer='adam',\n    loss={'series_id': 'categorical_crossentropy', 'study_id': 'categorical_crossentropy', 'condition': 'categorical_crossentropy', 'level': 'categorical_crossentropy'},\n    metrics={'series_id': 'accuracy', 'study_id': 'accuracy', 'condition': 'accuracy', 'level': 'accuracy'}\n)\n\n# Train the model\nhistory = model.fit(\n    X_train, \n    [y_series_train, y_study_train, y_condition_train, y_level_train], \n    validation_data=(X_val, [y_series_val, y_study_val, y_condition_val, y_level_val]), \n    epochs=10, \n    batch_size=32\n)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:08:52.176548Z","iopub.execute_input":"2024-08-28T15:08:52.176891Z","iopub.status.idle":"2024-08-28T15:13:47.771761Z","shell.execute_reply.started":"2024-08-28T15:08:52.176856Z","shell.execute_reply":"2024-08-28T15:13:47.770240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate on validation set\nval_loss, val_series_acc, val_study_acc, val_condition_acc, val_level_acc = model.evaluate(X_val, [y_series_val, y_study_val, y_condition_val, y_level_val])\nprint(f\"Validation Series ID Accuracy: {val_series_acc}\")\nprint(f\"Validation Study ID Accuracy: {val_study_acc}\")\nprint(f\"Validation Condition Accuracy: {val_condition_acc}\")\nprint(f\"Validation Level Accuracy: {val_level_acc}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:13:47.774296Z","iopub.execute_input":"2024-08-28T15:13:47.775145Z","iopub.status.idle":"2024-08-28T15:13:49.389965Z","shell.execute_reply.started":"2024-08-28T15:13:47.775088Z","shell.execute_reply":"2024-08-28T15:13:49.388715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\n\n# Define the folder where you want to save the files\nsave_dir = 'model_and_encoders_1'\nos.makedirs(save_dir, exist_ok=True)\n\n# Save the model\nmodel.save(os.path.join(save_dir, 'series_study_condition_level_model.h5'))\n\n# Save label encoders\nwith open(os.path.join(save_dir, 'le_series.pkl'), 'wb') as f:\n    pickle.dump(le_series, f)\nwith open(os.path.join(save_dir, 'le_study.pkl'), 'wb') as f:\n    pickle.dump(le_study, f)\nwith open(os.path.join(save_dir, 'le_condition.pkl'), 'wb') as f:\n    pickle.dump(le_condition, f)\nwith open(os.path.join(save_dir, 'le_level.pkl'), 'wb') as f:\n    pickle.dump(le_level, f)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:13:49.391482Z","iopub.execute_input":"2024-08-28T15:13:49.391832Z","iopub.status.idle":"2024-08-28T15:13:49.543657Z","shell.execute_reply.started":"2024-08-28T15:13:49.391797Z","shell.execute_reply":"2024-08-28T15:13:49.542710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pydicom\nimport tensorflow as tf\nimport numpy as np\nfrom tensorflow.keras.models import load_model\nimport pickle\nimport os\n\n# Path to the image\nimg_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/1012284084/1.dcm'\n\n# Load the DICOM image\nds = pydicom.dcmread(img_path)\nimg = ds.pixel_array\n\n# Preprocess the image\nif len(img.shape) == 2:\n    img = np.expand_dims(img, axis=-1)  # Add channel dimension\nimg = tf.image.resize(img, (128, 128))  # Resize image\nimg = np.expand_dims(img, axis=0)  # Add batch dimension\n\n# Define the new path to the model and encoders\nmodel_path = 'model_and_encoders_1/series_study_condition_level_model.h5'\nle_series_path = 'model_and_encoders_1/le_series.pkl'\nle_study_path = 'model_and_encoders_1/le_study.pkl'\nle_condition_path = 'model_and_encoders_1/le_condition.pkl'\nle_level_path = 'model_and_encoders_1/le_level.pkl'\n\n# Load the model\nif not os.path.isfile(model_path):\n    raise FileNotFoundError(f\"Model file not found at {model_path}\")\nmodel = load_model(model_path)\n\n# Load label encoders\nif not os.path.isfile(le_series_path):\n    raise FileNotFoundError(f\"Label encoder file not found at {le_series_path}\")\nwith open(le_series_path, 'rb') as f:\n    le_series = pickle.load(f)\n\nif not os.path.isfile(le_study_path):\n    raise FileNotFoundError(f\"Label encoder file not found at {le_study_path}\")\nwith open(le_study_path, 'rb') as f:\n    le_study = pickle.load(f)\n\nif not os.path.isfile(le_condition_path):\n    raise FileNotFoundError(f\"Label encoder file not found at {le_condition_path}\")\nwith open(le_condition_path, 'rb') as f:\n    le_condition = pickle.load(f)\n\nif not os.path.isfile(le_level_path):\n    raise FileNotFoundError(f\"Label encoder file not found at {le_level_path}\")\nwith open(le_level_path, 'rb') as f:\n    le_level = pickle.load(f)\n\n# Make predictions\npredictions = model.predict(img)\n\n# Extract the predictions\npred_series = predictions[0]\npred_study = predictions[1]\npred_condition = predictions[2]\npred_level = predictions[3]\n\n# Decode the predictions\npred_series_id = le_series.inverse_transform(np.argmax(pred_series, axis=1))[0]\npred_study_id = le_study.inverse_transform(np.argmax(pred_study, axis=1))[0]\npred_condition_label = le_condition.inverse_transform(np.argmax(pred_condition, axis=1))[0]\npred_level_label = le_level.inverse_transform(np.argmax(pred_level, axis=1))[0]\n\nprint(f\"Predicted Study ID: {pred_study_id}\")\nprint(f\"Predicted Series ID: {pred_series_id}\")\nprint(f\"Predicted Condition: {pred_condition_label}\")\nprint(f\"Predicted Level: {pred_level_label}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:13:49.545101Z","iopub.execute_input":"2024-08-28T15:13:49.545483Z","iopub.status.idle":"2024-08-28T15:13:49.865145Z","shell.execute_reply.started":"2024-08-28T15:13:49.545443Z","shell.execute_reply":"2024-08-28T15:13:49.863981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **Model Training Using The AlexNet CNN Architecture**","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input, Conv2D, MaxPooling2D, Flatten, Dense, Dropout, BatchNormalization\n\ndef build_alexnet_model(input_shape, num_series_classes, num_study_classes, num_condition_classes, num_level_classes):\n    inputs = Input(shape=input_shape)\n    \n    # Layer 1\n    x = Conv2D(96, (11, 11), strides=4, activation='relu', padding='same')(inputs)\n    x = MaxPooling2D((3, 3), strides=2)(x)\n    \n    # Layer 2\n    x = Conv2D(256, (5, 5), activation='relu', padding='same')(x)\n    x = MaxPooling2D((3, 3), strides=2)(x)\n    \n    # Layer 3\n    x = Conv2D(384, (3, 3), activation='relu', padding='same')(x)\n    \n    # Layer 4\n    x = Conv2D(384, (3, 3), activation='relu', padding='same')(x)\n    \n    # Layer 5\n    x = Conv2D(256, (3, 3), activation='relu', padding='same')(x)\n    x = MaxPooling2D((3, 3), strides=2)(x)\n    \n    # Flatten and Fully Connected Layers\n    x = Flatten()(x)\n    x = Dense(4096, activation='relu')(x)\n    x = Dropout(0.5)(x)\n    x = Dense(4096, activation='relu')(x)\n    x = Dropout(0.5)(x)\n    \n    # Output layers\n    series_output = Dense(num_series_classes, activation='softmax', name='series_id')(x)\n    study_output = Dense(num_study_classes, activation='softmax', name='study_id')(x)\n    condition_output = Dense(num_condition_classes, activation='softmax', name='condition')(x)\n    level_output = Dense(num_level_classes, activation='softmax', name='level')(x)\n    \n    model = Model(inputs=inputs, outputs=[series_output, study_output, condition_output, level_output])\n    return model\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:13:49.866461Z","iopub.execute_input":"2024-08-28T15:13:49.866798Z","iopub.status.idle":"2024-08-28T15:13:49.878311Z","shell.execute_reply.started":"2024-08-28T15:13:49.866762Z","shell.execute_reply":"2024-08-28T15:13:49.877167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nfrom tensorflow.keras.utils import to_categorical\n\n# Initialize label encoders\nle_series = LabelEncoder()\nle_study = LabelEncoder()\nle_condition = LabelEncoder()\nle_level = LabelEncoder()\n\n# Fit and transform labels\nseries_ids_encoded = le_series.fit_transform(series_ids)\nstudy_ids_encoded = le_study.fit_transform(study_ids_labels)\nconditions_encoded = le_condition.fit_transform(conditions)\nlevels_encoded = le_level.fit_transform(levels)\n\n# Convert to categorical\nseries_ids_categorical = to_categorical(series_ids_encoded)\nstudy_ids_categorical = to_categorical(study_ids_encoded)\nconditions_categorical = to_categorical(conditions_encoded)\nlevels_categorical = to_categorical(levels_encoded)\n\n# Now you can define the model\ninput_shape = (128, 128, 1)  # Image dimensions and channels\nnum_series_classes = len(le_series.classes_)\nnum_study_classes = len(le_study.classes_)\nnum_condition_classes = len(le_condition.classes_)\nnum_level_classes = len(le_level.classes_)\n\nmodel = build_alexnet_model(input_shape, num_series_classes, num_study_classes, num_condition_classes, num_level_classes)\n\n# Compile the model\nmodel.compile(\n    optimizer='adam',\n    loss={'series_id': 'categorical_crossentropy', 'study_id': 'categorical_crossentropy', 'condition': 'categorical_crossentropy', 'level': 'categorical_crossentropy'},\n    metrics={'series_id': 'accuracy', 'study_id': 'accuracy', 'condition': 'accuracy', 'level': 'accuracy'}\n)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:13:49.890368Z","iopub.execute_input":"2024-08-28T15:13:49.890885Z","iopub.status.idle":"2024-08-28T15:13:50.184491Z","shell.execute_reply.started":"2024-08-28T15:13:49.890837Z","shell.execute_reply":"2024-08-28T15:13:50.183189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n# Split the data into training and validation sets\nX_train, X_val, y_series_train, y_series_val, y_study_train, y_study_val, y_condition_train, y_condition_val, y_level_train, y_level_val = train_test_split(\n    images, series_ids_categorical, study_ids_categorical, conditions_categorical, levels_categorical, test_size=0.2, random_state=42\n)\n\n# Now you can train the model\nhistory = model.fit(\n    X_train, \n    [y_series_train, y_study_train, y_condition_train, y_level_train], \n    validation_data=(X_val, [y_series_val, y_study_val, y_condition_val, y_level_val]), \n    epochs=10, \n    batch_size=32\n)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:13:50.185907Z","iopub.execute_input":"2024-08-28T15:13:50.186260Z","iopub.status.idle":"2024-08-28T15:24:54.470200Z","shell.execute_reply.started":"2024-08-28T15:13:50.186224Z","shell.execute_reply":"2024-08-28T15:24:54.468751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate on validation set\nval_loss, val_series_acc, val_study_acc, val_condition_acc, val_level_acc = model.evaluate(X_val, [y_series_val, y_study_val, y_condition_val, y_level_val])\nprint(f\"Validation Series ID Accuracy: {val_series_acc}\")\nprint(f\"Validation Study ID Accuracy: {val_study_acc}\")\nprint(f\"Validation Condition Accuracy: {val_condition_acc}\")\nprint(f\"Validation Level Accuracy: {val_level_acc}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:24:54.472698Z","iopub.execute_input":"2024-08-28T15:24:54.473181Z","iopub.status.idle":"2024-08-28T15:24:57.237405Z","shell.execute_reply.started":"2024-08-28T15:24:54.473128Z","shell.execute_reply":"2024-08-28T15:24:57.236215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nimport os\n\n# Check if model is properly trained\nif model is None:\n    raise ValueError(\"The model is not defined. Please ensure it has been trained correctly.\")\n\n# Define the folder where you want to save the files\nsave_dir = 'model_and_encoders_1'\nos.makedirs(save_dir, exist_ok=True)\n\n# Verify directory creation\nprint(f\"Saving files to directory: {save_dir}\")\n\n# Save the model\ntry:\n    model.save(os.path.join(save_dir, 'alexnet_series_study_condition_level_model.h5'))\n    print(\"Model saved successfully.\")\nexcept Exception as e:\n    print(f\"Failed to save model: {e}\")\n\n# Save label encoders\ntry:\n    with open(os.path.join(save_dir, 'le_series.pkl'), 'wb') as f:\n        pickle.dump(le_series, f)\n    with open(os.path.join(save_dir, 'le_study.pkl'), 'wb') as f:\n        pickle.dump(le_study, f)\n    with open(os.path.join(save_dir, 'le_condition.pkl'), 'wb') as f:\n        pickle.dump(le_condition, f)\n    with open(os.path.join(save_dir, 'le_level.pkl'), 'wb') as f:\n        pickle.dump(le_level, f)\n    print(\"Label encoders saved successfully.\")\nexcept Exception as e:\n    print(f\"Failed to save label encoders: {e}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:24:57.238664Z","iopub.execute_input":"2024-08-28T15:24:57.239036Z","iopub.status.idle":"2024-08-28T15:24:57.873957Z","shell.execute_reply.started":"2024-08-28T15:24:57.238996Z","shell.execute_reply":"2024-08-28T15:24:57.872602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nimport numpy as np\nimport pydicom\nimport tensorflow as tf\nfrom tensorflow.keras.models import load_model\n\n# Define the path to the saved model and label encoders\nsave_dir = 'model_and_encoders_1'\nmodel_path = os.path.join(save_dir, 'alexnet_series_study_condition_level_model.h5')\nle_series_path = os.path.join(save_dir, 'le_series.pkl')\nle_study_path = os.path.join(save_dir, 'le_study.pkl')\nle_condition_path = os.path.join(save_dir, 'le_condition.pkl')\nle_level_path = os.path.join(save_dir, 'le_level.pkl')\n\n# Load the model\nmodel = load_model(model_path)\n\n# Load label encoders\nwith open(le_series_path, 'rb') as f:\n    le_series = pickle.load(f)\nwith open(le_study_path, 'rb') as f:\n    le_study = pickle.load(f)\nwith open(le_condition_path, 'rb') as f:\n    le_condition = pickle.load(f)\nwith open(le_level_path, 'rb') as f:\n    le_level = pickle.load(f)\n\n# Load and preprocess the image\ndef preprocess_image(img_path, target_size=(128, 128)):\n    ds = pydicom.dcmread(img_path)\n    img = ds.pixel_array\n    if len(img.shape) == 2:\n        img = np.expand_dims(img, axis=-1)  # Add channel dimension\n    img = tf.image.resize(img, target_size)  # Resize image\n    img = img.numpy()  # Convert Tensor to numpy array\n    img = img / 255.0  # Normalize image to [0, 1]\n    return np.expand_dims(img, axis=0)  # Add batch dimension\n\n# Path to the image\nimg_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/1002894806/1252873726/10.dcm'\n\n# Preprocess the image\npreprocessed_img = preprocess_image(img_path)\n\n# Make predictions\npredictions = model.predict(preprocessed_img)\n\n# Decode predictions\ndef decode_predictions(predictions, le_series, le_study, le_condition, le_level):\n    series_pred = np.argmax(predictions[0], axis=1)\n    study_pred = np.argmax(predictions[1], axis=1)\n    condition_pred = np.argmax(predictions[2], axis=1)\n    level_pred = np.argmax(predictions[3], axis=1)\n    \n    series_label = le_series.inverse_transform(series_pred)[0]\n    study_label = le_study.inverse_transform(study_pred)[0]\n    condition_label = le_condition.inverse_transform(condition_pred)[0]\n    level_label = le_level.inverse_transform(level_pred)[0]\n    \n    return series_label, study_label, condition_label, level_label\n\n# Decode the predictions\nseries_label, study_label, condition_label, level_label = decode_predictions(\n    predictions, le_series, le_study, le_condition, le_level\n)\n\n# Print the results\nprint(f\"Predicted Series ID: {series_label}\")\nprint(f\"Predicted Study ID: {study_label}\")\nprint(f\"Predicted Condition: {condition_label}\")\nprint(f\"Predicted Level: {level_label}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:24:57.875460Z","iopub.execute_input":"2024-08-28T15:24:57.875855Z","iopub.status.idle":"2024-08-28T15:24:58.669161Z","shell.execute_reply.started":"2024-08-28T15:24:57.875814Z","shell.execute_reply":"2024-08-28T15:24:58.668061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\nimport os\n\n# Define the directory you want to download\nmodel_folder = 'model_and_encoders_1'\n\n# Create a ZIP file of the model folder\nshutil.make_archive(model_folder, 'zip', model_folder)\n\n# This will create a file 'model_and_encoders_1.zip' in the current directory\nfrom IPython.display import FileLink\n\n# Create a download link for the ZIP file\nFileLink(f\"{model_folder}.zip\")\n","metadata":{"execution":{"iopub.status.busy":"2024-08-28T15:24:58.670742Z","iopub.execute_input":"2024-08-28T15:24:58.671209Z","iopub.status.idle":"2024-08-28T15:25:27.790143Z","shell.execute_reply.started":"2024-08-28T15:24:58.671160Z","shell.execute_reply":"2024-08-28T15:25:27.788686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"-------------------------------------------------------------","metadata":{}},{"cell_type":"markdown","source":"[see the next notepad](https://www.kaggle.com/code/thevindutvithanage/notebook2fd93b77b8)","metadata":{}}]}