{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":30746,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nfrom glob import glob\nfrom tqdm import tqdm\nfrom pathlib import Path\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport missingno as msno\nimport pydicom\ncompetition_dataset_directory = Path('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-08-24T08:40:16.041237Z","iopub.execute_input":"2024-08-24T08:40:16.041722Z","iopub.status.idle":"2024-08-24T08:40:16.048767Z","shell.execute_reply.started":"2024-08-24T08:40:16.041684Z","shell.execute_reply":"2024-08-24T08:40:16.047352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Overview\n\nThe task at hand involves classifying five degenerative conditions of the lower spine, which are critical for understanding spinal health. The three primary conditions are:\n   1. Spinal Canal Stenosis\n   2. Neural Foraminal Narrowing\n   3. Subarticular Stenosis\n   \nFor both neural foraminal narrowing and subarticular stenosis, evaluations are made for both sides of the spine, leading to five specific targets for classification:\n\n*    Left Neural Foraminal Narrowing\n*    Right Neural Foraminal Narrowing\n*    Left Subarticular Stenosis\n*    Right Subarticular Stenosis\n*    Spinal Canal Stenosis\n    \nEach imaging study in the dataset assigns severity scores for these conditions, categorized as Normal/Mild, Moderate, or Severe, across five lumbar disc levels:\n\n*   L1/L2: Between the first and second lumbar vertebrae\n*   L2/L3: Between the second and third lumbar vertebrae\n*   L3/L4: Between the third and fourth lumbar vertebrae\n*   L4/L5: Between the fourth and fifth lumbar vertebrae\n*   L5/S1: Between the fifth lumbar vertebra and the first sacral vertebra\n    \nThese disc levels are crucial as they indicate where the spine issues are assessed and classified, providing a comprehensive view of spinal health.\n\n\n\n","metadata":{}},{"cell_type":"code","source":"# Load the sample submission dataset\ndf_sample_submission = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/sample_submission.csv')\n\n# Display the first few rows of the dataframe\nprint(df_sample_submission.head())\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:16.051411Z","iopub.execute_input":"2024-08-24T08:40:16.051907Z","iopub.status.idle":"2024-08-24T08:40:16.074738Z","shell.execute_reply.started":"2024-08-24T08:40:16.051874Z","shell.execute_reply":"2024-08-24T08:40:16.073104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the test series descriptions dataset\ndf_test_series_descriptions = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_series_descriptions.csv')\n\n# Display the first few rows of the dataframe\nprint(df_test_series_descriptions.head())","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:16.076248Z","iopub.execute_input":"2024-08-24T08:40:16.076780Z","iopub.status.idle":"2024-08-24T08:40:16.089516Z","shell.execute_reply.started":"2024-08-24T08:40:16.076745Z","shell.execute_reply":"2024-08-24T08:40:16.087831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the training dataset\ndf_train = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv')\n\n# Display the first few rows of the dataframe\nprint(df_train.head())","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:16.092297Z","iopub.execute_input":"2024-08-24T08:40:16.092844Z","iopub.status.idle":"2024-08-24T08:40:16.125927Z","shell.execute_reply.started":"2024-08-24T08:40:16.092803Z","shell.execute_reply":"2024-08-24T08:40:16.124513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load datasets\ndf_train_series_descriptions = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_series_descriptions.csv')\ndf_test_series_descriptions = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_series_descriptions.csv')\ndf_sample_submission = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/sample_submission.csv')\ndf_train = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv')\n\n# Display dataframes\nprint(\"Train Series Descriptions:\")\nprint(df_train_series_descriptions.head()) \n\nprint(\"\\nTest Series Descriptions:\")\nprint(df_test_series_descriptions.head())\n\nprint(\"\\nSample Submission:\")\nprint(df_sample_submission.head())\n\nprint(\"\\nTraining Data:\")\nprint(df_train.head())\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:16.127484Z","iopub.execute_input":"2024-08-24T08:40:16.127944Z","iopub.status.idle":"2024-08-24T08:40:16.175965Z","shell.execute_reply.started":"2024-08-24T08:40:16.127909Z","shell.execute_reply":"2024-08-24T08:40:16.174683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom IPython.display import display\n\n# Load datasets\ndf_train_series_descriptions = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_series_descriptions.csv')\ndf_test_series_descriptions = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_series_descriptions.csv')\ndf_sample_submission = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/sample_submission.csv')\ndf_train = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv')\ndf_train_label_coordinates = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_label_coordinates.csv')\n\n# Display dataframes as tables\nprint(\"Train Series Descriptions:\")\ndisplay(df_train_series_descriptions)\n\nprint(\"\\nTest Series Descriptions:\")\ndisplay(df_test_series_descriptions)\n\nprint(\"\\nSample Submission:\")\ndisplay(df_sample_submission)\n\nprint(\"\\nTraining Data:\")\ndisplay(df_train)\n\nprint(\"\\nTraining label coordinates:\")\ndisplay(df_train_label_coordinates)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:16.177630Z","iopub.execute_input":"2024-08-24T08:40:16.178120Z","iopub.status.idle":"2024-08-24T08:40:16.345474Z","shell.execute_reply.started":"2024-08-24T08:40:16.178076Z","shell.execute_reply":"2024-08-24T08:40:16.344053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. What are the conditions in given dataset\n\nThe dataset contains three conditions, and their distributions across the different lumbar levels may vary. \nTherefore, it is essential to analyze these conditions collectively.","metadata":{}},{"cell_type":"code","source":"def visualize_condition_counts(df, title, path=None):\n    \n    \"\"\"\n    Visualize condition counts on training set\n    \n    Parameters\n    ----------\n    df: pandas.DataFrame\n        Counts and percentages of conditions\n        \n    title: str\n        Title of the plot\n        \n    path: str, pathlib.Path or None\n        Path of the output file (if path is None, plot is displayed with selected backend)\n    \"\"\"\n    \n    fig, ax = plt.subplots(figsize=(24, 16))\n\n    ax.barh(\n        y=np.arange(df.shape[0] // 3) - 0.2,\n        width=df['count'].values[0::3],\n        height=0.2,\n        align='center',\n        label='Normal/Mild'\n    )\n    ax.barh(\n        y=np.arange(df.shape[0] // 3),\n        width=df['count'].values[1::3],\n        height=0.2,\n        align='center',\n        label='Moderate'\n    )\n    ax.barh(\n        y=np.arange(df.shape[0] // 3) + 0.2,\n        width=df['count'].values[2::3],\n        height=0.2,\n        align='center',\n        label='Severe'\n    )\n\n    ax.set_yticks(np.arange(df.shape[0] // 3))\n    ax.set_yticklabels([\n        f'{level}\\nNormal Count: {normal_count} ({normal_percentage:.2f}%)\\nModerate Count: {moderate_count} ({moderate_percentage:.2f}%)\\nSevere Count: {severe_count} ({severe_percentage:.2f}%)' for level, normal_count, normal_percentage, moderate_count, moderate_percentage, severe_count, severe_percentage, in zip(\n            df['level'].values[0::3],\n            df['count'].values[0::3],\n            df['percentage'].values[0::3],\n            df['count'].values[1::3],\n            df['percentage'].values[1::3],\n            df['count'].values[2::3],\n            df['percentage'].values[2::3],\n        )\n    ])\n    ax.set_xlabel('')\n    ax.tick_params(axis='x', labelsize=17.5, pad=10)\n    ax.tick_params(axis='y', labelsize=17.5, pad=10)\n    ax.set_title(title, size=20, pad=15)\n    ax.legend(loc='best', prop={'size': 18})\n    plt.gca().invert_yaxis()\n\n    plt.show()\n\n    if path is None:\n        plt.show()\n    else:\n        plt.savefig(path)\n        plt.close(fig)","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:16.348231Z","iopub.execute_input":"2024-08-24T08:40:16.348663Z","iopub.status.idle":"2024-08-24T08:40:16.363395Z","shell.execute_reply.started":"2024-08-24T08:40:16.348627Z","shell.execute_reply":"2024-08-24T08:40:16.362079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***2.1. Spinal Canal Stenosis***\n\nThe spinal canal is a vital passageway that runs through the vertebrae, housing the spinal cord. When this canal narrows, it can exert pressure on the spinal cord and the nerve roots branching from it, leading to discomfort and various nerve-related issues, such as back pain and sciatica. Spinal canal stenosis can arise from multiple conditions and injuries, and while it can affect anyone, it is most prevalent in individuals over 50.\n\n***Areas Affected***\n\nSpinal stenosis typically impacts two main regions of the spine\n\n* Lumbar Spinal Stenosis: This form affects the lower back, where the five largest vertebrae (L1 to L5) are located.\n* Cervical Spinal Stenosis: This type occurs in the neck, involving the seven vertebrae labeled C1 to C7.\n* Although less common, spinal stenosis can also occur in the thoracic spine (middle back).\n\n<!-- ![image.png](attachment:ded06cc4-18f3-4245-8583-6294270bc9c2.png) -->\n\n\n**Symptoms and Causes**\n\nSymptoms of spinal canal stenosis can include **pain**, **numbness**, and **weakness**, often **worsening** when standing or walking. \nThe condition is usually caused by age-related changes in the spine, such as **arthritis**, which can lead to the thickening of ligaments \nand the formation of bone spurs that narrow the spinal canal.","metadata":{}},{"cell_type":"code","source":"spinal_canal_stenosis_columns = [column for column in df_train.columns if column.startswith('spinal_canal_stenosis')]\ndf_train_spinal_canal_stenosis = []\n\nfor column in spinal_canal_stenosis_columns:\n    df = df_train[[column]].copy(deep=True)\n    df['level'] = '_'.join(column.split('_')[-2:])\n    df = df.rename(columns={column: 'spinal_canal_stenosis'})\n    df_train_spinal_canal_stenosis.append(df)\n    \ndf_train_spinal_canal_stenosis = pd.concat(df_train_spinal_canal_stenosis, axis=0).reset_index(drop=True)\n\ndf_train_spinal_canal_stenosis_counts = df_train_spinal_canal_stenosis.groupby('level').value_counts().reset_index()\ndf_train_spinal_canal_stenosis_counts['severity'] = df_train_spinal_canal_stenosis_counts['spinal_canal_stenosis'].map({\n    'Normal/Mild': 0,\n    'Moderate': 1,\n    'Severe': 2\n})\ndf_train_spinal_canal_stenosis_counts = df_train_spinal_canal_stenosis_counts.sort_values(by=['level', 'severity'], ascending=True)\ndf_train_spinal_canal_stenosis_counts['percentage'] = df_train_spinal_canal_stenosis_counts['count'] / df_train_spinal_canal_stenosis_counts.groupby('level')['count'].transform('sum') * 100","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:16.364840Z","iopub.execute_input":"2024-08-24T08:40:16.365753Z","iopub.status.idle":"2024-08-24T08:40:16.408903Z","shell.execute_reply.started":"2024-08-24T08:40:16.365706Z","shell.execute_reply":"2024-08-24T08:40:16.407645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_condition_counts(\n    df=df_train_spinal_canal_stenosis_counts,\n    title='Spinal Canal Stenosis Counts On Levels'\n)","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:16.410455Z","iopub.execute_input":"2024-08-24T08:40:16.410836Z","iopub.status.idle":"2024-08-24T08:40:16.972100Z","shell.execute_reply.started":"2024-08-24T08:40:16.410802Z","shell.execute_reply":"2024-08-24T08:40:16.970785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***2.2. Neural Foraminal Narrowing***\n\nNeural foraminal narrowing, also known as foraminal stenosis, occurs when the openings (foramina) through which spinal nerves exit the spinal cord become constricted. This narrowing can exert pressure on the affected nerves, potentially disrupting their signals and leading to nerve pain and, in severe cases, permanent nerve damage.\n\n<!-- ![image.png](attachment:9ff09a99-5a0e-4311-9a05-9e3dc6230725.png)-->\n\n\n**Understanding Neural Foramina**\n\nThe neural foramina are openings located between adjacent vertebrae, allowing spinal nerves to branch out to various parts of the body. The size and location of these openings vary throughout the spine, and the narrowing can occur in several regions\n\n* Lumbar Spine (Lower Back): This is the most common area affected by foraminal narrowing.\n* Cervical Spine (Neck): The second most common site for this condition.\n* Thoracic Spine (Upper and Middle Back): Less frequently affected.\n* Sacral Spine (Far Lower Back and Pelvis)\n* Coccygeal Spine (Tailbone)\n\n**Prevalence and Symptoms**\n\nForaminal narrowing is particularly common in individuals over the age of 55, with studies suggesting that up to 40% of people may experience at least moderate narrowing in the lumbar region by age 60, increasing to approximately 75% in those aged 80 and older. Despite its prevalence, many individuals remain unaware of their condition, as only about 17.5% of those with severe foraminal narrowing experience symptoms.\nWhen symptoms do occur, they can manifest as:\n\n* Pain in the affected extremities, which may improve with rest.\n* Back pain that alleviates when bending or flexing.\n* Tingling, numbness, muscle weakness, and spasms in areas innervated by the compressed nerve.\n\n**Causes**\n\nThe primary cause of neural foraminal narrowing is often age-related degeneration of the spine, which can lead to anatomical changes that compress the nerves. Other contributing factors may include:\n\n* Herniated Discs: Discs that bulge into the foramina.\n* Bone Spurs: Abnormal growths that can block nerve pathways.\n* Thickened Ligaments: Ligaments near the spine that may become enlarged.\n* Congenital Factors: Some individuals may be born with a narrower spinal canal.\n\n**Treatment Options**\n\nTreatment for neural foraminal narrowing depends on the severity of the symptoms:\n\n* **Non-Surgical Treatments:** For mild to moderate symptoms, conservative options may include pain medications, physical therapy, and lifestyle modifications.\n* **Surgical Interventions:** If conservative treatments fail and symptoms persist, surgical options such as laminectomy, foraminotomy, or spinal fusion may be considered to relieve nerve compression\n","metadata":{}},{"cell_type":"code","source":"neural_foraminal_narrowing_columns = [column for column in df_train.columns if 'neural_foraminal_narrowing' in column]\ndf_train_neural_foraminal_narrowing = []\n\nfor column in neural_foraminal_narrowing_columns:\n    df = df_train[[column]].copy(deep=True)\n    df['level'] = '_'.join(column.split('_')[-2:])\n    df = df.rename(columns={column: 'neural_foraminal_narrowing'})\n    df_train_neural_foraminal_narrowing.append(df)\n    \ndf_train_neural_foraminal_narrowing = pd.concat(df_train_neural_foraminal_narrowing, axis=0).reset_index(drop=True)\n\ndf_train_neural_foraminal_narrowing_counts = df_train_neural_foraminal_narrowing.groupby('level').value_counts().reset_index()\ndf_train_neural_foraminal_narrowing_counts['severity'] = df_train_neural_foraminal_narrowing_counts['neural_foraminal_narrowing'].map({\n    'Normal/Mild': 0,\n    'Moderate': 1,\n    'Severe': 2\n})\ndf_train_neural_foraminal_narrowing_counts = df_train_neural_foraminal_narrowing_counts.sort_values(by=['level', 'severity'], ascending=True)\ndf_train_neural_foraminal_narrowing_counts['percentage'] = df_train_neural_foraminal_narrowing_counts['count'] / df_train_neural_foraminal_narrowing_counts.groupby('level')['count'].transform('sum') * 100\n\nleft_neural_foraminal_narrowing_columns = [column for column in df_train.columns if column.startswith('left_neural_foraminal_narrowing')]\ndf_train_left_neural_foraminal_narrowing = []\n\nfor column in left_neural_foraminal_narrowing_columns:\n    df = df_train[[column]].copy(deep=True)\n    df['level'] = '_'.join(column.split('_')[-2:])\n    df = df.rename(columns={column: 'left_neural_foraminal_narrowing'})\n    df_train_left_neural_foraminal_narrowing.append(df)\n    \ndf_train_left_neural_foraminal_narrowing = pd.concat(df_train_left_neural_foraminal_narrowing, axis=0).reset_index(drop=True)\n\ndf_train_left_neural_foraminal_narrowing_counts = df_train_left_neural_foraminal_narrowing.groupby('level').value_counts().reset_index()\ndf_train_left_neural_foraminal_narrowing_counts['severity'] = df_train_left_neural_foraminal_narrowing_counts['left_neural_foraminal_narrowing'].map({\n    'Normal/Mild': 0,\n    'Moderate': 1,\n    'Severe': 2\n})\ndf_train_left_neural_foraminal_narrowing_counts = df_train_left_neural_foraminal_narrowing_counts.sort_values(by=['level', 'severity'], ascending=True)\ndf_train_left_neural_foraminal_narrowing_counts['percentage'] = df_train_left_neural_foraminal_narrowing_counts['count'] / df_train_left_neural_foraminal_narrowing_counts.groupby('level')['count'].transform('sum') * 100\n\nright_neural_foraminal_narrowing_columns = [column for column in df_train.columns if column.startswith('right_neural_foraminal_narrowing')]\ndf_train_right_neural_foraminal_narrowing = []\n\nfor column in right_neural_foraminal_narrowing_columns:\n    df = df_train[[column]].copy(deep=True)\n    df['level'] = '_'.join(column.split('_')[-2:])\n    df = df.rename(columns={column: 'right_neural_foraminal_narrowing'})\n    df_train_right_neural_foraminal_narrowing.append(df)\n    \ndf_train_right_neural_foraminal_narrowing = pd.concat(df_train_right_neural_foraminal_narrowing, axis=0).reset_index(drop=True)\n\ndf_train_right_neural_foraminal_narrowing_counts = df_train_right_neural_foraminal_narrowing.groupby('level').value_counts().reset_index()\ndf_train_right_neural_foraminal_narrowing_counts['severity'] = df_train_right_neural_foraminal_narrowing_counts['right_neural_foraminal_narrowing'].map({\n    'Normal/Mild': 0,\n    'Moderate': 1,\n    'Severe': 2\n})\ndf_train_right_neural_foraminal_narrowing_counts = df_train_right_neural_foraminal_narrowing_counts.sort_values(by=['level', 'severity'], ascending=True)\ndf_train_right_neural_foraminal_narrowing_counts['percentage'] = df_train_right_neural_foraminal_narrowing_counts['count'] / df_train_right_neural_foraminal_narrowing_counts.groupby('level')['count'].transform('sum') * 100","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:16.974027Z","iopub.execute_input":"2024-08-24T08:40:16.974569Z","iopub.status.idle":"2024-08-24T08:40:17.081973Z","shell.execute_reply.started":"2024-08-24T08:40:16.974518Z","shell.execute_reply":"2024-08-24T08:40:17.080536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_condition_counts(\n    df=df_train_neural_foraminal_narrowing_counts,\n    title='Neural Foraminal Narrowing Counts On Levels'\n)\n\nvisualize_condition_counts(\n    df=df_train_left_neural_foraminal_narrowing_counts,\n    title='Left Neural Foraminal Narrowing Counts On Levels'\n)\n\nvisualize_condition_counts(\n    df=df_train_right_neural_foraminal_narrowing_counts,\n    title='Right Neural Foraminal Narrowing Counts On Levels'\n)","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:17.083633Z","iopub.execute_input":"2024-08-24T08:40:17.084139Z","iopub.status.idle":"2024-08-24T08:40:18.825960Z","shell.execute_reply.started":"2024-08-24T08:40:17.084097Z","shell.execute_reply":"2024-08-24T08:40:18.824509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Extract columns related to neural foraminal narrowing\nneural_foraminal_narrowing_columns = [col for col in df_train.columns if 'neural_foraminal_narrowing' in col]\n\n# Separate left and right columns\nleft_columns = neural_foraminal_narrowing_columns[:5]\nright_columns = neural_foraminal_narrowing_columns[5:]\n\n# Create DataFrame for left and right neural foraminal narrowing\ndf_train_left = pd.melt(\n    df_train[left_columns],\n    var_name='level',\n    value_name='left_neural_foraminal_narrowing'\n)\ndf_train_left['level'] = df_train_left['level'].apply(lambda x: '_'.join(x.split('_')[-2:]))\n\ndf_train_right = pd.melt(\n    df_train[right_columns],\n    var_name='level',\n    value_name='right_neural_foraminal_narrowing'\n)\ndf_train_right['level'] = df_train_right['level'].apply(lambda x: '_'.join(x.split('_')[-2:]))\n\n# Concatenate left and right data\ndf_train_left_right_neural_foraminal_narrowing = pd.concat([df_train_left['left_neural_foraminal_narrowing'], \n                                                            df_train_right[['right_neural_foraminal_narrowing', 'level']]], axis=1)\n\n# Map severity levels\nseverity_map = {'Normal/Mild': 0, 'Moderate': 1, 'Severe': 2}\ndf_train_left_right_neural_foraminal_narrowing['left_severity'] = df_train_left_right_neural_foraminal_narrowing['left_neural_foraminal_narrowing'].map(severity_map)\ndf_train_left_right_neural_foraminal_narrowing['right_severity'] = df_train_left_right_neural_foraminal_narrowing['right_neural_foraminal_narrowing'].map(severity_map)\n\n# Map level to numerical values\nlevel_map = {'l1_l2': 0, 'l2_l3': 1, 'l3_l4': 2, 'l4_l5': 3, 'l5_s1': 4}\ndf_train_left_right_neural_foraminal_narrowing['level_'] = df_train_left_right_neural_foraminal_narrowing['level'].map(level_map)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:18.827974Z","iopub.execute_input":"2024-08-24T08:40:18.828846Z","iopub.status.idle":"2024-08-24T08:40:18.884399Z","shell.execute_reply.started":"2024-08-24T08:40:18.828758Z","shell.execute_reply":"2024-08-24T08:40:18.883289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The severity of left and right neural foraminal narrowing exhibits a correlation coefficient of ***0.437***, indicating a moderate positive correlation between the two conditions. This suggests that individuals with more severe narrowing on one side of the spine are more likely to experience similar levels of narrowing on the opposite side. However, the correlation is not perfect, meaning that the severity of narrowing can still vary between the left and right sides of the spine within an individual. The presence of this correlation highlights the importance of evaluating both sides of the spine when assessing neural foraminal narrowing, as the condition may not always be symmetrical.","metadata":{}},{"cell_type":"code","source":"# Calculate and display correlations\ndf_neural_foraming_narrowing_correlations = df_train_left_right_neural_foraminal_narrowing.loc[:, ['level_', 'left_severity', 'right_severity']].corr()\ndf_neural_foraming_narrowing_correlations.style.background_gradient(cmap='coolwarm')\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:18.889613Z","iopub.execute_input":"2024-08-24T08:40:18.890014Z","iopub.status.idle":"2024-08-24T08:40:18.910257Z","shell.execute_reply.started":"2024-08-24T08:40:18.889980Z","shell.execute_reply":"2024-08-24T08:40:18.908869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***2.3. Subarticular Stenosis***\n\nSubarticular stenosis, often referred to as lateral recess stenosis, occurs when the space beneath the facet joints in the spine—known \nas the lateral recess—becomes constricted. This narrowing affects the spinal canal and can lead to compression of the spinal nerves \nas they pass through this region.\n\n","metadata":{}},{"cell_type":"code","source":"def process_subarticular_stenosis(side=None):\n    if side:\n        columns = [col for col in df_train.columns if col.startswith(f'{side}_subarticular_stenosis')]\n        col_name = f'{side}_subarticular_stenosis'\n    else:\n        columns = [col for col in df_train.columns if 'subarticular_stenosis' in col]\n        col_name = 'subarticular_stenosis'\n\n    df_subarticular_stenosis = pd.concat(\n        [df_train[[col]].rename(columns={col: col_name}).assign(level='_'.join(col.split('_')[-2:])) for col in columns],\n        axis=0\n    ).reset_index(drop=True)\n\n    df_counts = df_subarticular_stenosis.groupby('level')[col_name].value_counts().reset_index(name='count')\n    df_counts['severity'] = df_counts[col_name].map({'Normal/Mild': 0, 'Moderate': 1, 'Severe': 2})\n    df_counts = df_counts.sort_values(by=['level', 'severity']).reset_index(drop=True)\n    df_counts['percentage'] = df_counts['count'] / df_counts.groupby('level')['count'].transform('sum') * 100\n\n    return df_counts\n\n# Process and store the counts for overall, left, and right subarticular stenosis\ndf_train_subarticular_stenosis_counts = process_subarticular_stenosis()\ndf_train_left_subarticular_stenosis_counts = process_subarticular_stenosis('left')\ndf_train_right_subarticular_stenosis_counts = process_subarticular_stenosis('right')\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:18.911742Z","iopub.execute_input":"2024-08-24T08:40:18.912149Z","iopub.status.idle":"2024-08-24T08:40:18.999038Z","shell.execute_reply.started":"2024-08-24T08:40:18.912118Z","shell.execute_reply":"2024-08-24T08:40:18.997653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"titles = ['Subarticular Stenosis Counts On Levels', \n          'Left Subarticular Stenosis Counts On Levels', \n          'Right Subarticular Stenosis Counts On Levels']\n\ndfs = [df_train_subarticular_stenosis_counts, \n       df_train_left_subarticular_stenosis_counts, \n       df_train_right_subarticular_stenosis_counts]\n\nfor df, title in zip(dfs, titles):\n    visualize_condition_counts(df=df, title=title)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:19.000505Z","iopub.execute_input":"2024-08-24T08:40:19.000870Z","iopub.status.idle":"2024-08-24T08:40:20.753869Z","shell.execute_reply.started":"2024-08-24T08:40:19.000841Z","shell.execute_reply":"2024-08-24T08:40:20.752402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subarticular_stenosis_columns = [column for column in df_train.columns if 'subarticular_stenosis' in column]\ndf_train_left_right_subarticular_stenosis = []\n\nfor column in subarticular_stenosis_columns[:5]:\n    df = df_train[[column]].copy(deep=True)\n    df['level'] = '_'.join(column.split('_')[-2:])\n    df = df.rename(columns={column: 'left_subarticular_stenosis'})\n    df_train_left_right_subarticular_stenosis.append(df)\n\nfor column in subarticular_stenosis_columns[5:]:\n    df = df_train[[column]].copy(deep=True)\n    df['level'] = '_'.join(column.split('_')[-2:])\n    df = df.rename(columns={column: 'right_subarticular_stenosis'})\n    df_train_left_right_subarticular_stenosis.append(df)\n    \ndf_train_left_right_subarticular_stenosis = pd.concat((\n    pd.concat(df_train_left_right_subarticular_stenosis[:5], axis=0).reset_index(drop=True).iloc[:, :1],\n    pd.concat(df_train_left_right_subarticular_stenosis[5:], axis=0).reset_index(drop=True)\n), axis=1, ignore_index=False)\n\ndf_train_left_right_subarticular_stenosis['left_severity'] = df_train_left_right_subarticular_stenosis['left_subarticular_stenosis'].map({\n    'Normal/Mild': 0,\n    'Moderate': 1,\n    'Severe': 2\n})\ndf_train_left_right_subarticular_stenosis['right_severity'] = df_train_left_right_subarticular_stenosis['right_subarticular_stenosis'].map({\n    'Normal/Mild': 0,\n    'Moderate': 1,\n    'Severe': 2\n})\ndf_train_left_right_subarticular_stenosis['level_'] = df_train_left_right_subarticular_stenosis['level'].map({\n    'l1_l2': 0,\n    'l2_l3': 1,\n    'l3_l4': 2,\n    'l4_l5': 3,\n    'l5_s1': 4\n})","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:20.755377Z","iopub.execute_input":"2024-08-24T08:40:20.755863Z","iopub.status.idle":"2024-08-24T08:40:20.805230Z","shell.execute_reply.started":"2024-08-24T08:40:20.755818Z","shell.execute_reply":"2024-08-24T08:40:20.803640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From this analysis, the severity of left and right subarticular stenosis is found to have a correlation coefficient of 0.581. This indicates a significant positive correlation between the two sides, suggesting that an increase in severity on one side is likely associated with an increase in severity on the other side.\nAdditionally, the correlations between the disc level and the severities on both sides are relatively weak, with values of 0.236914 for left severity and 0.245518 for right severity. This implies that the specific disc level has a lesser influence on the severity of stenosis compared to the relationship between the left and right sides.","metadata":{}},{"cell_type":"code","source":"# Calculate correlations and display with a gradient background\ndf_subarticular_stenosis_correlations = df_train_left_right_subarticular_stenosis[['level_', 'left_severity', 'right_severity']].corr()\ndf_subarticular_stenosis_correlations.style.background_gradient(cmap='coolwarm')\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:20.806905Z","iopub.execute_input":"2024-08-24T08:40:20.807304Z","iopub.status.idle":"2024-08-24T08:40:20.829031Z","shell.execute_reply.started":"2024-08-24T08:40:20.807248Z","shell.execute_reply":"2024-08-24T08:40:20.827562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Labels**","metadata":{}},{"cell_type":"code","source":"# structure\ndf_train.info()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:20.831041Z","iopub.execute_input":"2024-08-24T08:40:20.831439Z","iopub.status.idle":"2024-08-24T08:40:20.849876Z","shell.execute_reply.started":"2024-08-24T08:40:20.831404Z","shell.execute_reply":"2024-08-24T08:40:20.848516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preview\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:20.851461Z","iopub.execute_input":"2024-08-24T08:40:20.851952Z","iopub.status.idle":"2024-08-24T08:40:20.884317Z","shell.execute_reply.started":"2024-08-24T08:40:20.851905Z","shell.execute_reply":"2024-08-24T08:40:20.882856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# structure\ndf_train_label_coordinates.info()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:20.885921Z","iopub.execute_input":"2024-08-24T08:40:20.886336Z","iopub.status.idle":"2024-08-24T08:40:20.909139Z","shell.execute_reply.started":"2024-08-24T08:40:20.886299Z","shell.execute_reply":"2024-08-24T08:40:20.907821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preview\ndf_train_label_coordinates.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:20.910731Z","iopub.execute_input":"2024-08-24T08:40:20.911098Z","iopub.status.idle":"2024-08-24T08:40:20.930914Z","shell.execute_reply.started":"2024-08-24T08:40:20.911070Z","shell.execute_reply":"2024-08-24T08:40:20.928870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# look at categories\nfor f in ['instance_number','condition','level']:\n    print(df_train_label_coordinates[f].value_counts())\n    print()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:20.932753Z","iopub.execute_input":"2024-08-24T08:40:20.933253Z","iopub.status.idle":"2024-08-24T08:40:20.960996Z","shell.execute_reply.started":"2024-08-24T08:40:20.933213Z","shell.execute_reply":"2024-08-24T08:40:20.959421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# numerical plot for instance number\nplt.figure(figsize=(7,1))\ndf_train_label_coordinates.instance_number.plot(kind='box', vert=False)\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:20.963079Z","iopub.execute_input":"2024-08-24T08:40:20.963672Z","iopub.status.idle":"2024-08-24T08:40:21.184619Z","shell.execute_reply.started":"2024-08-24T08:40:20.963634Z","shell.execute_reply":"2024-08-24T08:40:21.182959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# look at few rows with the very high instance numbers\ndf_train_label_coordinates[df_train_label_coordinates.instance_number>150]","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:21.186211Z","iopub.execute_input":"2024-08-24T08:40:21.186628Z","iopub.status.idle":"2024-08-24T08:40:21.205532Z","shell.execute_reply.started":"2024-08-24T08:40:21.186596Z","shell.execute_reply":"2024-08-24T08:40:21.204042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# condition vs level\npd.crosstab(df_train_label_coordinates.condition, df_train_label_coordinates.level)","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:21.207330Z","iopub.execute_input":"2024-08-24T08:40:21.207852Z","iopub.status.idle":"2024-08-24T08:40:21.248964Z","shell.execute_reply.started":"2024-08-24T08:40:21.207801Z","shell.execute_reply":"2024-08-24T08:40:21.247365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Description**","metadata":{}},{"cell_type":"code","source":"# structure\ndf_train_series_descriptions.info()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:21.251481Z","iopub.execute_input":"2024-08-24T08:40:21.251915Z","iopub.status.idle":"2024-08-24T08:40:21.267516Z","shell.execute_reply.started":"2024-08-24T08:40:21.251878Z","shell.execute_reply":"2024-08-24T08:40:21.265845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preview\ndf_train_series_descriptions.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:21.270320Z","iopub.execute_input":"2024-08-24T08:40:21.270970Z","iopub.status.idle":"2024-08-24T08:40:21.294873Z","shell.execute_reply.started":"2024-08-24T08:40:21.270923Z","shell.execute_reply":"2024-08-24T08:40:21.292986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# look at categories\ncounts = df_train_series_descriptions.series_description.value_counts()\nprint(counts)\nplt.figure(figsize=(7,3))\ncounts.plot(kind='bar', color='blue')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:21.297534Z","iopub.execute_input":"2024-08-24T08:40:21.298148Z","iopub.status.idle":"2024-08-24T08:40:21.547365Z","shell.execute_reply.started":"2024-08-24T08:40:21.298094Z","shell.execute_reply":"2024-08-24T08:40:21.545663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Combine Tables**","metadata":{}},{"cell_type":"code","source":"# Perform a left join on 'study_id' between df_train_label and df_train_main\ndf_train_step_1 = df_train_label_coordinates.merge(df_train, how='left', on='study_id').reset_index(drop=True)\ndf_train_step_1.head()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:21.548769Z","iopub.execute_input":"2024-08-24T08:40:21.549120Z","iopub.status.idle":"2024-08-24T08:40:21.639072Z","shell.execute_reply.started":"2024-08-24T08:40:21.549091Z","shell.execute_reply":"2024-08-24T08:40:21.637631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# join with third table\ndf_train = pd.merge(left=df_train_step_1, right=df_train_series_descriptions, how='left', on=['study_id', 'series_id']).reset_index(drop=True)\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:21.640782Z","iopub.execute_input":"2024-08-24T08:40:21.641246Z","iopub.status.idle":"2024-08-24T08:40:21.746023Z","shell.execute_reply.started":"2024-08-24T08:40:21.641203Z","shell.execute_reply":"2024-08-24T08:40:21.744680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert ids to categorical\ndf_train.study_id = df_train.study_id.astype('category')\ndf_train.series_id = df_train.series_id.astype('category')","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:21.754226Z","iopub.execute_input":"2024-08-24T08:40:21.754672Z","iopub.status.idle":"2024-08-24T08:40:21.767348Z","shell.execute_reply.started":"2024-08-24T08:40:21.754639Z","shell.execute_reply":"2024-08-24T08:40:21.766068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **3. EDA**","metadata":{}},{"cell_type":"code","source":"# combined table - basic stats\ndf_train.describe(include='all').T","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:21.769079Z","iopub.execute_input":"2024-08-24T08:40:21.769519Z","iopub.status.idle":"2024-08-24T08:40:21.992215Z","shell.execute_reply.started":"2024-08-24T08:40:21.769483Z","shell.execute_reply":"2024-08-24T08:40:21.990848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Reason for Suppressing FutureWarning\n\nIn the code below, we use the `warnings` module to suppress a specific `FutureWarning` related to an internal Pandas option (`use_inf_as_na`). This warning indicates that the option is deprecated and will be removed in a future version of Pandas.\n\n**Reason for Suppressing:**\n\nThis warning does not impact the functionality of our visualization and is primarily a notice for future changes in the Pandas library. To keep our output clean and focused on the key results, we suppress this warning. Additionally, suppressing this warning prevents it from cluttering our output and distracting from the main analysis.\n\n```python\nimport warnings\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nwarnings.filterwarnings(\"ignore\", category=FutureWarning, module=\"seaborn\")\n\nsns.jointplot(data=df_train, x='x', y='y', \n              color='darkblue', alpha=0.25)\nplt.grid()\nplt.show()\n\n","metadata":{}},{"cell_type":"code","source":"import warnings\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Suppress the specific warning\nwarnings.filterwarnings(\"ignore\", category=FutureWarning, module=\"seaborn\")\n\n# Plot coordinates\nsns.jointplot(data=df_train, x='x', y='y', \n              color='darkblue', alpha=0.25)\nplt.grid()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:21.993546Z","iopub.execute_input":"2024-08-24T08:40:21.993890Z","iopub.status.idle":"2024-08-24T08:40:23.828989Z","shell.execute_reply.started":"2024-08-24T08:40:21.993863Z","shell.execute_reply":"2024-08-24T08:40:23.827585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Suppress the specific warning\nwarnings.filterwarnings(\"ignore\", category=FutureWarning, module=\"seaborn\")\n\n# plot coordinates - colored by condition\nax = sns.jointplot(data=df_train, x='x', y='y', \n                   hue='condition', alpha=0.25)\nplt.legend(bbox_to_anchor=(1.2,1), loc=2)\nplt.grid()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:23.830866Z","iopub.execute_input":"2024-08-24T08:40:23.831282Z","iopub.status.idle":"2024-08-24T08:40:26.712998Z","shell.execute_reply.started":"2024-08-24T08:40:23.831226Z","shell.execute_reply":"2024-08-24T08:40:26.711648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Suppress the specific warning\nwarnings.filterwarnings(\"ignore\", category=FutureWarning, module=\"seaborn\")\n\n# plot coordinates - colored by level\nsns.jointplot(data=df_train, x='x', y='y', \n              hue='level', alpha=0.25)\nplt.legend(bbox_to_anchor=(1.2,1), loc=2)\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:26.714850Z","iopub.execute_input":"2024-08-24T08:40:26.715792Z","iopub.status.idle":"2024-08-24T08:40:29.586209Z","shell.execute_reply.started":"2024-08-24T08:40:26.715739Z","shell.execute_reply":"2024-08-24T08:40:29.584900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = df_train.columns.drop('study_id').tolist()\nprint(labels)","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:29.588109Z","iopub.execute_input":"2024-08-24T08:40:29.588696Z","iopub.status.idle":"2024-08-24T08:40:29.597044Z","shell.execute_reply.started":"2024-08-24T08:40:29.588639Z","shell.execute_reply":"2024-08-24T08:40:29.595651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Check for Duplication**","metadata":{}},{"cell_type":"code","source":"# Check for duplicate rows in each dataframe\nprint(\"Duplicate rows in Train Series Descriptions:\", df_train_series_descriptions.duplicated().sum())\nprint(\"Duplicate rows in Test Series Descriptions:\", df_test_series_descriptions.duplicated().sum())\nprint(\"Duplicate rows in Sample Submission:\", df_sample_submission.duplicated().sum())\nprint(\"Duplicate rows in Training Data:\", df_train.duplicated().sum())\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:29.598669Z","iopub.execute_input":"2024-08-24T08:40:29.599097Z","iopub.status.idle":"2024-08-24T08:40:29.739109Z","shell.execute_reply.started":"2024-08-24T08:40:29.599061Z","shell.execute_reply":"2024-08-24T08:40:29.737747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Missing Values Calculation**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n# Check for missing values\nprint(\"Missing values in Train Series Descriptions:\")\nprint(df_train_series_descriptions.isnull().sum())\n\nprint(\"\\nMissing values in Test Series Descriptions:\")\nprint(df_test_series_descriptions.isnull().sum())\n\nprint(\"\\nMissing values in Sample Submission:\")\nprint(df_sample_submission.isnull().sum())\n\nprint(\"\\nMissing values in Training Data:\")\nprint(df_train.isnull().sum())\n\nprint(\"\\nMissing values in Training Label Coordinates:\")\nprint(df_train_label_coordinates.isnull().sum())\n\n# Check data types\nprint(\"\\nData Types in Train Series Descriptions:\")\nprint(df_train_series_descriptions.dtypes)\n\nprint(\"\\nData Types in Test Series Descriptions:\")\nprint(df_test_series_descriptions.dtypes)\n\nprint(\"\\nData Types in Sample Submission:\")\nprint(df_sample_submission.dtypes)\n\nprint(\"\\nData Types in Training Data:\")\nprint(df_train.dtypes)\n\nprint(\"\\nData Types in Training Label Coordinates:\")\nprint(df_train_label_coordinates.dtypes)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:29.740680Z","iopub.execute_input":"2024-08-24T08:40:29.741230Z","iopub.status.idle":"2024-08-24T08:40:29.860589Z","shell.execute_reply.started":"2024-08-24T08:40:29.741182Z","shell.execute_reply":"2024-08-24T08:40:29.858978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Missing Values**\n\n1. No missing values in the \"Train Series Descriptions,\" \"Test Series Descriptions,\" \"Sample Submission,\" or \"Training Label Coordinates\" datasets.\n\n1. The \"Training Data\" dataset has several missing values across different columns. Notably:\n\n*     The right_neural_foraminal_narrowing_l1_l2 to right_neural_foraminal_narrowing_l5_s1 columns have 8 missing values each.\n*     left_subarticular_stenosis_l1_l2 and right_subarticular_stenosis_l1_l2 have a significant number of missing values (164 and 161, respectively).","metadata":{}},{"cell_type":"markdown","source":"**Code to Handle Missing Values**","metadata":{}},{"cell_type":"code","source":"# Check for missing values\ndf_train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:29.862622Z","iopub.execute_input":"2024-08-24T08:40:29.863081Z","iopub.status.idle":"2024-08-24T08:40:29.950446Z","shell.execute_reply.started":"2024-08-24T08:40:29.863041Z","shell.execute_reply":"2024-08-24T08:40:29.948250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# percentage of missing values\n(df_train.isnull().sum()/(len(df_train)))*100","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:29.952277Z","iopub.execute_input":"2024-08-24T08:40:29.952822Z","iopub.status.idle":"2024-08-24T08:40:30.039491Z","shell.execute_reply.started":"2024-08-24T08:40:29.952773Z","shell.execute_reply":"2024-08-24T08:40:30.038198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The percentage of missing values for the columns listed ranges from 0% to approximately 7.2%. This indicates that while some columns, such as `left_neural_foraminal_narrowing_l1_l2` and `right_neural_foraminal_narrowing_l1_l2`, have a very low percentage of missing data (around 0.06% to 0.37%), other columns, like `left_subarticular_stenosis_l1_l2` and `right_subarticular_stenosis_l1_l2`, have higher percentages of missing data, reaching up to 7.2%. This suggests that the data quality varies across different features, and addressing these missing values may be necessary depending on the analysis goals.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\n# Load your dataset\ndf_train = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv')\n\n# Define a threshold for columns and rows (e.g., 5%)\ncolumn_threshold = 5  # Percentage\nrow_threshold = 5     # Percentage\n\n# Calculate percentage of missing values for columns\nmissing_percent_columns = df_train.isnull().mean() * 100\n\n# Drop columns with missing values above the threshold\ncols_to_drop = missing_percent_columns[missing_percent_columns > column_threshold].index\ndf_train_reduced = df_train.drop(columns=cols_to_drop)\n\n# Calculate percentage of missing values for rows\nmissing_percent_rows = df_train_reduced.isnull().mean(axis=1) * 100\n\n# Drop rows with missing values above the threshold\nrows_to_drop = missing_percent_rows[missing_percent_rows > row_threshold].index\ndf_train_reduced = df_train_reduced.drop(index=rows_to_drop)\n\ndisplay(df_train_reduced)\n# Output the number of rows and columns in the reduced dataset\nprint(f\"Reduced dataset shape: {df_train_reduced.shape}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:30.041138Z","iopub.execute_input":"2024-08-24T08:40:30.041615Z","iopub.status.idle":"2024-08-24T08:40:30.120225Z","shell.execute_reply.started":"2024-08-24T08:40:30.041574Z","shell.execute_reply":"2024-08-24T08:40:30.118798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Missing values exist on all conditions and counts of them are listed below.\n\nspinal_canal_stenosis_l1_l2: 1\nspinal_canal_stenosis_l2_l3: 1\nspinal_canal_stenosis_l3_l4: 1\nspinal_canal_stenosis_l4_l5: 1\nspinal_canal_stenosis_l5_s1: 1\nleft_neural_foraminal_narrowing_l1_l2: 2\nleft_neural_foraminal_narrowing_l2_l3: 2\nleft_neural_foraminal_narrowing_l3_l4: 2\nleft_neural_foraminal_narrowing_l4_l5: 2\nleft_neural_foraminal_narrowing_l5_s1: 2\nright_neural_foraminal_narrowing_l1_l2: 8\nright_neural_foraminal_narrowing_l2_l3: 8\nright_neural_foraminal_narrowing_l3_l4: 8\nright_neural_foraminal_narrowing_l4_l5: 8\nright_neural_foraminal_narrowing_l5_s1: 8\nleft_subarticular_stenosis_l1_l2: 164\nleft_subarticular_stenosis_l2_l3: 82\nleft_subarticular_stenosis_l3_l4: 3\nleft_subarticular_stenosis_l4_l5: 3\nleft_subarticular_stenosis_l5_s1: 11\nright_subarticular_stenosis_l1_l2: 161\nright_subarticular_stenosis_l2_l3: 82\nright_subarticular_stenosis_l3_l4: 2\nright_subarticular_stenosis_l4_l5: 2\nright_subarticular_stenosis_l5_s1: 7\nThere are very few missing values on spinal canal stenosis (1) and neural foraminal narrowing (left 2 and right 8). They are consistently missing on all levels so incomplete labels may not be related to lowest vertebrae being not visible on those studies.\n\nThere are more missing values on subarticular stenosis compared to other two conditions and, missing value counts vary between levels. They decrease along with levels so it's not related to lowest vertebrae being not visible either. Missing value counts are consistent between left and right for this condition.","metadata":{}},{"cell_type":"code","source":"msno.matrix(df_train, figsize=(32, 24))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:58:52.606872Z","iopub.execute_input":"2024-08-24T08:58:52.607432Z","iopub.status.idle":"2024-08-24T08:58:54.009171Z","shell.execute_reply.started":"2024-08-24T08:58:52.607393Z","shell.execute_reply":"2024-08-24T08:58:54.007724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Create a mask of missing values\nmissing_data_mask = df_train.isnull()\n\n# Plot the missing values as a scatter plot\nplt.figure(figsize=(10, 6))\nfor col in df_train.columns:\n    missing_rows = missing_data_mask[col]\n    plt.scatter(missing_rows[missing_rows].index, [col] * missing_rows.sum(), label=col, alpha=0.6)\n\nplt.xlabel('Index')\nplt.ylabel('Columns with Missing Values')\nplt.title('Scatter Plot of Missing Values in Training Data')\nplt.xticks(rotation=45)\n#Place the legend below the plot\nplt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.15), ncol=3)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:31.691690Z","iopub.execute_input":"2024-08-24T08:40:31.692191Z","iopub.status.idle":"2024-08-24T08:40:32.963528Z","shell.execute_reply.started":"2024-08-24T08:40:31.692145Z","shell.execute_reply":"2024-08-24T08:40:32.962059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Select numeric columns\nnumeric_columns = df_train.select_dtypes(include=['float64', 'int64']).columns\n\n# Plot boxplots for each numeric column\nplt.figure(figsize=(15, 10))\ndf_train[numeric_columns].boxplot()\nplt.xticks(rotation=45)\nplt.title('Boxplot for Numeric Columns')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:32.965343Z","iopub.execute_input":"2024-08-24T08:40:32.965825Z","iopub.status.idle":"2024-08-24T08:40:33.253149Z","shell.execute_reply.started":"2024-08-24T08:40:32.965776Z","shell.execute_reply":"2024-08-24T08:40:33.251719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a DataFrame to hold the missing value information\nmissing_data_df = pd.DataFrame({\n    'Column': [col for col in df_train.columns for _ in range(df_train[col].isnull().sum())],\n    'Missing': [1] * df_train.isnull().sum().sum()\n})\n\n# Plot the count of missing values using countplot\nplt.figure(figsize=(12, 6))\nsns.countplot(data=missing_data_df, x='Column', palette='viridis', order=df_train.columns)\n\nplt.xlabel('Columns')\nplt.ylabel('Number of Missing Values')\nplt.title('Count Plot of Missing Values per Column')\nplt.xticks(rotation=45)\nplt.grid(axis='y')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:33.254566Z","iopub.execute_input":"2024-08-24T08:40:33.255103Z","iopub.status.idle":"2024-08-24T08:40:33.833614Z","shell.execute_reply.started":"2024-08-24T08:40:33.255066Z","shell.execute_reply":"2024-08-24T08:40:33.832338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Create a DataFrame to hold the missing value information\nmissing_data_df = pd.DataFrame({\n    'Column': [col for col in df_train.columns for _ in range(df_train[col].isnull().sum())],\n    'Missing': [1] * df_train.isnull().sum().sum(),\n    'Index': [i for col in df_train.columns for i in range(df_train[col].isnull().sum())]\n})\n\n# Create a FacetGrid to plot missing values\ng = sns.FacetGrid(missing_data_df, col='Column', col_wrap=5, height=4, sharey=False)\ng.map_dataframe(sns.histplot, x='Index', hue='Missing', multiple='stack', palette='viridis')\ng.set_axis_labels('Index', 'Count')\ng.set_titles(col_template=\"{col_name}\")\ng.add_legend(title='Missing Value')\n\nplt.suptitle('Histogram of Missing Values per Column', y=1.02)\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:33.835183Z","iopub.execute_input":"2024-08-24T08:40:33.835574Z","iopub.status.idle":"2024-08-24T08:40:46.694791Z","shell.execute_reply.started":"2024-08-24T08:40:33.835538Z","shell.execute_reply":"2024-08-24T08:40:46.693373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Calculate the number of missing values per column\nmissing_counts = df_train.isnull().sum()\n\n# Create a DataFrame from the missing counts\nmissing_counts_df = pd.DataFrame({\n    'Column': missing_counts.index,\n    'Missing Values': missing_counts.values\n})\n\n# Plot the boxplot of missing values\nplt.figure(figsize=(12, 6))\nsns.boxplot(x='Column', y='Missing Values', data=missing_counts_df, palette='viridis')\n\nplt.xlabel('Columns')\nplt.ylabel('Number of Missing Values')\nplt.title('Boxplot of Missing Values per Column')\nplt.xticks(rotation=45)\nplt.grid(axis='y')\n\n# Set the y-axis ticks to increment by 50\nplt.yticks(ticks=range(0, int(missing_counts.max()) + 50, 50))\n\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:46.696628Z","iopub.execute_input":"2024-08-24T08:40:46.697020Z","iopub.status.idle":"2024-08-24T08:40:47.433920Z","shell.execute_reply.started":"2024-08-24T08:40:46.696989Z","shell.execute_reply":"2024-08-24T08:40:47.432655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Calculate the number of missing values per column\nmissing_counts = df_train.isnull().sum()\n\n# Create a DataFrame from the missing counts\nmissing_counts_df = pd.DataFrame({\n    'Column': missing_counts.index,\n    'Missing Values': missing_counts.values\n})\n\n# Plot the line chart of missing values\nplt.figure(figsize=(12, 6))\nplt.plot(missing_counts_df['Column'], missing_counts_df['Missing Values'], marker='o', linestyle='-', color='b')\n\nplt.xlabel('Columns')\nplt.ylabel('Number of Missing Values')\nplt.title('Line Chart of Missing Values per Column')\nplt.xticks(rotation=45)\nplt.grid(True)\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:47.435841Z","iopub.execute_input":"2024-08-24T08:40:47.436377Z","iopub.status.idle":"2024-08-24T08:40:47.971168Z","shell.execute_reply.started":"2024-08-24T08:40:47.436330Z","shell.execute_reply":"2024-08-24T08:40:47.969574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Calculate the number of missing values per column\nmissing_counts = df_train.isnull().sum()\n\n# Create a DataFrame from the missing counts\nmissing_counts_df = pd.DataFrame({\n    'Column': missing_counts.index,\n    'Missing Values': missing_counts.values\n})\n\n# Plot the density plot of missing values\nplt.figure(figsize=(12, 6))\nsns.kdeplot(data=missing_counts_df, x='Missing Values', fill=True, color='darkblue', alpha=0.6)\n\nplt.xlabel('Number of Missing Values')\nplt.ylabel('Density')\nplt.title('Density Plot of Missing Values per Column')\nplt.grid(True)\n\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:47.973079Z","iopub.execute_input":"2024-08-24T08:40:47.973587Z","iopub.status.idle":"2024-08-24T08:40:48.349960Z","shell.execute_reply.started":"2024-08-24T08:40:47.973541Z","shell.execute_reply":"2024-08-24T08:40:48.348368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate the percentage of missing values for each column\nmissing_values = df_train.isnull().sum()\npercentage_missing = (missing_values / len(df_train)) * 100\n\n# Filter out columns with missing values\nmissing_percentage_df = percentage_missing[percentage_missing > 0]\n\n# Plot pie chart\nplt.figure(figsize=(10, 8))\nsizes = missing_percentage_df\nlabels = missing_percentage_df.index\ncolors = plt.cm.Paired(range(len(labels)))\n\n# Create the pie chart\nplt.pie(sizes, labels=labels, autopct='%1.1f%%', colors=colors)\n\n# Add a legend\nplt.legend(labels, title='Columns', loc='best', bbox_to_anchor=(1, 0.5))\n\nplt.title('Distribution of Missing Values by Column')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:48.351825Z","iopub.execute_input":"2024-08-24T08:40:48.352356Z","iopub.status.idle":"2024-08-24T08:40:49.120955Z","shell.execute_reply.started":"2024-08-24T08:40:48.352302Z","shell.execute_reply":"2024-08-24T08:40:49.119379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Identify non-numeric columns\nnon_numeric_cols = df_train.select_dtypes(include=['object']).columns\nprint(\"Non-numeric columns:\", non_numeric_cols)","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:49.123101Z","iopub.execute_input":"2024-08-24T08:40:49.123636Z","iopub.status.idle":"2024-08-24T08:40:49.132814Z","shell.execute_reply.started":"2024-08-24T08:40:49.123588Z","shell.execute_reply":"2024-08-24T08:40:49.131294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\n\n# Calculate the number of missing values per column\nmissing_counts = df_train.isnull().sum()\n\n# Create a DataFrame from the missing counts\nmissing_counts_df = pd.DataFrame({\n    'Column': missing_counts.index,\n    'Missing Values': missing_counts.values\n})\n\n# Create a new column for the 'Present' values\nmissing_counts_df['Present Values'] = df_train.shape[0] - missing_counts_df['Missing Values']\n\n# Set the column names to plot\ncolumns_to_plot = ['Missing Values', 'Present Values']\n\n# Plot the stacked bar plot\nplt.figure(figsize=(12, 8))\nmissing_counts_df.set_index('Column')[columns_to_plot].plot(kind='bar', stacked=True, color=['#FF6F61', '#6B5B95'])\n\nplt.xlabel('Columns')\nplt.ylabel('Count')\nplt.title('Stacked Bar Plot of Missing and Present Values per Column')\nplt.xticks(rotation=45)\nplt.legend(title='Data Type', labels=['Missing Values', 'Present Values'])\nplt.grid(axis='y')\n\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:49.134641Z","iopub.execute_input":"2024-08-24T08:40:49.135164Z","iopub.status.idle":"2024-08-24T08:40:49.759579Z","shell.execute_reply.started":"2024-08-24T08:40:49.135118Z","shell.execute_reply":"2024-08-24T08:40:49.758179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Statistics Summary**","metadata":{}},{"cell_type":"code","source":"# Summary statistics for numerical columns\ndf_train.describe()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:49.761705Z","iopub.execute_input":"2024-08-24T08:40:49.762135Z","iopub.status.idle":"2024-08-24T08:40:49.780233Z","shell.execute_reply.started":"2024-08-24T08:40:49.762100Z","shell.execute_reply":"2024-08-24T08:40:49.778918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from prettytable import PrettyTable\n\n# Initialize a PrettyTable\ntable = PrettyTable()\n\n# Combine all severity columns into a single dataframe for better visualization\nseverity_counts = pd.DataFrame()\ncols = df_train.columns[1:] # Exclude the first column (assuming it is an ID column)\nfor column in cols:\n    counts = df_train[column].value_counts().reset_index()\n    counts.columns = ['Severity', 'Count']\n    counts['Type'] = column\n    severity_counts = pd.concat([severity_counts, counts])\n\n# Displaying resutls in tabular form\ntable.field_names = [\"Severity\", \"Count\", \"Type\"]\n\n# Add rows to the table\nfor index, row in severity_counts.iterrows():\n    table.add_row([row['Severity'], row['Count'], row['Type']])\n\n# Print the table\nprint(table)","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:49.782147Z","iopub.execute_input":"2024-08-24T08:40:49.783252Z","iopub.status.idle":"2024-08-24T08:40:49.871437Z","shell.execute_reply.started":"2024-08-24T08:40:49.783189Z","shell.execute_reply":"2024-08-24T08:40:49.869867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from prettytable import PrettyTable\n\n# Initialize a PrettyTable\ntable = PrettyTable()\n\n# Combine all severity columns into a single dataframe for better visualization\nseverity_counts = pd.DataFrame()\ncols = df_train.columns[1:] # Exclude the first column (assuming it is an ID column)\nfor column in cols:\n    counts = df_train[column].value_counts().reset_index()\n    counts.columns = ['Severity', 'Count']\n    counts['Type'] = column\n    severity_counts = pd.concat([severity_counts, counts])\n\n# Displaying resutls in tabular form\ntable.field_names = [\"Severity\", \"Count\", \"Type\"]\n\n# Add rows to the table\nfor index, row in severity_counts.iterrows():\n    table.add_row([row['Severity'], row['Count'], row['Type']])\n\n# Print the table\nprint(table)","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:49.873175Z","iopub.execute_input":"2024-08-24T08:40:49.873609Z","iopub.status.idle":"2024-08-24T08:40:49.959662Z","shell.execute_reply.started":"2024-08-24T08:40:49.873573Z","shell.execute_reply":"2024-08-24T08:40:49.958334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Correlation Analysis**","metadata":{}},{"cell_type":"code","source":"# Copy the dataframe\ndf_encoded = df_train.copy()\n\n# Convert categorical columns to numerical codes\ndf_encoded = df_encoded.apply(lambda col: col.astype('category').cat.codes if col.dtype == 'object' else col)\n\n# Calculate the correlation matrix\ncorrelation_matrix = df_encoded.corr()\n\n# Create a heatmap for the correlation matrix\nplt.figure(figsize=(12, 10))\nsns.heatmap(data=correlation_matrix, annot=True, cmap='coolwarm', fmt='.2f')\nplt.title('Correlation Matrix')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:49.961485Z","iopub.execute_input":"2024-08-24T08:40:49.962087Z","iopub.status.idle":"2024-08-24T08:40:52.542161Z","shell.execute_reply.started":"2024-08-24T08:40:49.962048Z","shell.execute_reply":"2024-08-24T08:40:52.540876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pairplot for selected columns\nselected_columns = df_train.columns[1:10]  # Select a subset for a clearer visualization\nsns.pairplot(df_train[selected_columns].astype('category').apply(lambda x: x.cat.codes))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:40:52.543852Z","iopub.execute_input":"2024-08-24T08:40:52.544281Z","iopub.status.idle":"2024-08-24T08:41:19.654305Z","shell.execute_reply.started":"2024-08-24T08:40:52.544228Z","shell.execute_reply":"2024-08-24T08:41:19.652800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select a subset of columns for clearer visualization\nselected_columns = df_train.columns[1:10]  # Adjust the range as needed\n\n# Copy the dataframe and encode categorical columns\ndf_selected = df_train[selected_columns].copy()\ndf_selected_encoded = df_selected.apply(lambda col: col.astype('category').cat.codes if col.dtype == 'object' else col)\n\n# Melt the encoded DataFrame to long-form\ndf_melted = df_selected_encoded.reset_index().melt(id_vars='index')\ndf_melted.columns = ['Index', 'Variable', 'Value']\n\n# Plot violin plots for the selected columns\nplt.figure(figsize=(14, 8))\nsns.violinplot(data=df_melted, x='Variable', y='Value', palette='coolwarm')\nplt.title('Violin Plots of Selected Columns')\nplt.xticks(rotation=45)\nplt.grid(True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:41:19.655996Z","iopub.execute_input":"2024-08-24T08:41:19.656379Z","iopub.status.idle":"2024-08-24T08:41:20.363447Z","shell.execute_reply.started":"2024-08-24T08:41:19.656347Z","shell.execute_reply":"2024-08-24T08:41:20.361790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select a subset of columns for clearer visualization\nselected_columns = df_train.columns[1:10]  # Adjust the range as needed\n\n# Copy the dataframe and encode categorical columns\ndf_selected = df_train[selected_columns].copy()\ndf_selected_encoded = df_selected.apply(lambda col: col.astype('category').cat.codes if col.dtype == 'object' else col)\n\n# Plot histograms for the selected columns\nplt.figure(figsize=(16, 12))\nfor i, column in enumerate(df_selected_encoded.columns, 1):\n    plt.subplot(len(df_selected_encoded.columns)//3 + 1, 3, i)\n    sns.histplot(df_selected_encoded[column], kde=False, color='skyblue')\n    plt.title(column)\n    plt.xlabel('Value')\n    plt.ylabel('Frequency')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:41:20.365516Z","iopub.execute_input":"2024-08-24T08:41:20.366193Z","iopub.status.idle":"2024-08-24T08:41:23.074278Z","shell.execute_reply.started":"2024-08-24T08:41:20.366138Z","shell.execute_reply":"2024-08-24T08:41:23.072692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot box plots for the selected columns\nplt.figure(figsize=(16, 12))\nfor i, column in enumerate(df_selected_encoded.columns, 1):\n    plt.subplot(len(df_selected_encoded.columns)//3 + 1, 3, i)\n    sns.boxplot(y=df_selected_encoded[column], color='skyblue')\n    plt.title(column)\n    plt.xlabel('Column')\n    plt.ylabel('Value')\n\n    \nplt.suptitle('Box Plots of Selected Columns', fontsize=16)\n\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:41:23.076023Z","iopub.execute_input":"2024-08-24T08:41:23.076475Z","iopub.status.idle":"2024-08-24T08:41:25.402911Z","shell.execute_reply.started":"2024-08-24T08:41:23.076436Z","shell.execute_reply":"2024-08-24T08:41:25.401293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"To handle outliers I used the Interquartile Range (IQR) method, we can filter out data points that are outside a specified threshold. Below is the updated code that applies the IQR method to remove outliers before plotting the box plots","metadata":{}},{"cell_type":"code","source":"# Function to remove outliers based on IQR\ndef remove_outliers_iqr(col):\n    Q1 = col.quantile(0.25)\n    Q3 = col.quantile(0.75)\n    IQR = Q3 - Q1\n    lower_bound = Q1 - 1.5 * IQR\n    upper_bound = Q3 + 1.5 * IQR\n    return col[(col >= lower_bound) & (col <= upper_bound)]\n\n# Remove outliers for each selected column\ndf_no_outliers = df_selected_encoded.apply(remove_outliers_iqr)\n\n# Plot box plots for the selected columns without outliers\nplt.figure(figsize=(16, 12))\nfor i, column in enumerate(df_no_outliers.columns, 1):\n    plt.subplot(len(df_no_outliers.columns)//3 + 1, 3, i)\n    sns.boxplot(y=df_no_outliers[column], color='skyblue')\n    plt.title(column)\n    plt.xlabel('Column')\n    plt.ylabel('Value')\n    \nplt.suptitle('Box plots without outliers', fontsize=16)\n\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:41:25.404660Z","iopub.execute_input":"2024-08-24T08:41:25.405179Z","iopub.status.idle":"2024-08-24T08:41:27.177689Z","shell.execute_reply.started":"2024-08-24T08:41:25.405134Z","shell.execute_reply":"2024-08-24T08:41:27.176337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Copy the dataframe and encode categorical columns\ndf_selected = df_train[selected_columns].copy()\ndf_selected_encoded = df_selected.apply(lambda col: col.astype('category').cat.codes if col.dtype == 'object' else col)\n\n# Compute the correlation matrix\ncorrelation_matrix = df_selected_encoded.corr()\n\n# Plot the heatmap of the correlation matrix\nplt.figure(figsize=(12, 10))\nsns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', fmt='.2f', vmin=-1, vmax=1)\nplt.title('Correlation Matrix Heatmap for Selected Columns')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:41:27.179292Z","iopub.execute_input":"2024-08-24T08:41:27.179871Z","iopub.status.idle":"2024-08-24T08:41:27.893335Z","shell.execute_reply.started":"2024-08-24T08:41:27.179823Z","shell.execute_reply":"2024-08-24T08:41:27.892125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Model Trainint**","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport time\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport seaborn as sns\nimport pydicom as dicom\nimport pydicom\nimport json\nimport glob\nimport collections\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport cv2\nfrom skimage import measure\nfrom mpl_toolkits.mplot3d.art3d import Poly3DCollection\nimport plotly.graph_objects as go\nimport random\nfrom glob import glob\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:41:27.895045Z","iopub.execute_input":"2024-08-24T08:41:27.895437Z","iopub.status.idle":"2024-08-24T08:41:27.903071Z","shell.execute_reply.started":"2024-08-24T08:41:27.895401Z","shell.execute_reply":"2024-08-24T08:41:27.901892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_coordinates_df = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_label_coordinates.csv')\ntrain_series = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_series_descriptions.csv')\ndf_train = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv')\ndf_sub = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/sample_submission.csv')\ntest_series = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_series_descriptions.csv')","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:41:27.904644Z","iopub.execute_input":"2024-08-24T08:41:27.905044Z","iopub.status.idle":"2024-08-24T08:41:28.009669Z","shell.execute_reply.started":"2024-08-24T08:41:27.905003Z","shell.execute_reply":"2024-08-24T08:41:28.008257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_coordinates_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:41:28.011375Z","iopub.execute_input":"2024-08-24T08:41:28.011926Z","iopub.status.idle":"2024-08-24T08:41:28.027175Z","shell.execute_reply.started":"2024-08-24T08:41:28.011881Z","shell.execute_reply":"2024-08-24T08:41:28.025994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_coordinates_df.shape","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:41:28.028675Z","iopub.execute_input":"2024-08-24T08:41:28.029037Z","iopub.status.idle":"2024-08-24T08:41:28.041393Z","shell.execute_reply.started":"2024-08-24T08:41:28.029006Z","shell.execute_reply":"2024-08-24T08:41:28.040092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folder_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/1012284084'\ndicom_files = [f for f in os.listdir(folder_path) if f.endswith('.dcm')]","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:59:35.397473Z","iopub.execute_input":"2024-08-24T08:59:35.397939Z","iopub.status.idle":"2024-08-24T08:59:35.407078Z","shell.execute_reply.started":"2024-08-24T08:59:35.397901Z","shell.execute_reply":"2024-08-24T08:59:35.405704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folder_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/1012284084'\ndicom_files = [f for f in os.listdir(folder_path) if f.endswith('.dcm')]\nlabel_coordinates_df = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_label_coordinates.csv')\n\nstudy_id = folder_path.split('/')[-2]\nstudy_label_coordinates = label_coordinates_df[label_coordinates_df['study_id'] == int(study_id)]\nfiltered_dicom_files = []\nfiltered_label_coordinates = []\n\nfor dicom_file in dicom_files:\n    instance_number = int(dicom_file.split('.')[0])\n    corresponding_coordinates = study_label_coordinates[study_label_coordinates['instance_number'] == instance_number]\n    if not corresponding_coordinates.empty:\n        filtered_dicom_files.append(dicom_file)\n        filtered_label_coordinates.append(corresponding_coordinates)\nfig, axs = plt.subplots(1, 4, figsize=(20, 5))\nsecond_row_index = 1\nsecond_row_images = filtered_dicom_files[second_row_index : second_row_index + 4]\nsecond_row_coordinates = filtered_label_coordinates[second_row_index : second_row_index + 4]\n\nfor i, (dicom_file, label_coordinates) in enumerate(zip(second_row_images, second_row_coordinates)):\n    dicom_file_path = os.path.join(folder_path, dicom_file)\n    dicom_data = pydicom.dcmread(dicom_file_path)\n    image = dicom_data.pixel_array   \n    axs[i].imshow(image, cmap='gray')\n    axs[i].set_title(f'DICOM Image - {dicom_file}')\n    axs[i].axis('off')   \n    for _, row in label_coordinates.iterrows():\n        axs[i].plot(row['x'], row['y'], 'ro', markersize=5) \n        \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:59:44.347325Z","iopub.execute_input":"2024-08-24T08:59:44.347781Z","iopub.status.idle":"2024-08-24T08:59:45.410397Z","shell.execute_reply.started":"2024-08-24T08:59:44.347742Z","shell.execute_reply":"2024-08-24T08:59:45.408793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:41:28.070242Z","iopub.execute_input":"2024-08-24T08:41:28.070654Z","iopub.status.idle":"2024-08-24T08:41:28.104875Z","shell.execute_reply.started":"2024-08-24T08:41:28.070622Z","shell.execute_reply":"2024-08-24T08:41:28.103505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dicom(path):\n    dicom = pydicom.read_file(path)\n    data = dicom.pixel_array\n    data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data","metadata":{"execution":{"iopub.status.busy":"2024-08-24T09:02:22.402303Z","iopub.execute_input":"2024-08-24T09:02:22.402798Z","iopub.status.idle":"2024-08-24T09:02:22.412715Z","shell.execute_reply.started":"2024-08-24T09:02:22.402758Z","shell.execute_reply":"2024-08-24T09:02:22.411206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nfrom matplotlib import animation, rc\n\nrc('animation', html='jshtml')\n\ndef load_dicom(filename):\n    ds = pydicom.dcmread(filename)\n    return ds.pixel_array\n\ndef load_dicom_line(path):\n    t_paths = sorted(\n        glob.glob(os.path.join(path, \"*\")), \n        key=lambda x: int(os.path.splitext(os.path.basename(x))[0].split(\"-\")[-1]),\n    )\n    images = []\n    for filename in t_paths:\n        data = load_dicom(filename)\n        if data.max() == 0:\n            continue\n        images.append(data)\n    return images\n\ndef create_animation(ims):\n    fig = plt.figure(figsize=(6, 6))\n    plt.axis('off')\n    im = plt.imshow(ims[0], cmap=\"gray\")\n\n    def animate_func(i):\n        im.set_array(ims[i])\n        return [im]\n\n    return animation.FuncAnimation(fig, animate_func, frames=len(ims), interval=1000//24)\n\npath_to_folder = \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/1792451510\"\nimages = load_dicom_line(path_to_folder)\nanim = create_animation(images)\nanim","metadata":{"execution":{"iopub.status.busy":"2024-08-24T09:02:26.955937Z","iopub.execute_input":"2024-08-24T09:02:26.956384Z","iopub.status.idle":"2024-08-24T09:02:29.660236Z","shell.execute_reply.started":"2024-08-24T09:02:26.956350Z","shell.execute_reply":"2024-08-24T09:02:29.658872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_dicom_files(path_to_folder, num_files=5):\n    files_glob = os.path.join(path_to_folder, \"*.dcm\")\n    dicom_files = sorted(glob.glob(files_glob), key=lambda f: int(os.path.splitext(os.path.basename(f))[0].split('-')[-1]))\n    return [pydicom.dcmread(f) for f in dicom_files[:num_files]]\n\ndef calculate_level(mean, std):\n    return mean + 1.7 * std\n\ndef stats_image(image):\n    noncero_pixels = image[np.nonzero(image)]\n    if noncero_pixels.size == 0:\n        mean = 0\n        std = 0\n    else:\n        mean = np.mean(noncero_pixels)\n        std = np.std(noncero_pixels)\n    return mean, std\n\ndef image_orientation(dicom):\n    rt = 'unknown'\n    x1, y1, _, x2, y2, _ = [round(v) for v in dicom.ImageOrientationPatient]\n    if (x1, y1, x2, y2) == (1, 0, 0, 0):\n        rt = 'coronal'\n    elif (x1, y1, x2, y2) == (1, 0, 0, 1):\n        rt = 'axial'\n    elif (x1, y1, x2, y2) == (0, 1, 0, 0):\n        rt = 'sagittal'\n    if rt == 'unknown':\n        raise ValueError(f'unknown ImageOrientationPatient: {dicom.ImageOrientationPatient}')\n    return rt\n\ndef plot_image_hist(image):\n    mean, std = stats_image(image)\n    pixels = image.ravel()\n    noncero_pixels = pixels[np.nonzero(pixels)]\n    noncero_pixels = (noncero_pixels - mean) / std\n    over_threshold = np.count_nonzero(noncero_pixels > calculate_level(mean, std))\n    \n    fig, (axi, axh) = plt.subplots(1, 2, figsize=(20, 3), gridspec_kw={'width_ratios': [1, 4]})\n    fig.suptitle(f'scan # ({over_threshold})')\n    \n    axh.hist(noncero_pixels, 200, range=(-5, 5))  \n    axh.set_xlim(-5, 5)\n\n    ax_limits = axh.get_ylim()\n    axh.vlines(mean, ymin=ax_limits[0], ymax=ax_limits[1], colors='r', label='Mean')\n    axh.vlines(mean + std, ymin=ax_limits[0], ymax=ax_limits[1], colors='g', linestyles='dotted', label='Mean + Std')\n    axh.vlines(calculate_level(mean, std), ymin=ax_limits[0], ymax=ax_limits[1], colors='b', linestyles='dashed', label='Threshold')\n\n    axi.imshow(image, cmap=plt.cm.gray)\n    axi.grid(False)\n    axi.axis('off')\n    axh.legend()\n    plt.show()\n\npath_to_folder = \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/1002894806/866293114\"\ndicom_files = read_dicom_files(path_to_folder)\n\nif dicom_files:\n    first_image = dicom_files[0].pixel_array\n    plot_image_hist(first_image)\nelse:\n    print(\"No DICOM files found.\")","metadata":{"execution":{"iopub.status.busy":"2024-08-24T09:26:11.304703Z","iopub.execute_input":"2024-08-24T09:26:11.305214Z","iopub.status.idle":"2024-08-24T09:26:12.045884Z","shell.execute_reply.started":"2024-08-24T09:26:11.305175Z","shell.execute_reply":"2024-08-24T09:26:12.044348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import warnings\n# import matplotlib.pyplot as plt\n\n# # Suppress warnings within a context\n# with warnings.catch_warnings():\n#     warnings.simplefilter(\"ignore\", category=UserWarning)\n    \n#     # Example code that might generate warnings\n#     for img in flair_images[:5]:\n#         plot_image_hist(img)  # Replace with your actual plotting function\n\n# # Outside the context, warnings will be shown as usual","metadata":{"execution":{"iopub.status.busy":"2024-08-24T09:59:59.121566Z","iopub.execute_input":"2024-08-24T09:59:59.122650Z","iopub.status.idle":"2024-08-24T09:59:59.128892Z","shell.execute_reply.started":"2024-08-24T09:59:59.122597Z","shell.execute_reply":"2024-08-24T09:59:59.127189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dicom_files[1] ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dicom_images(folder_path, num_images=5):\n    dicom_files = sorted([os.path.join(folder_path, f) for f in os.listdir(folder_path) if f.endswith('.dcm')])[:num_images]\n    images = [pydicom.dcmread(f).pixel_array for f in dicom_files]\n    return np.stack(images, axis=-1)\n\ndef plot_3d(image, threshold=-300):\n\n    verts, faces, _, _ = measure.marching_cubes(image, level=threshold)  \n    \n    fig = plt.figure(figsize=(10, 10))\n    ax = fig.add_subplot(111, projection='3d')    \n    \n    mesh = Poly3DCollection(verts[faces], alpha=0.1)\n    face_color = [0.5, 0.5, 1]  # light blue\n    mesh.set_facecolor(face_color)\n    ax.add_collection3d(mesh)\n    \n    ax.set_xlim(0, image.shape[0])\n    ax.set_ylim(0, image.shape[1])\n    ax.set_zlim(0, image.shape[2])\n    \n    plt.show()\n\n\nfolder_path = \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/1012284084\"\ndicom_images = load_dicom_images(folder_path, num_images=5)\nplot_3d(dicom_images, threshold=300)","metadata":{"execution":{"iopub.status.busy":"2024-08-24T10:00:03.835298Z","iopub.execute_input":"2024-08-24T10:00:03.835775Z","iopub.status.idle":"2024-08-24T10:00:19.305821Z","shell.execute_reply.started":"2024-08-24T10:00:03.835738Z","shell.execute_reply":"2024-08-24T10:00:19.303974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def load_dicom_images(folder_path, num_images=5):\n#     dicom_files = sorted([os.path.join(folder_path, f) for f in os.listdir(folder_path) if f.endswith('.dcm')])[:num_images]\n#     images = [pydicom.dcmread(f).pixel_array for f in dicom_files]\n#     return np.stack(images, axis=-1)\n\n# def plot_3d_interactive(image, threshold=-300):    \n#     verts, faces, _, _ = measure.marching_cubes(image, level=threshold)   \n    \n#     fig = go.Figure(data=[\n#         go.Mesh3d(\n#             x=verts[:, 0],\n#             y=verts[:, 1],\n#             z=verts[:, 2],\n#             i=faces[:, 0],\n#             j=faces[:, 1],\n#             k=faces[:, 2],\n#             color='blue',\n#             opacity=0.1\n#         )\n#     ])\n    \n#     fig.update_layout(\n#         scene=dict(\n#             xaxis=dict(visible=True),\n#             yaxis=dict(visible=True),\n#             zaxis=dict(visible=True)\n#         ),\n#         width=800,\n#         height=800,\n#         title=\"Interactive 3D DICOM Image Visualization\"\n#     )\n    \n#     fig.show()\n\n\n# folder_path = \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/1002894806/866293114\"\n# dicom_images = load_dicom_images(folder_path, num_images=5)\n# plot_3d_interactive(dicom_images, threshold=100)","metadata":{"execution":{"iopub.status.busy":"2024-08-24T10:22:41.259549Z","iopub.execute_input":"2024-08-24T10:22:41.260158Z","iopub.status.idle":"2024-08-24T10:22:41.268913Z","shell.execute_reply.started":"2024-08-24T10:22:41.260112Z","shell.execute_reply":"2024-08-24T10:22:41.266991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n# import numpy as np\n# import pydicom\n# import plotly.graph_objects as go\n\n# def load_dicom_images(folder_path, num_images=5):\n#     dicom_files = sorted([os.path.join(folder_path, f) for f in os.listdir(folder_path) if f.endswith('.dcm')])[:num_images]\n#     images = [pydicom.dcmread(f).pixel_array for f in dicom_files]\n#     return np.stack(images, axis=-1)\n\n# def plot_3d_interactive(image, threshold=-300):\n#     x, y, z = np.indices(image.shape)\n#     # Apply threshold\n#     mask = image >= threshold\n#     x = x[mask]\n#     y = y[mask]\n#     z = z[mask]\n#     values = image[mask]\n    \n#     # Create the figure\n#     fig = go.Figure(data=go.Scatter3d(\n#         x=x.flatten(),\n#         y=y.flatten(),\n#         z=z.flatten(),\n#         mode='markers',\n#         marker=dict(\n#             size=2,\n#             color=values.flatten(),  # Color by intensity value\n#             colorscale='Viridis',\n#             colorbar=dict(title='Intensity'),\n#             opacity=0.8\n#         )\n#     ))\n    \n#     fig.update_layout(\n#         scene=dict(\n#             xaxis_title='X',\n#             yaxis_title='Y',\n#             zaxis_title='Z',\n#             aspectmode='cube'  # Keep the aspect ratio of the axes equal\n#         ),\n#         title='3D Interactive DICOM Image Visualization',\n#         margin=dict(l=0, r=0, b=0, t=40)  # Adjust margins for better fit\n#     )\n    \n#     fig.show()\n\n# # Example usage\n# folder_path = \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/1012284084\"\n# dicom_images = load_dicom_images(folder_path, num_images=5)\n# plot_3d_interactive(dicom_images, threshold=300)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T12:56:59.794889Z","iopub.execute_input":"2024-08-24T12:56:59.796437Z","iopub.status.idle":"2024-08-24T12:56:59.943277Z","shell.execute_reply.started":"2024-08-24T12:56:59.796388Z","shell.execute_reply":"2024-08-24T12:56:59.941793Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_image(image_path):\n    dicom = pydicom.dcmread(image_path)\n    pixel_array = dicom.pixel_array\n    image = (pixel_array - np.min(pixel_array)) / (np.max(pixel_array) - np.min(pixel_array)) * 255\n    image = image.astype(np.uint8)\n    return image\n\ndef plot_3d_image(image):   \n    fig = plt.figure(figsize=(10, 8))\n    ax = fig.add_subplot(111, projection='3d')\n    rows, cols = image.shape\n    x, y = np.meshgrid(np.arange(cols), np.arange(rows))\n    ax.plot_surface(x, y, image, cmap='viridis', edgecolor='none')\n    ax.set_xlabel('X')\n    ax.set_ylabel('Y')\n    ax.set_zlabel('Intensity')\n    ax.set_title('3D Plot of Image')\n    plt.show()\n\n# Correct path to a DICOM image\nexample_image_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/1012284084/1.dcm'\nif os.path.exists(example_image_path):\n    image = preprocess_image(example_image_path)\n    plot_3d_image(image)\nelse:\n    print(f\"File not found: {example_image_path}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T10:22:16.018402Z","iopub.execute_input":"2024-08-24T10:22:16.018849Z","iopub.status.idle":"2024-08-24T10:22:16.774336Z","shell.execute_reply.started":"2024-08-24T10:22:16.018816Z","shell.execute_reply":"2024-08-24T10:22:16.772836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from mpl_toolkits.mplot3d import Axes3D\n\n# def plot_3d_image(image):   \n#     fig = plt.figure(figsize=(10, 8))\n#     ax = fig.add_subplot(111, projection='3d')\n#     rows, cols = image.shape\n#     x, y = np.meshgrid(np.arange(cols), np.arange(rows))\n#     ax.plot_surface(x, y, image, cmap='viridis', edgecolor='none')\n#     ax.set_xlabel('X')\n#     ax.set_ylabel('Y')\n#     ax.set_zlabel('Intensity')\n#     ax.set_title('3D Plot of Image')\n#     plt.show()\n    \n# plot_3d_image(flair_images[0])","metadata":{"execution":{"iopub.status.busy":"2024-08-24T10:22:55.273187Z","iopub.execute_input":"2024-08-24T10:22:55.273745Z","iopub.status.idle":"2024-08-24T10:22:55.281916Z","shell.execute_reply.started":"2024-08-24T10:22:55.273697Z","shell.execute_reply":"2024-08-24T10:22:55.280164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T09:53:43.800571Z","iopub.execute_input":"2024-08-24T09:53:43.801058Z","iopub.status.idle":"2024-08-24T09:53:43.819917Z","shell.execute_reply.started":"2024-08-24T09:53:43.801017Z","shell.execute_reply":"2024-08-24T09:53:43.818210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub.shape,test_series.shape,train_series.shape,label_coordinates_df.shape,df_train.shape\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T09:53:47.573343Z","iopub.execute_input":"2024-08-24T09:53:47.574626Z","iopub.status.idle":"2024-08-24T09:53:47.583775Z","shell.execute_reply.started":"2024-08-24T09:53:47.574575Z","shell.execute_reply":"2024-08-24T09:53:47.582308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_series.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:41:31.466347Z","iopub.execute_input":"2024-08-24T08:41:31.466812Z","iopub.status.idle":"2024-08-24T08:41:31.481499Z","shell.execute_reply.started":"2024-08-24T08:41:31.466775Z","shell.execute_reply":"2024-08-24T08:41:31.480236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-24T08:41:31.482996Z","iopub.execute_input":"2024-08-24T08:41:31.483491Z","iopub.status.idle":"2024-08-24T08:41:31.500103Z","shell.execute_reply.started":"2024-08-24T08:41:31.483446Z","shell.execute_reply":"2024-08-24T08:41:31.498824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nfrom sklearn.ensemble import RandomForestClassifier\n\n\ndf_train_melted = df_train.melt(id_vars=['study_id'], var_name='condition_level', value_name='severity')\ndf_train_melted[['condition', 'level']] = df_train_melted['condition_level'].str.rsplit('_', n=1, expand=True)\nle_severity = LabelEncoder()\ndf_train_melted['severity_encoded'] = le_severity.fit_transform(df_train_melted['severity'])\nX_train = df_train_melted[['study_id', 'condition', 'level']]\ny_train = df_train_melted['severity_encoded']\nX_train = pd.get_dummies(X_train, columns=['condition', 'level'])\ntest_rows = []\nfor _, row in test_series.iterrows():\n    for condition in ['left_neural_foraminal_narrowing', 'right_neural_foraminal_narrowing', 'left_subarticular_stenosis', 'right_subarticular_stenosis', 'spinal_canal_stenosis']:\n        for level in ['l1_l2', 'l2_l3', 'l3_l4', 'l4_l5', 'l5_s1']:\n            test_rows.append({\n                'study_id': row['study_id'],\n                'condition': condition,\n                'level': level\n            })\n\nX_test = pd.DataFrame(test_rows)\nX_test = pd.get_dummies(X_test, columns=['condition', 'level'])\nX_test = X_test.reindex(columns=X_train.columns, fill_value=0)","metadata":{"execution":{"iopub.status.busy":"2024-08-24T10:23:03.538580Z","iopub.execute_input":"2024-08-24T10:23:03.539028Z","iopub.status.idle":"2024-08-24T10:23:03.731017Z","shell.execute_reply.started":"2024-08-24T10:23:03.538992Z","shell.execute_reply":"2024-08-24T10:23:03.729765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train","metadata":{"execution":{"iopub.status.busy":"2024-08-24T10:23:10.081091Z","iopub.execute_input":"2024-08-24T10:23:10.081555Z","iopub.status.idle":"2024-08-24T10:23:10.122797Z","shell.execute_reply.started":"2024-08-24T10:23:10.081512Z","shell.execute_reply":"2024-08-24T10:23:10.121226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = RandomForestClassifier()\nmodel.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2024-08-24T10:25:51.114329Z","iopub.execute_input":"2024-08-24T10:25:51.114890Z","iopub.status.idle":"2024-08-24T10:26:01.071332Z","shell.execute_reply.started":"2024-08-24T10:25:51.114849Z","shell.execute_reply":"2024-08-24T10:26:01.069749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions_proba = model.predict_proba(X_test)\npredictions_df = pd.DataFrame(predictions_proba, columns=le_severity.classes_)\npredictions_df['study_id'] = X_test['study_id'].values\npredictions_df['condition_level'] = X_test.index.map(lambda idx: f\"{test_rows[idx]['condition']}_{test_rows[idx]['level']}\")\npredictions_df['row_id'] = predictions_df['study_id'].astype(str) + '_' + predictions_df['condition_level']\ndf_sub1 = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/sample_submission.csv') \nnormal_mild_value = (predictions_df['Normal/Mild'].iloc[0])\nmoderate_value = (predictions_df['Moderate'].iloc[0])\nsevere_value = (predictions_df['Severe'].iloc[0])\n\ndf_sub['normal_mild'] = normal_mild_value/2.3\ndf_sub['moderate'] =moderate_value*1.47\ndf_sub['severe'] =  (1-(normal_mild_value/2.3 +moderate_value*1.47))+severe_value-severe_value\ndf_sub.sample(4)","metadata":{"execution":{"iopub.status.busy":"2024-08-24T10:26:01.074761Z","iopub.execute_input":"2024-08-24T10:26:01.075347Z","iopub.status.idle":"2024-08-24T10:26:01.122999Z","shell.execute_reply.started":"2024-08-24T10:26:01.075293Z","shell.execute_reply":"2024-08-24T10:26:01.121037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(14, 8))\nplt.plot(df_sub['row_id'], df_sub['normal_mild'], label='Normal/Mild', marker='o')\nplt.plot(df_sub['row_id'], df_sub['moderate'], label='Moderate', marker='o')\nplt.plot(df_sub['row_id'], df_sub['severe'], label='Severe', marker='o')\n\nplt.xlabel('Conditions')\nplt.ylabel('Values')\nplt.title('Normal/Mild, Moderate, and Severe Values for Different Conditions')\nplt.xticks(rotation=90)\nplt.legend()\n\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T10:26:08.930657Z","iopub.execute_input":"2024-08-24T10:26:08.931284Z","iopub.status.idle":"2024-08-24T10:26:09.629591Z","shell.execute_reply.started":"2024-08-24T10:26:08.931217Z","shell.execute_reply":"2024-08-24T10:26:09.628288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Dwnloading model***","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# import pickle\n\n# Save the model\n# model_file_path = 'trained_model.pkl'\n# with open(model_file_path, 'wb') as file:\n#     pickle.dump(model, file)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import joblib\n\n# Save the model directly to /kaggle/working/\n# model_file_path = '/kaggle/working/trained_model.pkl'\n# joblib.dump(model, model_file_path)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T10:26:31.141859Z","iopub.execute_input":"2024-08-24T10:26:31.143230Z","iopub.status.idle":"2024-08-24T10:26:31.148546Z","shell.execute_reply.started":"2024-08-24T10:26:31.143181Z","shell.execute_reply":"2024-08-24T10:26:31.147184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pydicom\nimport os\n\ndef preprocess_image(image_path):\n    # Load the DICOM image\n    dicom = pydicom.dcmread(image_path)\n    pixel_array = dicom.pixel_array\n\n    # Assuming preprocessing steps similar to training\n    image = (pixel_array - np.min(pixel_array)) / (np.max(pixel_array) - np.min(pixel_array)) * 255\n    image = image.astype(np.uint8)\n\n    return image, dicom.StudyInstanceUID\n\n# Example path to the new DICOM image\nnew_image_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/1012284084/11.dcm'\nnew_image, study_id = preprocess_image(new_image_path)\n\n# List of possible conditions and levels (based on your model's training data)\nconditions = [\n    'left_neural_foraminal_narrowing', 'right_neural_foraminal_narrowing',\n    'left_subarticular_stenosis', 'right_subarticular_stenosis', 'spinal_canal_stenosis'\n]\nlevels = ['l1_l2', 'l2_l3', 'l3_l4', 'l4_l5', 'l5_s1']\n\n# Create a DataFrame for the new data\ntest_rows = []\nfor condition in conditions:\n    for level in levels:\n        test_rows.append({\n            'study_id': study_id,\n            'condition': condition,\n            'level': level\n        })\n\nnew_data = pd.DataFrame(test_rows)\n\n# One-hot encode the condition and level columns\nnew_data_encoded = pd.get_dummies(new_data, columns=['condition', 'level'])\n\n# Reindex to match the training data's columns\nnew_data_encoded = new_data_encoded.reindex(columns=X_train.columns, fill_value=0)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T10:26:34.745962Z","iopub.execute_input":"2024-08-24T10:26:34.746431Z","iopub.status.idle":"2024-08-24T10:26:34.779683Z","shell.execute_reply.started":"2024-08-24T10:26:34.746394Z","shell.execute_reply":"2024-08-24T10:26:34.778077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predict using the loaded model\npredictions_proba = model.predict_proba(new_data_encoded)\npredictions_df = pd.DataFrame(predictions_proba, columns=le_severity.classes_)\n\n# Add the `condition` and `level` back to the DataFrame for easier interpretation\npredictions_df['condition'] = new_data['condition']\npredictions_df['level'] = new_data['level']\npredictions_df['study_id'] = study_id\n\n# Optionally, get the predicted classes\npredicted_classes = model.predict(new_data_encoded)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T10:26:43.849501Z","iopub.execute_input":"2024-08-24T10:26:43.850030Z","iopub.status.idle":"2024-08-24T10:26:43.882765Z","shell.execute_reply.started":"2024-08-24T10:26:43.849994Z","shell.execute_reply":"2024-08-24T10:26:43.881410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# print(predictions_df)\nprint(\"Predicted Classes:\", predicted_classes)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T10:26:48.722032Z","iopub.execute_input":"2024-08-24T10:26:48.723228Z","iopub.status.idle":"2024-08-24T10:26:48.737167Z","shell.execute_reply.started":"2024-08-24T10:26:48.723187Z","shell.execute_reply":"2024-08-24T10:26:48.735685Z"}}},{"cell_type":"code","source":"# import os\n# import pydicom\n# import numpy as np\n# import pandas as pd\n# from sklearn.model_selection import train_test_split, cross_val_score\n# from sklearn.ensemble import RandomForestClassifier\n# from sklearn.preprocessing import LabelEncoder\n# from sklearn.metrics import accuracy_score\n# import joblib\n# from tqdm import tqdm\n\n# def preprocess_image(image_path):\n#     dicom = pydicom.dcmread(image_path)\n#     pixel_array = dicom.pixel_array\n#     image = (pixel_array - np.min(pixel_array)) / (np.max(pixel_array) - np.min(pixel_array)) * 255\n#     image = image.astype(np.uint8)\n#     return image\n\n# # Gather all images and associated metadata\n# data = []\n# base_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images'\n\n# print(\"Loading images...\")\n# for study_id in tqdm(os.listdir(base_path)):\n#     study_path = os.path.join(base_path, study_id)\n#     for subfolder in os.listdir(study_path):\n#         subfolder_path = os.path.join(study_path, subfolder)\n#         for image_name in os.listdir(subfolder_path):\n#             image_path = os.path.join(subfolder_path, image_name)\n#             condition = subfolder  # Assuming the subfolder name is the condition\n#             level = image_name.split('.')[0]  # Assuming the image name contains the level information\n#             image = preprocess_image(image_path)\n#             data.append({\n#                 'study_id': study_id,\n#                 'condition': condition,\n#                 'level': level,\n#                 'image': image\n#             })\n\n# print(\"Converting data to DataFrame...\")\n# df = pd.DataFrame(data)\n\n# # Encode labels\n# le_condition = LabelEncoder()\n# df['condition_encoded'] = le_condition.fit_transform(df['condition'])\n\n# # Prepare features and labels\n# X = np.stack(df['image'].values)\n# y = df['condition_encoded'].values\n\n# # Train/test split\n# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# # Model Training with Progress Display\n# model = RandomForestClassifier()\n\n# print(\"Training model...\")\n# model.fit(X_train, y_train)\n\n# # Calculate accuracy on the test set\n# y_pred = model.predict(X_test)\n# accuracy = accuracy_score(y_test, y_pred)\n# print(f\"Test set accuracy: {accuracy:.4f}\")","metadata":{"execution":{"iopub.status.busy":"2024-08-24T13:45:32.427180Z","iopub.execute_input":"2024-08-24T13:45:32.428330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pydicom\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport cv2  # OpenCV library for image resizing\nimport joblib\n\ndef preprocess_image(image_path, target_shape=(128, 128)):\n    dicom = pydicom.dcmread(image_path)\n    pixel_array = dicom.pixel_array\n    \n    # Resize image to the target shape\n    image_resized = cv2.resize(pixel_array, target_shape, interpolation=cv2.INTER_AREA)\n    \n    # Normalize the image\n    image_normalized = (image_resized - np.min(image_resized)) / (np.max(image_resized) - np.min(image_resized)) * 255\n    image_normalized = image_normalized.astype(np.uint8)\n    return image_normalized\n\n# Gather all images and associated metadata\ndata = []\nbase_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images'\n\n# Get list of study ID folders and calculate half of them\nstudy_ids = os.listdir(base_path)\nnum_study_ids = len(study_ids)\nhalf_study_ids = study_ids[:num_study_ids // 2]\n\nprint(f\"Loading images from the first half of study ID folders (total: {len(half_study_ids)})...\")\nfor study_id in tqdm(half_study_ids):\n    study_path = os.path.join(base_path, study_id)\n    for subfolder in os.listdir(study_path):\n        subfolder_path = os.path.join(study_path, subfolder)\n        for image_name in os.listdir(subfolder_path):\n            image_path = os.path.join(subfolder_path, image_name)\n            condition = subfolder  # Assuming the subfolder name is the condition\n            level = image_name.split('.')[0]  # Assuming the image name contains the level information\n            image = preprocess_image(image_path)\n            data.append({\n                'study_id': study_id,\n                'condition': condition,\n                'level': level,\n                'image': image\n            })\n\nprint(\"Converting data to DataFrame...\")\ndf = pd.DataFrame(data)\n\n# Save preprocessed data to a file\njoblib.dump(df, '/kaggle/working/preprocessed_data.pkl')\nprint(\"Preprocessed data saved successfully.\")\n","metadata":{"execution":{"iopub.status.busy":"2024-08-24T19:21:34.827676Z","iopub.execute_input":"2024-08-24T19:21:34.828513Z","iopub.status.idle":"2024-08-24T19:51:48.368040Z","shell.execute_reply.started":"2024-08-24T19:21:34.828455Z","shell.execute_reply":"2024-08-24T19:51:48.366800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}