{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sea\nimport matplotlib.pyplot as plt\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2024-05-22T19:44:32.351676Z","iopub.execute_input":"2024-05-22T19:44:32.352415Z","iopub.status.idle":"2024-05-22T19:44:35.396132Z","shell.execute_reply.started":"2024-05-22T19:44:32.352357Z","shell.execute_reply":"2024-05-22T19:44:35.394630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT = \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification\"\n\ntrain = pd.read_csv(f\"{ROOT}/train.csv\")\ntrain_coord = pd.read_csv(f\"{ROOT}/train_label_coordinates.csv\")\ntrain_desc = pd.read_csv(f\"{ROOT}/train_series_descriptions.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-05-22T19:44:35.398334Z","iopub.execute_input":"2024-05-22T19:44:35.399085Z","iopub.status.idle":"2024-05-22T19:44:35.587519Z","shell.execute_reply.started":"2024-05-22T19:44:35.399048Z","shell.execute_reply":"2024-05-22T19:44:35.586001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Train Data Dimensions: {train.shape}\\n\")\nprint(f\"Train Label Coordinates Data Dimensions: {train_coord.shape}\\n\")\nprint(f\"Train Description Data Dimensions: {train_desc.shape}\\n\")","metadata":{"execution":{"iopub.status.busy":"2024-05-22T19:44:35.596025Z","iopub.execute_input":"2024-05-22T19:44:35.596520Z","iopub.status.idle":"2024-05-22T19:44:35.606586Z","shell.execute_reply.started":"2024-05-22T19:44:35.596478Z","shell.execute_reply":"2024-05-22T19:44:35.604900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Data Exploration\n\n\n
    \n
  1. Missing values present\n
  2. Higher levels tend to have higher proportions of moderate and severe cases for all conditions; this trend
    \n only extends to level L4/L5, except in NFN\n
  3. There is a drastic increase in severe cases for level L4/L5 in Subarticular Stenosis (SS), making the
    \n distribution almost uniform\n
  4. Left and Right Distributions for both NFN and SS display similar patterns for their specific classes\n
\n","metadata":{}},{"cell_type":"code","source":"train.head(4)","metadata":{"execution":{"iopub.status.busy":"2024-05-22T19:44:35.608321Z","iopub.execute_input":"2024-05-22T19:44:35.608784Z","iopub.status.idle":"2024-05-22T19:44:35.656679Z","shell.execute_reply.started":"2024-05-22T19:44:35.608736Z","shell.execute_reply":"2024-05-22T19:44:35.655079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isna().sum().sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-22T19:44:35.659375Z","iopub.execute_input":"2024-05-22T19:44:35.660549Z","iopub.status.idle":"2024-05-22T19:44:35.689038Z","shell.execute_reply.started":"2024-05-22T19:44:35.660475Z","shell.execute_reply":"2024-05-22T19:44:35.687560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spinal_conditions = {\"spinal_canal_stenosis\":['spinal_canal_stenosis_l1_l2',\n 'spinal_canal_stenosis_l2_l3', \n 'spinal_canal_stenosis_l3_l4',\n 'spinal_canal_stenosis_l4_l5', \n 'spinal_canal_stenosis_l5_s1'],\n \n \"neural_foraminal_narrowing\":[ # Left\n 'left_neural_foraminal_narrowing_l1_l2',\n 'left_neural_foraminal_narrowing_l2_l3',\n 'left_neural_foraminal_narrowing_l3_l4',\n 'left_neural_foraminal_narrowing_l4_l5',\n 'left_neural_foraminal_narrowing_l5_s1',\n # Right\n 'right_neural_foraminal_narrowing_l1_l2',\n 'right_neural_foraminal_narrowing_l2_l3',\n 'right_neural_foraminal_narrowing_l3_l4',\n 'right_neural_foraminal_narrowing_l4_l5',\n 'right_neural_foraminal_narrowing_l5_s1'],\n \n \"subarticular_stenosis\":[ # Left\n 'left_subarticular_stenosis_l1_l2',\n 'left_subarticular_stenosis_l2_l3',\n 'left_subarticular_stenosis_l3_l4', \n 'left_subarticular_stenosis_l4_l5',\n 'left_subarticular_stenosis_l5_s1', \n # Right\n 'right_subarticular_stenosis_l1_l2',\n 'right_subarticular_stenosis_l2_l3',\n 'right_subarticular_stenosis_l3_l4',\n 'right_subarticular_stenosis_l4_l5',\n 'right_subarticular_stenosis_l5_s1']}\n\nscs_df = train[spinal_conditions[\"spinal_canal_stenosis\"]]\nnfn_df = train[spinal_conditions[\"neural_foraminal_narrowing\"]]\nss_df = train[spinal_conditions[\"subarticular_stenosis\"]]","metadata":{"execution":{"iopub.status.busy":"2024-05-22T19:44:35.691421Z","iopub.execute_input":"2024-05-22T19:44:35.692413Z","iopub.status.idle":"2024-05-22T19:44:35.711385Z","shell.execute_reply.started":"2024-05-22T19:44:35.692361Z","shell.execute_reply":"2024-05-22T19:44:35.709854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Explores spinal_canal_stenosis\nfig,axes = plt.subplots(nrows=1, ncols=5, figsize=(16, 5))\naxes = axes.ravel()\n\nfor k,feature in enumerate(scs_df.columns):\n counts = scs_df[feature].value_counts()\n props = counts/counts.values.sum()\n props = props.reset_index()\n sea.barplot(data=props, x=feature, y=\"count\", \n order=[\"Normal/Mild\", \"Moderate\", \"Severe\"], \n ax=axes[k])\n axes[k].set_title(f\"SCS for level {feature[-5:-3]}/{feature[-2:]}\")\n axes[k].set_xlabel(None)\n axes[k].set_ylabel(None)\n axes[k].set_ylim((0,1))\n axes[k].set_xticklabels(axes[k].get_xticklabels(), rotation=45)\n \naxes[0].set_ylabel(\"Proportion\")\n \nfig.suptitle(\"Spinal Canal Stenosis (SCS) Proportions in Severity Across Different levels\")\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-22T19:57:49.640177Z","iopub.execute_input":"2024-05-22T19:57:49.640683Z","iopub.status.idle":"2024-05-22T19:57:50.546213Z","shell.execute_reply.started":"2024-05-22T19:57:49.640646Z","shell.execute_reply":"2024-05-22T19:57:50.544856Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Explores neural_foraminal_narrowing\nfig,axes = plt.subplots(nrows=2, ncols=5, figsize=(18, 9))\naxes = axes.ravel()\n\nfor k,feature in enumerate(nfn_df.columns):\n counts = nfn_df[feature].value_counts()\n props = counts/counts.values.sum()\n props = props.reset_index()\n sea.barplot(data=props, x=feature, y=\"count\", \n order=[\"Normal/Mild\", \"Moderate\", \"Severe\"], \n ax=axes[k])\n \n if k<5: position = \"Left\"\n else: position = \"Right\"\n \n axes[k].set_title(f\"{position} NFN for level {feature[-5:-3]}/{feature[-2:]}\")\n axes[k].set_xlabel(None)\n axes[k].set_ylabel(None)\n axes[k].set_ylim((0,1))\n if k%5 == 0:\n axes[k].set_ylabel(\"Proportion\")\n \nfig.suptitle(\"Neural Foraminal Narrowing (NFN) Proportions in Severity Across Different levels\")\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-22T20:12:40.023036Z","iopub.execute_input":"2024-05-22T20:12:40.023464Z","iopub.status.idle":"2024-05-22T20:12:43.057187Z","shell.execute_reply.started":"2024-05-22T20:12:40.023432Z","shell.execute_reply":"2024-05-22T20:12:43.056131Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Explores subarticular_stenosis\nfig,axes = plt.subplots(nrows=2, ncols=5, figsize=(18, 9))\naxes = axes.ravel()\n\nfor k,feature in enumerate(ss_df.columns):\n counts = ss_df[feature].value_counts()\n props = counts/counts.values.sum()\n props = props.reset_index()\n sea.barplot(data=props, x=feature, y=\"count\", \n order=[\"Normal/Mild\", \"Moderate\", \"Severe\"], \n ax=axes[k])\n \n if k<5: position = \"Left\"\n else: position = \"Right\"\n \n axes[k].set_title(f\"{position} SS for level {feature[-5:-3]}/{feature[-2:]}\")\n axes[k].set_xlabel(None)\n axes[k].set_ylim((0,1))\n axes[k].set_ylabel(None)\n if k%5 == 0:\n axes[k].set_ylabel(\"Proportion\")\n \nfig.suptitle(\"Subarticular Stenosis (SS) Proportions in Severity Across Different levels\")\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-22T20:13:04.442763Z","iopub.execute_input":"2024-05-22T20:13:04.444284Z","iopub.status.idle":"2024-05-22T20:13:06.471453Z","shell.execute_reply.started":"2024-05-22T20:13:04.444231Z","shell.execute_reply":"2024-05-22T20:13:06.469830Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Coordinates Data Exploration\n\n\n
    \n
  1. Theoretically, all study_id should have a frequency of 25 as each patient must have 5 recorded conditions mapped to 5 levels. This clearly is not the case, and may explain the NA values in train data above\n
  2. Positive linear relationship between x and y coordinates across various conditions and levels\n
\n","metadata":{}},{"cell_type":"code","source":"train_coord.head(4)","metadata":{"execution":{"iopub.status.busy":"2024-05-22T19:44:40.936021Z","iopub.execute_input":"2024-05-22T19:44:40.936382Z","iopub.status.idle":"2024-05-22T19:44:40.953601Z","shell.execute_reply.started":"2024-05-22T19:44:40.936350Z","shell.execute_reply":"2024-05-22T19:44:40.952230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(train_coord.study_id.value_counts().sort_values(ascending=False))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-22T19:44:40.955526Z","iopub.execute_input":"2024-05-22T19:44:40.956018Z","iopub.status.idle":"2024-05-22T19:44:40.982511Z","shell.execute_reply.started":"2024-05-22T19:44:40.955974Z","shell.execute_reply":"2024-05-22T19:44:40.981023Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Only appear 15 times; missing left and right NFN conditions\ntrain_coord.loc[train_coord.study_id==2492114990]","metadata":{"execution":{"iopub.status.busy":"2024-05-22T19:44:40.984552Z","iopub.execute_input":"2024-05-22T19:44:40.984948Z","iopub.status.idle":"2024-05-22T19:44:41.007288Z","shell.execute_reply.started":"2024-05-22T19:44:40.984913Z","shell.execute_reply":"2024-05-22T19:44:41.005592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sea.countplot(x=train_coord.study_id.value_counts().sort_values(ascending=False))\nplt.title(\"Frequencies of Study IDs\")\nplt.xlabel(\"Frequency\")\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-22T19:44:41.009902Z","iopub.execute_input":"2024-05-22T19:44:41.010442Z","iopub.status.idle":"2024-05-22T19:44:41.551904Z","shell.execute_reply.started":"2024-05-22T19:44:41.010386Z","shell.execute_reply":"2024-05-22T19:44:41.550676Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(train_coord.series_id.value_counts().sort_values(ascending=False))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-22T19:44:41.553539Z","iopub.execute_input":"2024-05-22T19:44:41.553950Z","iopub.status.idle":"2024-05-22T19:44:41.571299Z","shell.execute_reply.started":"2024-05-22T19:44:41.553916Z","shell.execute_reply":"2024-05-22T19:44:41.570183Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sea.countplot(x=train_coord.series_id.value_counts().sort_values(ascending=False))\nplt.title(\"Frequencies of Series IDs\")\nplt.xlabel(\"Frequency\")\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-22T19:44:41.572569Z","iopub.execute_input":"2024-05-22T19:44:41.573558Z","iopub.status.idle":"2024-05-22T19:44:41.887650Z","shell.execute_reply.started":"2024-05-22T19:44:41.573521Z","shell.execute_reply":"2024-05-22T19:44:41.886671Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(train_coord.instance_number.value_counts().sort_values(ascending=False))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-22T19:44:41.889113Z","iopub.execute_input":"2024-05-22T19:44:41.889660Z","iopub.status.idle":"2024-05-22T19:44:41.905405Z","shell.execute_reply.started":"2024-05-22T19:44:41.889627Z","shell.execute_reply":"2024-05-22T19:44:41.903829Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sea.histplot(train_coord.instance_number.value_counts().sort_values(ascending=False))\nplt.title(\"Frequencies of Instance Numbers\")\nplt.xlabel(\"Frequency\")\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-22T19:44:41.907410Z","iopub.execute_input":"2024-05-22T19:44:41.907903Z","iopub.status.idle":"2024-05-22T19:44:42.306406Z","shell.execute_reply.started":"2024-05-22T19:44:41.907824Z","shell.execute_reply":"2024-05-22T19:44:42.304818Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(18,6))\nplt.subplot(121)\nsea.histplot(data=train_coord, x=\"x\")\nplt.title(\"x-value Coordinate Distribution\")\nplt.xlabel(None)\n\nplt.subplot(122)\nsea.histplot(data=train_coord, x=\"y\")\nplt.title(\"y-value Coordinate Distribution\")\nplt.xlabel(None)\n\n\nplt.figure(figsize=(18,6))\nplt.subplot(121)\nsea.histplot( (train_coord.x - train_coord.x.mean())/train_coord.x.std() )\nplt.title(\"Standardized x-value Coordinate Distribution\")\nplt.xlabel(None)\n\nplt.subplot(122)\nsea.histplot( (train_coord.y - train_coord.y.mean())/train_coord.y.std() )\nplt.title(\"Standardized y-value Coordinate Distribution\")\nplt.xlabel(None)\n\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-22T19:44:42.308099Z","iopub.execute_input":"2024-05-22T19:44:42.308496Z","iopub.status.idle":"2024-05-22T19:44:44.422951Z","shell.execute_reply.started":"2024-05-22T19:44:42.308464Z","shell.execute_reply":"2024-05-22T19:44:44.421357Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_coord_plot = train_coord.copy()\ntrain_coord_plot.condition = train_coord_plot.condition.map({\n \"Spinal Canal Stenosis\":\"SCS\",\n \"Left Neural Foraminal Narrowing\":\"L_NFN\",\n \"Right Neural Foraminal Narrowing\":\"R_NFN\",\n \"Left Subarticular Stenosis\":\"L_SS\",\n \"Right Subarticular Stenosis\":\"R_SS\"}\n )\ntrain_coord_plot.rename(columns={\"condition\":\"cond\",\n \"level\":\"lvl\"},\n inplace=True)\n\ntrain_coord_plot.head(3)","metadata":{"execution":{"iopub.status.busy":"2024-05-22T19:44:44.425266Z","iopub.execute_input":"2024-05-22T19:44:44.425659Z","iopub.status.idle":"2024-05-22T19:44:44.457373Z","shell.execute_reply.started":"2024-05-22T19:44:44.425624Z","shell.execute_reply":"2024-05-22T19:44:44.455646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid = sea.FacetGrid(train_coord_plot, row=\"cond\", col=\"lvl\")\ngrid.map(sea.scatterplot, \"x\", \"y\")\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-22T19:44:44.458918Z","iopub.execute_input":"2024-05-22T19:44:44.459279Z","iopub.status.idle":"2024-05-22T19:44:54.478775Z","shell.execute_reply.started":"2024-05-22T19:44:44.459248Z","shell.execute_reply":"2024-05-22T19:44:54.475619Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Description Data Exploration","metadata":{}},{"cell_type":"code","source":"train_desc.head(4)","metadata":{"execution":{"iopub.status.busy":"2024-05-22T19:44:54.480542Z","iopub.execute_input":"2024-05-22T19:44:54.481013Z","iopub.status.idle":"2024-05-22T19:44:54.493978Z","shell.execute_reply.started":"2024-05-22T19:44:54.480975Z","shell.execute_reply":"2024-05-22T19:44:54.492919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_desc.study_id.value_counts().sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-22T19:44:54.496339Z","iopub.execute_input":"2024-05-22T19:44:54.496764Z","iopub.status.idle":"2024-05-22T19:44:54.530039Z","shell.execute_reply.started":"2024-05-22T19:44:54.496727Z","shell.execute_reply":"2024-05-22T19:44:54.528316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sea.countplot(x=train_desc.study_id.value_counts())\nplt.title(\"Frequencies of Study IDs\")\nplt.xlabel(\"Frequency\")\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-22T19:44:54.532092Z","iopub.execute_input":"2024-05-22T19:44:54.533397Z","iopub.status.idle":"2024-05-22T19:44:54.836718Z","shell.execute_reply.started":"2024-05-22T19:44:54.533350Z","shell.execute_reply":"2024-05-22T19:44:54.835591Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_desc.series_id.value_counts().sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-22T19:44:54.838320Z","iopub.execute_input":"2024-05-22T19:44:54.842194Z","iopub.status.idle":"2024-05-22T19:44:54.856083Z","shell.execute_reply.started":"2024-05-22T19:44:54.842147Z","shell.execute_reply":"2024-05-22T19:44:54.854773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sea.countplot(x=train_desc.series_id.value_counts())\nplt.title(\"Frequencies of Series IDs\")\nplt.xlabel(\"Frequency\")\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-22T19:44:54.858188Z","iopub.execute_input":"2024-05-22T19:44:54.858749Z","iopub.status.idle":"2024-05-22T19:44:55.110930Z","shell.execute_reply.started":"2024-05-22T19:44:54.858712Z","shell.execute_reply":"2024-05-22T19:44:55.109640Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_desc.series_description.value_counts().sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-22T19:44:55.112784Z","iopub.execute_input":"2024-05-22T19:44:55.113529Z","iopub.status.idle":"2024-05-22T19:44:55.125407Z","shell.execute_reply.started":"2024-05-22T19:44:55.113486Z","shell.execute_reply":"2024-05-22T19:44:55.123781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sea.countplot(x=train_desc.series_description, order=[\"Axial T2\", \"Sagittal T1\", \"Sagittal T2/STIR\"])\nplt.title(\"Frequencies of Series Descriptions\")\nplt.xlabel(None)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-22T19:44:55.130741Z","iopub.execute_input":"2024-05-22T19:44:55.131167Z","iopub.status.idle":"2024-05-22T19:44:55.388829Z","shell.execute_reply.started":"2024-05-22T19:44:55.131136Z","shell.execute_reply":"2024-05-22T19:44:55.387598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}