{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport keras\nfrom keras.layers import *\nfrom keras.models import *\nimport scipy\nimport os\nimport glob\nimport cv2\n\nimport PIL\nfrom tqdm import tqdm\nimport math, random\nfrom sklearn.model_selection import *\n\nfrom PIL import *\nimport sys\nimport warnings","metadata":{"execution":{"iopub.status.busy":"2024-07-05T15:14:05.931498Z","iopub.execute_input":"2024-07-05T15:14:05.931900Z","iopub.status.idle":"2024-07-05T15:14:23.826269Z","shell.execute_reply.started":"2024-07-05T15:14:05.931870Z","shell.execute_reply":"2024-07-05T15:14:23.825287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Data Description\n","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv')\n\ntrain_label = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_label_coordinates.csv')\n\ntrain_desp = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_series_descriptions.csv')","metadata":{"execution":{"iopub.status.busy":"2024-07-05T15:14:23.828413Z","iopub.execute_input":"2024-07-05T15:14:23.829219Z","iopub.status.idle":"2024-07-05T15:14:24.046214Z","shell.execute_reply.started":"2024-07-05T15:14:23.829178Z","shell.execute_reply":"2024-07-05T15:14:24.045080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_data.shape)\ntrain_data.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-05T15:14:24.047567Z","iopub.execute_input":"2024-07-05T15:14:24.047897Z","iopub.status.idle":"2024-07-05T15:14:24.092504Z","shell.execute_reply.started":"2024-07-05T15:14:24.047870Z","shell.execute_reply":"2024-07-05T15:14:24.091399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data['spinal_canal_stenosis_l1_l2'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-07-05T15:14:24.095399Z","iopub.execute_input":"2024-07-05T15:14:24.095798Z","iopub.status.idle":"2024-07-05T15:14:24.112218Z","shell.execute_reply.started":"2024-07-05T15:14:24.095763Z","shell.execute_reply":"2024-07-05T15:14:24.111091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_label.shape)\ntrain_label.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-05T15:14:24.114070Z","iopub.execute_input":"2024-07-05T15:14:24.114434Z","iopub.status.idle":"2024-07-05T15:14:24.131695Z","shell.execute_reply.started":"2024-07-05T15:14:24.114405Z","shell.execute_reply":"2024-07-05T15:14:24.130392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for column in train_label.columns[2:5]:\n    print(f'{column}=======================')\n    print(f'Number of unique values--------{train_label[column].nunique()}')\n    print(train_label[column].value_counts())","metadata":{"execution":{"iopub.status.busy":"2024-07-05T15:14:24.133310Z","iopub.execute_input":"2024-07-05T15:14:24.133764Z","iopub.status.idle":"2024-07-05T15:14:24.174695Z","shell.execute_reply.started":"2024-07-05T15:14:24.133723Z","shell.execute_reply":"2024-07-05T15:14:24.173552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_desp.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-05T15:14:24.176371Z","iopub.execute_input":"2024-07-05T15:14:24.176818Z","iopub.status.idle":"2024-07-05T15:14:24.189441Z","shell.execute_reply.started":"2024-07-05T15:14:24.176781Z","shell.execute_reply":"2024-07-05T15:14:24.188079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_desp['series_description'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-07-05T15:14:24.191080Z","iopub.execute_input":"2024-07-05T15:14:24.191547Z","iopub.status.idle":"2024-07-05T15:14:24.205205Z","shell.execute_reply.started":"2024-07-05T15:14:24.191506Z","shell.execute_reply":"2024-07-05T15:14:24.203874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_base_dir = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images'\ntest_base_dir = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_images'\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-07-05T15:14:24.207165Z","iopub.execute_input":"2024-07-05T15:14:24.207638Z","iopub.status.idle":"2024-07-05T15:14:24.218211Z","shell.execute_reply.started":"2024-07-05T15:14:24.207587Z","shell.execute_reply":"2024-07-05T15:14:24.216971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_file_path(base_dir):\n    df_path =[]\n    file_pattern = os.path.join(base_dir,'**', '*.dcm')\n    dcm_files = glob.glob(file_pattern, recursive=True)\n    print(len(dcm_files))\n    \n    for files in dcm_files:\n        path_df= pd.DataFrame({\n            'path': files,\n            'study_id':[int(os.path.basename(os.path.dirname(os.path.dirname(files))))],\n            'series_id':[int(os.path.basename(os.path.dirname(files)))],\n            'instance_number':[int(os.path.splitext(os.path.basename(files))[0])]\n            \n        })\n        df_path.append(path_df)\n    main_df = pd.concat(df_path).sort_values(by='study_id')\n    main_df = main_df.drop_duplicates()\n    main_df = main_df.reset_index(drop=True)\n        \n    return main_df\n    ","metadata":{"execution":{"iopub.status.busy":"2024-07-05T15:14:24.222972Z","iopub.execute_input":"2024-07-05T15:14:24.224057Z","iopub.status.idle":"2024-07-05T15:14:24.234029Z","shell.execute_reply.started":"2024-07-05T15:14:24.223988Z","shell.execute_reply":"2024-07-05T15:14:24.232574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = get_file_path(test_base_dir)\n\ntrain_df =get_file_path(train_base_dir)","metadata":{"execution":{"iopub.status.busy":"2024-07-05T15:14:24.235556Z","iopub.execute_input":"2024-07-05T15:14:24.235949Z","iopub.status.idle":"2024-07-05T15:16:24.343640Z","shell.execute_reply.started":"2024-07-05T15:14:24.235917Z","shell.execute_reply":"2024-07-05T15:16:24.342529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['study_id'].nunique(), train_df['series_id'].nunique(), train_label['study_id'].nunique(), train_label['series_id'].nunique()","metadata":{"execution":{"iopub.status.busy":"2024-07-05T15:16:24.384054Z","iopub.execute_input":"2024-07-05T15:16:24.384386Z","iopub.status.idle":"2024-07-05T15:16:24.397551Z","shell.execute_reply.started":"2024-07-05T15:16:24.384358Z","shell.execute_reply":"2024-07-05T15:16:24.396212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info(), train_df.columns","metadata":{"execution":{"iopub.status.busy":"2024-07-05T15:16:24.425206Z","iopub.execute_input":"2024-07-05T15:16:24.425553Z","iopub.status.idle":"2024-07-05T15:16:24.458927Z","shell.execute_reply.started":"2024-07-05T15:16:24.425525Z","shell.execute_reply":"2024-07-05T15:16:24.457955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_df['study_id'].unique())","metadata":{"execution":{"iopub.status.busy":"2024-07-05T15:49:11.617971Z","iopub.execute_input":"2024-07-05T15:49:11.618953Z","iopub.status.idle":"2024-07-05T15:49:11.628430Z","shell.execute_reply.started":"2024-07-05T15:49:11.618915Z","shell.execute_reply":"2024-07-05T15:49:11.627075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_data.columns)\nprint(train_label.columns)\nprint(train_desp.columns)","metadata":{"execution":{"iopub.status.busy":"2024-07-05T15:56:17.278505Z","iopub.execute_input":"2024-07-05T15:56:17.279133Z","iopub.status.idle":"2024-07-05T15:56:17.288093Z","shell.execute_reply.started":"2024-07-05T15:56:17.279080Z","shell.execute_reply":"2024-07-05T15:56:17.286484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merge_train = pd.merge(train_label, train_desp, on=['study_id', 'series_id',], how='inner')\nmerge_train_2 = pd.merge(merge_train, train_data, on=['study_id'], how='inner')\nmerge_train_2","metadata":{"execution":{"iopub.status.busy":"2024-07-05T16:07:39.162942Z","iopub.execute_input":"2024-07-05T16:07:39.163468Z","iopub.status.idle":"2024-07-05T16:07:39.280897Z","shell.execute_reply.started":"2024-07-05T16:07:39.163435Z","shell.execute_reply":"2024-07-05T16:07:39.279687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label[train_label['study_id']==3008676218]","metadata":{"execution":{"iopub.status.busy":"2024-07-05T16:07:39.365350Z","iopub.execute_input":"2024-07-05T16:07:39.366347Z","iopub.status.idle":"2024-07-05T16:07:39.378466Z","shell.execute_reply.started":"2024-07-05T16:07:39.366313Z","shell.execute_reply":"2024-07-05T16:07:39.376782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# merge_train = pd.merge(train_desp, train_data, on=['study_id'], how='inner')\n# merge_train_2 = pd.merge(merge_train, train_label, on=['study_id', 'series_id'], how='left')\n","metadata":{"execution":{"iopub.status.busy":"2024-07-05T16:07:39.473846Z","iopub.execute_input":"2024-07-05T16:07:39.476817Z","iopub.status.idle":"2024-07-05T16:07:39.481748Z","shell.execute_reply.started":"2024-07-05T16:07:39.476773Z","shell.execute_reply":"2024-07-05T16:07:39.480268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(merge_train_2['study_id'].nunique())\nmerge_train_2","metadata":{"execution":{"iopub.status.busy":"2024-07-05T16:07:40.667430Z","iopub.execute_input":"2024-07-05T16:07:40.667919Z","iopub.status.idle":"2024-07-05T16:07:40.741853Z","shell.execute_reply.started":"2024-07-05T16:07:40.667887Z","shell.execute_reply":"2024-07-05T16:07:40.740557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merge_train_2[merge_train_2['study_id']==3008676218]","metadata":{"execution":{"iopub.status.busy":"2024-07-05T16:07:40.968973Z","iopub.execute_input":"2024-07-05T16:07:40.970146Z","iopub.status.idle":"2024-07-05T16:07:40.987911Z","shell.execute_reply.started":"2024-07-05T16:07:40.970108Z","shell.execute_reply":"2024-07-05T16:07:40.986627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label_path = pd.merge(merge_train_2, train_df, on=['study_id', 'series_id', 'instance_number'], how='left')\ntrain_label_path","metadata":{"execution":{"iopub.status.busy":"2024-07-05T16:07:42.262092Z","iopub.execute_input":"2024-07-05T16:07:42.262498Z","iopub.status.idle":"2024-07-05T16:07:42.418109Z","shell.execute_reply.started":"2024-07-05T16:07:42.262471Z","shell.execute_reply":"2024-07-05T16:07:42.416972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label_path['study_id'].nunique()","metadata":{"execution":{"iopub.status.busy":"2024-07-05T16:07:48.364946Z","iopub.execute_input":"2024-07-05T16:07:48.366426Z","iopub.status.idle":"2024-07-05T16:07:48.375460Z","shell.execute_reply.started":"2024-07-05T16:07:48.366376Z","shell.execute_reply":"2024-07-05T16:07:48.374103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(set(train_desp['study_id'].unique()) - set(train_label['study_id'].unique()))","metadata":{"execution":{"iopub.status.busy":"2024-07-05T16:07:49.862225Z","iopub.execute_input":"2024-07-05T16:07:49.862653Z","iopub.status.idle":"2024-07-05T16:07:49.871400Z","shell.execute_reply.started":"2024-07-05T16:07:49.862622Z","shell.execute_reply":"2024-07-05T16:07:49.870230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_id_val = train_df[train_df['study_id']==3008676218]\nmissing_id_val","metadata":{"execution":{"iopub.status.busy":"2024-07-05T16:07:50.597968Z","iopub.execute_input":"2024-07-05T16:07:50.598399Z","iopub.status.idle":"2024-07-05T16:07:50.614240Z","shell.execute_reply.started":"2024-07-05T16:07:50.598369Z","shell.execute_reply":"2024-07-05T16:07:50.612935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_id_val['series_id'].unique()","metadata":{"execution":{"iopub.status.busy":"2024-07-05T16:07:55.066780Z","iopub.execute_input":"2024-07-05T16:07:55.067208Z","iopub.status.idle":"2024-07-05T16:07:55.077323Z","shell.execute_reply.started":"2024-07-05T16:07:55.067177Z","shell.execute_reply":"2024-07-05T16:07:55.076067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_desp[train_desp['study_id']==3008676218]","metadata":{"execution":{"iopub.status.busy":"2024-07-05T16:08:24.070684Z","iopub.execute_input":"2024-07-05T16:08:24.071124Z","iopub.status.idle":"2024-07-05T16:08:24.084674Z","shell.execute_reply.started":"2024-07-05T16:08:24.071093Z","shell.execute_reply":"2024-07-05T16:08:24.083235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label_path.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2024-07-05T16:08:46.262398Z","iopub.execute_input":"2024-07-05T16:08:46.262876Z","iopub.status.idle":"2024-07-05T16:08:46.426900Z","shell.execute_reply.started":"2024-07-05T16:08:46.262844Z","shell.execute_reply":"2024-07-05T16:08:46.425604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label_path = train_label_path.fillna(-100)","metadata":{"execution":{"iopub.status.busy":"2024-07-05T16:09:38.300162Z","iopub.execute_input":"2024-07-05T16:09:38.300676Z","iopub.status.idle":"2024-07-05T16:09:38.504549Z","shell.execute_reply.started":"2024-07-05T16:09:38.300644Z","shell.execute_reply":"2024-07-05T16:09:38.503478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_label_path.columns)","metadata":{"execution":{"iopub.status.busy":"2024-07-05T16:17:11.077172Z","iopub.execute_input":"2024-07-05T16:17:11.079110Z","iopub.status.idle":"2024-07-05T16:17:11.091846Z","shell.execute_reply.started":"2024-07-05T16:17:11.079050Z","shell.execute_reply":"2024-07-05T16:17:11.089117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label_path.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-05T16:23:03.236131Z","iopub.execute_input":"2024-07-05T16:23:03.236803Z","iopub.status.idle":"2024-07-05T16:23:03.265753Z","shell.execute_reply.started":"2024-07-05T16:23:03.236764Z","shell.execute_reply":"2024-07-05T16:23:03.264075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_map = {'Normal/Mild': 0, 'Moderate':1, 'Severe':2}\ntrain_label_path = train_label_path.replace(label_map).infer_objects(copy=False)\ntrain_label_path.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-05T16:24:27.408513Z","iopub.execute_input":"2024-07-05T16:24:27.409050Z","iopub.status.idle":"2024-07-05T16:24:27.507627Z","shell.execute_reply.started":"2024-07-05T16:24:27.408989Z","shell.execute_reply":"2024-07-05T16:24:27.506041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conditions= list(train_label_path['condition'].unique())\nprint(f'Conditions- {conditions}')\nprint(f'Length of Conditions- {len(conditions)}')\n\nlevels= list(train_label_path['level'].unique())\nprint(f'LEVELS- {levels}')\n\nprint(f'Length of LEVELS- {len(levels)}')","metadata":{"execution":{"iopub.status.busy":"2024-07-05T16:27:18.647576Z","iopub.execute_input":"2024-07-05T16:27:18.649054Z","iopub.status.idle":"2024-07-05T16:27:18.670932Z","shell.execute_reply.started":"2024-07-05T16:27:18.648978Z","shell.execute_reply":"2024-07-05T16:27:18.669221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TEST","metadata":{}},{"cell_type":"code","source":"\ntest_desp = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_series_descriptions.csv')","metadata":{"execution":{"iopub.status.busy":"2024-07-05T16:31:43.550284Z","iopub.execute_input":"2024-07-05T16:31:43.550782Z","iopub.status.idle":"2024-07-05T16:31:43.561691Z","shell.execute_reply.started":"2024-07-05T16:31:43.550749Z","shell.execute_reply":"2024-07-05T16:31:43.560418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_desp.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-05T16:31:55.563108Z","iopub.execute_input":"2024-07-05T16:31:55.563624Z","iopub.status.idle":"2024-07-05T16:31:55.577068Z","shell.execute_reply.started":"2024-07-05T16:31:55.563588Z","shell.execute_reply":"2024-07-05T16:31:55.575688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-05T16:32:17.898709Z","iopub.execute_input":"2024-07-05T16:32:17.899166Z","iopub.status.idle":"2024-07-05T16:32:17.912503Z","shell.execute_reply.started":"2024-07-05T16:32:17.899129Z","shell.execute_reply":"2024-07-05T16:32:17.911205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.shape","metadata":{"execution":{"iopub.status.busy":"2024-07-05T16:32:46.482984Z","iopub.execute_input":"2024-07-05T16:32:46.483462Z","iopub.status.idle":"2024-07-05T16:32:46.491863Z","shell.execute_reply.started":"2024-07-05T16:32:46.483430Z","shell.execute_reply":"2024-07-05T16:32:46.490327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/sample_submission.csv')\nsample","metadata":{"execution":{"iopub.status.busy":"2024-07-05T16:33:34.333249Z","iopub.execute_input":"2024-07-05T16:33:34.333712Z","iopub.status.idle":"2024-07-05T16:33:34.356976Z","shell.execute_reply.started":"2024-07-05T16:33:34.333681Z","shell.execute_reply":"2024-07-05T16:33:34.355666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}