{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":8236972,"sourceType":"datasetVersion","datasetId":4885707},{"sourceId":8942685,"sourceType":"datasetVersion","datasetId":5380929},{"sourceId":9096998,"sourceType":"datasetVersion","datasetId":5490102}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install duckdb --no-index --find-links=file:///kaggle/input/polars-and-duckdb/kaggle/working/mysitepackages/duck_pkg","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-08-03T15:48:30.708082Z","iopub.execute_input":"2024-08-03T15:48:30.708961Z","iopub.status.idle":"2024-08-03T15:48:44.265034Z","shell.execute_reply.started":"2024-08-03T15:48:30.708919Z","shell.execute_reply":"2024-08-03T15:48:44.264145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport glob\nimport os\nimport polars as pl\nimport duckdb as dd\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport cv2\nfrom pydicom import dcmread\nimport warnings\nfrom sklearn.preprocessing import LabelEncoder\nimport pickle\nimport gc\nimport ctypes","metadata":{"execution":{"iopub.status.busy":"2024-08-03T15:48:49.937422Z","iopub.execute_input":"2024-08-03T15:48:49.938267Z","iopub.status.idle":"2024-08-03T15:48:51.533174Z","shell.execute_reply.started":"2024-08-03T15:48:49.938233Z","shell.execute_reply":"2024-08-03T15:48:51.532244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Conditions\n### spinal_canal_stenosis\n### neural_foraminal_narrowing\n#### left/right\n### subarticular_stenosis\n#### left/right\n\n## Levels of vertebrae\n#### l1_l2\n#### l2_l3\n#### l3_l4\n#### l4_l5\n#### l5_s1\n\n> Each Study has series of images for these three conditions. The series description tells us what type of image is in the dcm file","metadata":{}},{"cell_type":"markdown","source":"### Reading the available training metadata files using polars\n### Bringing the metadata files in a desired shape","metadata":{}},{"cell_type":"code","source":"train_meta_data = pl.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv'\\\n                              , low_memory=True)\ntrain_series_desc = pl.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_series_descriptions.csv'\\\n                                , low_memory=True)\ntrain_label_coordinates = pl.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_label_coordinates.csv'\\\n                                      , low_memory=True)\n\ntrain_meta_data_w_srs_desc = dd.sql(\"select b.series_description, b.series_id, a.* from train_meta_data a \\\nleft join train_series_desc b on a.study_id = b.study_id\").pl()\n\nmelt_cols = train_meta_data_w_srs_desc.columns[3:]\ntrain_meta_data_melted = train_meta_data_w_srs_desc.melt(id_vars=[\"study_id\",\"series_id\",\"series_description\"], value_vars=melt_cols)\ntrain_meta_data_melted.columns = ['study_id', 'series_id', 'series_description', 'condition_vert_level', 'severity']\n\ntrain_label_coordinates_formatted = dd.sql(\"select study_id, series_id, instance_number \\\n,lower(replace(condition,' ','_')||'_'||replace(level,'/','_')) as condition, x, y \\\nfrom train_label_coordinates\").pl()\n\nfull_training_data = dd.sql(\"select t1.study_id, t1.series_id, t1.series_description, t2.instance_number \\\n, t1.condition_vert_level, t1.severity, t2.x, t2.y \\\nfrom train_meta_data_melted t1 \\\nleft join train_label_coordinates_formatted t2 on t1.study_id = t2.study_id \\\nand t1.condition_vert_level = t2.condition \\\n\").pl()\n\nprint(full_training_data.shape)","metadata":{"execution":{"iopub.status.busy":"2024-07-20T07:15:35.021117Z","iopub.execute_input":"2024-07-20T07:15:35.021680Z","iopub.status.idle":"2024-07-20T07:15:35.434470Z","shell.execute_reply.started":"2024-07-20T07:15:35.021630Z","shell.execute_reply":"2024-07-20T07:15:35.433287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dd.sql(\"select * from train_meta_data_melted where study_id = 3008676218 \").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Checking a specific study/patient's scans and the corresponding details, only to get a better sense of the data","metadata":{}},{"cell_type":"code","source":"pl.Config(fmt_str_lengths=100)\npl.Config.set_tbl_rows(1000)\n\n#print(train_meta_data_melted.shape)\n# print(train_meta_data.filter(pl.col('study_id')==100206310).transpose(include_header=True))\n#print(train_meta_data_melted.filter(pl.col('study_id')==100206310))\n# print(train_series_desc.filter(pl.col('study_id')==100206310))\n#print(train_label_coordinates_formatted.shape)\n#print(train_label_coordinates_formatted.filter(pl.col('study_id')==100206310))\n#print(train_series_coordinates.filter(pl.col('study_id')==100206310).sort(['study_id','series_id']))\nprint(full_training_data.filter(pl.col('study_id')==100206310).sort(['study_id','series_id']))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"condition_list = dd.sql(\"select distinct condition_vert_level from full_training_data\").pl().to_series().to_list()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dd.sql(\"select series_description, severity, count(distinct(study_id||'/'||series_id)) as ss_count from full_training_data \\\nwhere condition_vert_level = 'left_neural_foraminal_narrowing_l1_l2' and severity is not null and severity != 'Severe' group by series_description, severity\").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dd.sql(\"select series_description, count(distinct(study_id||'/'||series_id)) as ss_count from full_training_data \\\nwhere severity = 'Severe' group by series_description\").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pl.Config(fmt_str_lengths=100)\npl.Config.set_tbl_rows(1000)\n\ndd.sql(\"select condition_vert_level, series_description, count(distinct(study_id||'/'||series_id)) as ss_count from full_training_data \\\nwhere severity = 'Severe' group by condition_vert_level, series_description\").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files_path_root = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/'\n\nfor condition in condition_list:\n    print(\"generating feature metadata for {0}\".format(condition))\n\n    full_feature_metadata = dd.sql(\"select distinct ('{0}'||study_id||'/'||series_id) as file_path_root, severity from full_training_data \\\n    where condition_vert_level = '{1}' and severity is not null\".format(train_files_path_root, condition)).pl()\n\n    full_feature_metadata_pd = full_feature_metadata.to_pandas()\n\n    label_mapping = {}\n\n    for file_path in range(full_feature_metadata_pd.shape[0]):\n        study_series_path = full_feature_metadata_pd['file_path_root'][file_path]\n        for image_file_path in os.listdir(study_series_path):\n            full_img_path = os.path.join(study_series_path, image_file_path)\n            label_mapping[full_img_path] = full_feature_metadata_pd['severity'][file_path]\n\n\n    # Create a list of tuples containing the audio file paths and labels\n    data = [(full_img_path, severity) for full_img_path, severity in label_mapping.items()]\n    # Create a Pandas DataFrame from the list of tuples\n    annotated_data = pd.DataFrame(data, columns=['full_img_path', 'severity'])\n\n    label_encoder = LabelEncoder()\n\n    annotated_data['encoded_severity'] = label_encoder.fit_transform(annotated_data['severity'])\n    print(annotated_data.shape)\n    print(annotated_data.head(5))\n    \n    file_name = \"{0}_feature_metadata\".format(condition)\n\n    annotated_data.to_csv(\"{0}.csv\".format(file_name), index=False)\n    print(\"saved file {0}\".format(file_name))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pl.Config(fmt_str_lengths=1000)\npl.Config.set_tbl_rows(1000)\n\ntest = pl.read_csv('/kaggle/input/left-neural-foraminal-narrowing-metadata/left_neural_foraminal_narrowing_l4_l5_feature_metadata.csv'\\\n            , low_memory=True)\n\nprint(test.shape)\nprint(test.head(5))","metadata":{"execution":{"iopub.status.busy":"2024-07-28T15:46:08.321407Z","iopub.execute_input":"2024-07-28T15:46:08.322514Z","iopub.status.idle":"2024-07-28T15:46:08.593283Z","shell.execute_reply.started":"2024-07-28T15:46:08.322463Z","shell.execute_reply":"2024-07-28T15:46:08.592111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Checking the output of downsampling","metadata":{}},{"cell_type":"code","source":"dicom_ds = dcmread('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/1038453736/3170465859/12.dcm')\nimg_array = dicom_ds.pixel_array\nIMG_normalized = cv2.normalize(img_array, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8U)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('The image has {} x {} voxels'.format(img_array.shape[0],img_array.shape[1]))\ndata_downsampling = img_array[::2, ::2]\nprint('The downsampled image has {} x {} voxels'.format(data_downsampling.shape[0], data_downsampling.shape[1]))\n\n# copy the data back to the original data set\ndicom_ds.PixelData = data_downsampling.tobytes()\n# update the information regarding the shape of the data array\ndicom_ds.Rows, dicom_ds.Columns = data_downsampling.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(IMG_normalized)\nplt.axis('off')  # Turn off axis numbers and ticks\nplt.title(\"left_neural_foraminal_narrowing_l4_l5 - Moderate\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(data_downsampling)\nplt.axis('off')  # Turn off axis numbers and ticks\nplt.title(\"left_neural_foraminal_narrowing_l4_l5 - downsampled - Moderate\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_training_data.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_df_w_series_desc = dd.sql(\"select  t.*, ftd.series_description \\\nfrom test t \\\njoin full_training_data ftd \\\non ftd.study_id = cast(split_part(t.full_img_path,'/',-3) as int64) \\\nand ftd.series_id = cast(split_part(t.full_img_path,'/',-2) as int64) \\\nwhere ftd.condition_vert_level = 'left_neural_foraminal_narrowing_l4_l5' and ftd.severity is not null\\\n\").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_df_w_series_desc.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_df = metadata_df_w_series_desc.filter(pl.col('series_description')=='Sagittal T1')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_df.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_type_list = dd.sql(\"select distinct series_description from metadata_df_w_series_desc\").pl().to_series().to_list()\nseries_type_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for s_type in series_type_list:\n    metadata_df = metadata_df_w_series_desc.filter(pl.col('series_description')==s_type)\n    print(\"for images of orientation :\", s_type)\n    labels = []\n    features = []\n    for i in tqdm(range(metadata_df.shape[0])):\n        dcm_image_path = metadata_df.item(i,0)\n        dicom_ds = dcmread(dcm_image_path)\n        img_array = dicom_ds.pixel_array\n        #IMG_normalized = cv2.normalize(img_array, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8U)\n        features.append(np.mean(img_array.T, axis=0))\n        labels.append(metadata_df.item(i,2))\n    extracted_training_features = pd.DataFrame(features, columns=['image_array'])\n    extracted_training_labels = pd.DataFrame(labels, columns=['encoded_severity'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features[10000].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dd.sql(\"select distinct severity, encoded_severity from test order by encoded_severity\").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### feature extraction from Image files","metadata":{}},{"cell_type":"code","source":"metadata_root_path = '/kaggle/input/left-neural-foraminal-narrowing-metadata'\n\nfor file in os.listdir(metadata_root_path):\n    if file == 'left_neural_foraminal_narrowing_l5_s1_feature_metadata.csv':\n        labels = []\n        features = []\n        data = {}\n        metadata_file_path = os.path.join(metadata_root_path, file)\n        print(metadata_file_path)\n        metadata_df = pl.read_csv(metadata_file_path, low_memory=True)\n        for j in tqdm(range(metadata_df.shape[0])):\n            dcm_image_path = metadata_df.item(j,0)\n            dicom_ds = dcmread(dcm_image_path)\n            img_array = dicom_ds.pixel_array\n            features.append(np.mean(img_array.T, axis=0))\n            # features.append(dcmread(dcm_image_path).pixel_array)\n            labels.append(metadata_df.item(j,2))\n\n        print(\"feature list length --> \", len(features))\n        print(\"label list length --> \", len(labels))\n\n        #extracted_training_features = np.array([np.resize(img,(128,128)) for img in features])\n        #training_labels = np.array(labels)\n\n        #del labels\n        #del features\n        #gc.collect()\n\n        extracted_training_features_file_name = \"{0}_training_features\".format(file.replace('_feature_metadata.csv',''))\n        labels_file_name = \"{0}_labels\".format(file.replace('.csv',''))\n\n        with open(extracted_training_features_file_name, \"wb\") as file:\n            pickle.dump(features, file)\n        with open(labels_file_name, \"wb\") as file:\n            pickle.dump(labels, file)\n\n        del labels\n        del features\n        gc.collect()\n        libc = ctypes.CDLL(\"libc.so.6\") # clearing cache \n        libc.malloc_trim(0)\n\n        #data[\"image_array\"] = features\n        #data[\"encoded_severity\"] = labels\n        #extracted_training_data = pd.DataFrame(data)\n        #extracted_training_data.to_csv(\"{0}.csv.gz\".format(file.replace('.csv','')), index=False, compression='gzip')\n\n        print('finished dumping features & labels for {0}'.format(file))","metadata":{"execution":{"iopub.status.busy":"2024-07-29T05:54:09.211439Z","iopub.execute_input":"2024-07-29T05:54:09.212442Z","iopub.status.idle":"2024-07-29T06:43:16.260597Z","shell.execute_reply.started":"2024-07-29T05:54:09.212406Z","shell.execute_reply":"2024-07-29T06:43:16.258037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = []\nlabels = []\n# data = {}\n\nfor j in tqdm(range(test.head(100).shape[0])):\n    dcm_image_path = test.item(j,0)\n    dicom_ds = dcmread(dcm_image_path)\n    img_array = dicom_ds.pixel_array\n    #print(img_array.shape)\n    # features.append(img_array)\n    features.append(np.mean(img_array.T, axis=0))\n    labels.append(test.item(j,2))\n\nprint(\"feature list length --> \", len(features))\nprint(\"label list length --> \", len(labels))\n\nextracted_training_features = features\nlabels = labels\n\n#data[\"image_array\"] = features\n#data[\"encoded_severity\"] = labels\n\n#extracted_training_data = pd.DataFrame(data)\n#extracted_training_data.to_csv(\"test.csv\", index=False)\n\nprint('finished dumping features & labels')","metadata":{"execution":{"iopub.status.busy":"2024-07-28T15:46:52.840483Z","iopub.execute_input":"2024-07-28T15:46:52.841328Z","iopub.status.idle":"2024-07-28T15:46:55.480033Z","shell.execute_reply.started":"2024-07-28T15:46:52.841292Z","shell.execute_reply":"2024-07-28T15:46:55.478975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"/kaggle/input/left-neural-foraminal-narrowing-training/left_neural_foraminal_narrowing_l1_l2_training_features\", \"rb\") as file:\n    extracted_training_features = pickle.load(file)\n    \nwith open(\"/kaggle/input/left-neural-foraminal-narrowing-training/left_neural_foraminal_narrowing_l1_l2_feature_metadata_labels\", \"rb\") as file:\n    labels = pickle.load(file)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T15:49:06.727209Z","iopub.execute_input":"2024-08-03T15:49:06.727968Z","iopub.status.idle":"2024-08-03T15:49:11.393178Z","shell.execute_reply.started":"2024-08-03T15:49:06.727935Z","shell.execute_reply":"2024-08-03T15:49:11.392407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(type(extracted_training_features[5]))\nprint(extracted_training_features[5].shape)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T15:41:14.405071Z","iopub.execute_input":"2024-08-03T15:41:14.405831Z","iopub.status.idle":"2024-08-03T15:41:14.412492Z","shell.execute_reply.started":"2024-08-03T15:41:14.405791Z","shell.execute_reply":"2024-08-03T15:41:14.411013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_vstacked = np.vstack(labels)\nlabels_vstacked.shape","metadata":{"execution":{"iopub.status.busy":"2024-08-03T15:52:54.673915Z","iopub.execute_input":"2024-08-03T15:52:54.674265Z","iopub.status.idle":"2024-08-03T15:52:54.996334Z","shell.execute_reply.started":"2024-08-03T15:52:54.674236Z","shell.execute_reply":"2024-08-03T15:52:54.995397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"extracted_training_features_resized = []\n\nfor img in extracted_training_features:\n    extracted_training_features_resized.append(np.resize(img,(128,)))\n    \nfinal = np.array(extracted_training_features_resized)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T15:49:18.306307Z","iopub.execute_input":"2024-08-03T15:49:18.307149Z","iopub.status.idle":"2024-08-03T15:49:19.921799Z","shell.execute_reply.started":"2024-08-03T15:49:18.307116Z","shell.execute_reply":"2024-08-03T15:49:19.921023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(type(final))\nprint(final.shape)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T15:49:26.269476Z","iopub.execute_input":"2024-08-03T15:49:26.269889Z","iopub.status.idle":"2024-08-03T15:49:26.274863Z","shell.execute_reply.started":"2024-08-03T15:49:26.269860Z","shell.execute_reply":"2024-08-03T15:49:26.273887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\n\nmodel = keras.models.Sequential()\nmodel.add(keras.layers.Flatten(input_shape=[128]))\nmodel.add(keras.layers.Dense(150, activation=\"relu\"))\nmodel.add(keras.layers.Dense(65, activation=\"relu\"))\nmodel.add(keras.layers.Dense(3, activation=\"softmax\"))","metadata":{"execution":{"iopub.status.busy":"2024-08-03T16:16:21.637684Z","iopub.execute_input":"2024-08-03T16:16:21.638038Z","iopub.status.idle":"2024-08-03T16:16:21.677673Z","shell.execute_reply.started":"2024-08-03T16:16:21.638009Z","shell.execute_reply":"2024-08-03T16:16:21.676838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nx_train, x_test, y_train, y_test = train_test_split(final, labels_vstacked, test_size=0.4, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T15:53:07.510354Z","iopub.execute_input":"2024-08-03T15:53:07.511219Z","iopub.status.idle":"2024-08-03T15:53:07.582254Z","shell.execute_reply.started":"2024-08-03T15:53:07.511188Z","shell.execute_reply":"2024-08-03T15:53:07.581466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(x_train.shape)\nprint(y_train.shape)\nprint(x_test.shape)\nprint(y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T15:53:41.535658Z","iopub.execute_input":"2024-08-03T15:53:41.535995Z","iopub.status.idle":"2024-08-03T15:53:41.541243Z","shell.execute_reply.started":"2024-08-03T15:53:41.535971Z","shell.execute_reply":"2024-08-03T15:53:41.540350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_test, x_valid, y_test, y_valid = train_test_split(x_test, y_test, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T15:54:30.737507Z","iopub.execute_input":"2024-08-03T15:54:30.738186Z","iopub.status.idle":"2024-08-03T15:54:30.771082Z","shell.execute_reply.started":"2024-08-03T15:54:30.738154Z","shell.execute_reply":"2024-08-03T15:54:30.769988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(x_train.shape)\nprint(y_train.shape)\nprint(x_test.shape)\nprint(y_test.shape)\nprint(x_valid.shape)\nprint(y_valid.shape)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T15:54:53.480282Z","iopub.execute_input":"2024-08-03T15:54:53.480898Z","iopub.status.idle":"2024-08-03T15:54:53.486152Z","shell.execute_reply.started":"2024-08-03T15:54:53.480866Z","shell.execute_reply":"2024-08-03T15:54:53.485141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow.keras.backend as K\nK.epsilon()","metadata":{"execution":{"iopub.status.busy":"2024-08-03T16:17:17.653020Z","iopub.execute_input":"2024-08-03T16:17:17.653820Z","iopub.status.idle":"2024-08-03T16:17:17.660792Z","shell.execute_reply.started":"2024-08-03T16:17:17.653783Z","shell.execute_reply":"2024-08-03T16:17:17.659739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"adam\"\n              , metrics=[keras.metrics.SparseCategoricalCrossentropy()])","metadata":{"execution":{"iopub.status.busy":"2024-08-03T16:30:05.914137Z","iopub.execute_input":"2024-08-03T16:30:05.914512Z","iopub.status.idle":"2024-08-03T16:30:05.932245Z","shell.execute_reply.started":"2024-08-03T16:30:05.914472Z","shell.execute_reply":"2024-08-03T16:30:05.931375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(x_train, y_train, epochs=120, validation_data=(x_valid, y_valid))","metadata":{"execution":{"iopub.status.busy":"2024-08-03T16:30:16.974193Z","iopub.execute_input":"2024-08-03T16:30:16.974577Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(x_test, y_test)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/training-files-lnfn/left_neural_foraminal_narrowing_l1_l2_feature_metadata.csv')\nprint(df.shape)","metadata":{"execution":{"iopub.status.busy":"2024-07-28T14:56:31.562339Z","iopub.execute_input":"2024-07-28T14:56:31.562740Z","iopub.status.idle":"2024-07-28T14:56:49.723641Z","shell.execute_reply.started":"2024-07-28T14:56:31.562706Z","shell.execute_reply":"2024-07-28T14:56:49.721681Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['image_array'][0]","metadata":{"execution":{"iopub.status.busy":"2024-07-28T14:58:23.742394Z","iopub.execute_input":"2024-07-28T14:58:23.742791Z","iopub.status.idle":"2024-07-28T14:58:23.751565Z","shell.execute_reply.started":"2024-07-28T14:58:23.742758Z","shell.execute_reply":"2024-07-28T14:58:23.750161Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\n\nprint(tf.__version__)\nprint(keras.__version__)","metadata":{"execution":{"iopub.status.busy":"2024-07-28T04:25:01.828628Z","iopub.execute_input":"2024-07-28T04:25:01.829022Z","iopub.status.idle":"2024-07-28T04:25:16.649429Z","shell.execute_reply.started":"2024-07-28T04:25:01.828991Z","shell.execute_reply":"2024-07-28T04:25:16.648049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fashion_mnist = keras.datasets.fashion_mnist\n(X_train_full, y_train_full),(X_test,y_test) = fashion_mnist.load_data()","metadata":{"execution":{"iopub.status.busy":"2024-07-28T04:25:20.527248Z","iopub.execute_input":"2024-07-28T04:25:20.527986Z","iopub.status.idle":"2024-07-28T04:25:21.777675Z","shell.execute_reply.started":"2024-07-28T04:25:20.527950Z","shell.execute_reply":"2024-07-28T04:25:21.776127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(type(X_train_full[0]))\nprint(X_train_full[0].shape)\nprint(X_train_full.shape)","metadata":{"execution":{"iopub.status.busy":"2024-07-28T04:26:04.195892Z","iopub.execute_input":"2024-07-28T04:26:04.196338Z","iopub.status.idle":"2024-07-28T04:26:04.202671Z","shell.execute_reply.started":"2024-07-28T04:26:04.196301Z","shell.execute_reply":"2024-07-28T04:26:04.201301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(y_train_full[0])","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:44:38.320591Z","iopub.execute_input":"2024-07-21T05:44:38.321047Z","iopub.status.idle":"2024-07-21T05:44:38.328087Z","shell.execute_reply.started":"2024-07-21T05:44:38.321013Z","shell.execute_reply":"2024-07-21T05:44:38.326825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import train_test_split\n\nx = test.select(pl.col('full_img_path'))\ny = test.select(pl.col('encoded_severity'))\n\nx_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.25, random_state=42)\n\nprint(\"full train shape & label shape = {0} & {1}\".format(x_train.shape, y_train.shape))\nprint(\"full test shape & label shape = {0} & {1}\".format(x_test.shape, y_test.shape))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dd.sql(\"select condition_vert_level, series_description, severity, count(distinct(study_id)) as study_count \\\n, count(distinct(series_id)) as series_count from full_training_data \\\ngroup by condition_vert_level, series_description, severity order by 1,2,3\").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Looking at the number of studies and series of images available for various scan orientations, it appears that the Saggital orientations are missing for a couple of studies","metadata":{}},{"cell_type":"code","source":"dd.sql(\"select series_description, count(distinct(study_id)) as studies, count(distinct(series_id)) as series \\\nfrom train_series_desc group by series_description\").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Checking whether all the different conditions for all the 5 vertebrae are available in all the studies.\n#### It appears that some of the condition/vertebrae combinations are not available in some of the studies\n##### e.g. The images with orientation \"Axial T2\" for the condition \"Right Subarticular Stenosis\" at the vertebra \"L1/L2\" is available in 1812 studies out of a total of 1975 studies","metadata":{}},{"cell_type":"code","source":"part_1 = list(os.listdir('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images'))\n# part_1 = list(filter(lambda x: x.find('.DS') == -1, part_1))\nlen(part_1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_obj = {{ \n    'study_id': x,\n    'folder_path': f\"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/{x}\",\n    'SeriesInstanceUIDs': [] \n}\n    for x in part_1 \n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_obj","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"p1 = [(x, f\"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/{x}\") for x in part_1]\nmeta_obj = { p[0]: { 'folder_path': p[1], \n                    'SeriesInstanceUIDs': [] \n                   } \n            for p in p1 }\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for m in meta_obj:\n    meta_obj[m]['SeriesInstanceUIDs'] = list(os.listdir(meta_obj[m]['folder_path']))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series_desc.filter((pl.col('study_id') == int('100206310')) & (pl.col('series_id') == int('1792451510')))\\\n.select(pl.col('series_description')).item(0,0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# grabs the correspoding series descriptions\nfor k in tqdm(meta_obj):\n    for s in meta_obj[k]['SeriesInstanceUIDs']:\n        if 'SeriesDescriptions' not in meta_obj[k]:\n            meta_obj[k]['SeriesDescriptions'] = []\n        try:\n            meta_obj[k]['SeriesDescriptions'].append(train_series_desc.filter((pl.col('study_id') == int(k)) \\\n                                                                              & (pl.col('series_id') == int(s)))\\\n                                                     .select(pl.col('series_description')).item(0,0))\n        except:\n            print(\"Failed on\", s, k)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ptobj = meta_obj[list(meta_obj.keys())[10]]\nptobj = meta_obj['100206310']\nptobj","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ptobj['image_files'] = []\n\nfor idx, i in enumerate(ptobj['SeriesInstanceUIDs']):\n    print(idx)\n    print(i)\n    print(\"**********\")\n    ptobj['image_files'].append(glob.glob(f\"{ptobj['folder_path']}/{ptobj['SeriesInstanceUIDs'][idx]}/*.dcm\"))\n    \n#len(images)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ptobj['SeriesDescriptions'][0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ptobj['image_files'][0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ptobj","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k in tqdm(meta_obj):\n    meta_obj[k]['image_files'] = []\n    for idx, i in enumerate(meta_obj[k]['SeriesInstanceUIDs']):\n        meta_obj[k]['image_files']\\\n        .append(len(glob.glob(f\"{meta_obj[k]['folder_path']}/{meta_obj[k]['SeriesInstanceUIDs'][idx]}/*.dcm\")))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_obj['100206310']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_obj[list(meta_obj.keys())[1]]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pl.Config(fmt_str_lengths=100)\npl.Config.set_tbl_rows(1000)\ndd.sql(\" \\\nselect tsd.series_description, tlc.condition, tlc.level, \\\ncount(distinct(tlc.study_id)) as studies, count(distinct(tlc.instance_number)) as label_defining_images, \\\nround(count(distinct(tlc.study_id))/1975, 5) as pct_of_total_studies, \\\nfrom train_series_desc tsd \\\njoin train_label_coordinates tlc \\\non tsd.study_id = tlc.study_id \\\nand tsd.series_id = tlc.series_id \\\ngroup by tsd.series_description, tlc.condition, tlc.level \\\norder by 1,2, 4 desc \\\n\").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Now let's check how many studies have moderate or severe levels of the condition \"Right Subarticular Stenosis\" at the vertebra \"L1/L2\"","metadata":{}},{"cell_type":"code","source":"dd.sql(\"select value, count(distinct(study_id)) from train_meta_data_melted where variable = 'right_subarticular_stenosis_l1_l2' \\\ngroup by value\").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dd.sql(\"select value, count(distinct(study_id)) from train_meta_data_melted where variable = 'right_subarticular_stenosis_l1_l2' \\\nand study_id in (select distinct(study_id) from train_label_coordinates where condition = 'Right Subarticular Stenosis'\\\nand level = 'L1/L2') group by value\").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label_coordinates_list = dd.sql(\"select distinct study_id from train_meta_data_melted where variable = 'right_subarticular_stenosis_l1_l2' \\\nand value = 'Normal/Mild' \\\nand study_id in (select distinct(study_id) from train_label_coordinates where condition = 'Right Subarticular Stenosis'\\\nand level = 'L1/L2')\").pl().to_series().unique().to_list()\n\ntrain_meta_data_list = dd.sql(\"select distinct study_id from train_meta_data_melted where variable = 'right_subarticular_stenosis_l1_l2' and value = 'Normal/Mild' \")\\\n.pl().to_series().unique().to_list()\n\nset(train_label_coordinates_list) ^ set(train_meta_data_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### The above analysis shows that there are two studies related to Right Subarticular Stenosis of L1/L2 which are not present in the train_label_coordinates file","metadata":{}},{"cell_type":"code","source":"dd.sql(\"select * from train_label_coordinates where study_id in ('3008676218','3303545110')\").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dd.sql(\"select * from train_meta_data where study_id in ('3008676218','3303545110')\").pl().transpose(include_header=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### It seems from above, that the study 3303545110 is only present for Right Subarticular Stenosis of L5/S1 and not of L1/L2 and the study 3008676218 is not present at all for any conditions/vertebrae combination\n#### The questions is then how is 3303545110 labelled as Normal/Mild for Right Subarticular Stenosis of L1/L2 and 3008676218 labelled for all the conditions/vertebrae combination","metadata":{}},{"cell_type":"markdown","source":"#### Below are some of the images from the study_id/patient 100206310 ","metadata":{}},{"cell_type":"code","source":"dcm_image_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/1012284084/20.dcm'\nleft_centre_coords = (int(180.355677), int(165.0342))\nright_centre_coords = (int(145.120536), int(162.285714))\nradius = 10\ncolor = (255, 0, 0)  # Red color in BGR\nthickness = 2\n\ndicom_ds = dcmread(dcm_image_path)\nIMG = dicom_ds.pixel_array\n\nIMG_normalized = cv2.normalize(IMG, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8U)\n\nleft_IMG_with_circle = cv2.circle(IMG_normalized.copy(), left_centre_coords, radius, color, thickness)\nleft_IMG_with_circle = cv2.cvtColor(left_IMG_with_circle, cv2.COLOR_BGR2RGB)\n\nright_IMG_with_circle = cv2.circle(IMG_normalized.copy(), right_centre_coords, radius, color, thickness)\nright_IMG_with_circle = cv2.cvtColor(right_IMG_with_circle, cv2.COLOR_BGR2RGB)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(left_IMG_with_circle)\nplt.axis('off')  # Turn off axis numbers and ticks\nplt.title(\"Left subarticular stenosis L1/L2 - Normal/Mild\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(right_IMG_with_circle)\nplt.axis('off')  # Turn off axis numbers and ticks\nplt.title(\"Right subarticular stenosis L1/L2 - Normal/Mild\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dcm_image_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/1012284084/46.dcm'\nleft_centre_coords = (int(168.536252), int(156.27907))\nright_centre_coords = (int(146.263393), int(160.0))\nradius = 10\ncolor = (255, 0, 0)  # Red color in BGR\nthickness = 2\n\ndicom_ds = dcmread(dcm_image_path)\nIMG = dicom_ds.pixel_array\n\nIMG_normalized = cv2.normalize(IMG, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8U)\n\nleft_IMG_with_circle = cv2.circle(IMG_normalized.copy(), left_centre_coords, radius, color, thickness)\nleft_IMG_with_circle = cv2.cvtColor(left_IMG_with_circle, cv2.COLOR_BGR2RGB)\n\nright_IMG_with_circle = cv2.circle(IMG_normalized.copy(), right_centre_coords, radius, color, thickness)\nright_IMG_with_circle = cv2.cvtColor(right_IMG_with_circle, cv2.COLOR_BGR2RGB)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(left_IMG_with_circle)\nplt.axis('off')  # Turn off axis numbers and ticks\nplt.title(\"Left subarticular stenosis L4/L5 - Severe\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(right_IMG_with_circle)\nplt.axis('off')  # Turn off axis numbers and ticks\nplt.title(\"Right subarticular stenosis L4/L5 - Moderate\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dcm_image_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/2092806862/4.dcm'\nright_centre_coords = (int(253.393075), int(278.680244))\nradius = 10\ncolor = (255, 0, 0)  # Red color in BGR\nthickness = 2\n\ndicom_ds = dcmread(dcm_image_path)\nIMG = dicom_ds.pixel_array\n\nIMG_normalized = cv2.normalize(IMG, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8U)\n\nright_IMG_with_circle = cv2.circle(IMG_normalized.copy(), right_centre_coords, radius, color, thickness)\nright_IMG_with_circle = cv2.cvtColor(right_IMG_with_circle, cv2.COLOR_BGR2RGB)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(right_IMG_with_circle)\nplt.axis('off')  # Turn off axis numbers and ticks\nplt.title(\"Right neural foraminal narrowing L4/L5 - Moderate\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Aim : To select image files to create Train and Validation set\n1. 25% of data will be set aside permanently as a validation set for all experiments. This set won't be used for training/testing/cross-validation\n2. 75% of data will be used for all experiments\n3. Images are 320 * 320 arrays\n### Modelling strategy\n1. Build 25 different models, one per condition/vertebrae combination","metadata":{}},{"cell_type":"markdown","source":"#### Pipeline to make a sample submission to test the submission data creation logic","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/sample_submission.csv')\nsubmission['row_id'] = 'samples'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nimport os\n\nconfig = {}\nconfig['root_file_path'] = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_images/'\n\nstudies = os.listdir(config['root_file_path'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"studies","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rows = {}\nweight_dict = {'normal_mild':1, 'moderate':2, 'severe':4}\nconditions = ['spinal_canal_stenosis', 'neural_foraminal_narrowing', 'subarticular_stenosis']\nsides = ['left', 'right']\nvertebrae_levels = ['l1_l2', 'l2_l3', 'l3_l4', 'l4_l5', 'l5_s1']\nseverity_levels = ['normal_mild', 'moderate', 'severe']\n\nfor c in conditions:\n    for v in vertebrae_levels:\n        if c != 'spinal_canal_stenosis':\n            for s in sides:\n                for st in studies:\n                    #print(st+'_'+s+'_'+c+'_'+v)\n                    rows[st+'_'+s+'_'+c+'_'+v] = np.array([0.333333 * 2, 0.333333 * 2, 0.333333 * 2])\n        else:\n            for st in studies:\n                #print(st+'_'+c+'_'+v)\n                rows[st+'_'+c+'_'+v] = np.array([0.333333 * 2, 0.333333 * 2, 0.333333 * 2])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for row_id, feature in tqdm(rows.items()):\n    feature_set_reshaped = feature.reshape(1, -1)\n    predictions = np.ascontiguousarray(feature_set_reshaped/2)\n    df = pd.DataFrame(predictions, columns=severity_levels)\n    df.insert(loc=0, column='row_id', value=row_id)\n    submission = pd.concat([submission,df]).reset_index(drop=True)\n    \ni = submission[(submission.row_id == 'samples')].index\nsubmission = submission.drop(i).reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}