{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":187731,"sourceType":"datasetVersion","datasetId":80814}],"dockerImageVersionId":30787,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nimport os\nimport pydicom\nimport cv2\nfrom tensorflow.keras.utils import Sequence\nfrom tensorflow.keras.utils import to_categorical\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nfrom tensorflow.data import Dataset\nfrom tensorflow.keras.layers import Input, Dense, Dropout, Flatten, Concatenate, GlobalAveragePooling2D\nfrom tensorflow.keras.models import Model\nfrom keras.applications.densenet import DenseNet121","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T18:09:16.395510Z","iopub.execute_input":"2025-01-30T18:09:16.396366Z","iopub.status.idle":"2025-01-30T18:09:28.892501Z","shell.execute_reply.started":"2025-01-30T18:09:16.396321Z","shell.execute_reply":"2025-01-30T18:09:28.891559Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/'\n\n# Load CSV files\ntrain_df = pd.read_csv(os.path.join(train_path, 'train.csv'))\nlabel_df = pd.read_csv(os.path.join(train_path, 'train_label_coordinates.csv'))\ntrain_description_df = pd.read_csv(os.path.join(train_path, 'train_series_descriptions.csv'))\ntest_description_df = pd.read_csv(os.path.join(train_path, 'test_series_descriptions.csv'))\nsubmission_df = pd.read_csv(os.path.join(train_path, 'sample_submission.csv'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T18:09:28.894099Z","iopub.execute_input":"2025-01-30T18:09:28.894984Z","iopub.status.idle":"2025-01-30T18:09:29.132093Z","shell.execute_reply.started":"2025-01-30T18:09:28.894941Z","shell.execute_reply":"2025-01-30T18:09:29.131414Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def image_path_generator(base_dir, df):\n    path_df = []\n    for _, row in df.iterrows():\n        image_df = {\n            'images' : [],\n            'study_id' : [],\n            'series_id' : []\n        }\n        \n        # for study_id, seriese_id in zip(df['study_id'], df['series_id']):\n        study_dir = os.path.join(base_dir, str(row['study_id']))\n        image_dir = os.path.join(study_dir, str(row['series_id']))\n\n        images = os.listdir(image_dir)\n        for dir in images:\n            image_df['study_id'].append(row['study_id'])\n            image_df['series_id'].append(row['series_id'])\n            image_df['images'].append(os.path.join(image_dir, str(dir)))\n\n        path_df.append(pd.DataFrame(image_df))\n\n    path_df = pd.concat(path_df, ignore_index=False)\n\n    return path_df\n        ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T18:09:29.133292Z","iopub.execute_input":"2025-01-30T18:09:29.133984Z","iopub.status.idle":"2025-01-30T18:09:29.140143Z","shell.execute_reply.started":"2025-01-30T18:09:29.133940Z","shell.execute_reply":"2025-01-30T18:09:29.139296Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dir = os.path.join(train_path, 'train_images')\ntest_dir = os.path.join(train_path, 'test_images')\nimg_path_df = image_path_generator(train_dir, train_description_df)\ntest_img_path = image_path_generator(test_dir, test_description_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T18:09:29.141886Z","iopub.execute_input":"2025-01-30T18:09:29.142135Z","iopub.status.idle":"2025-01-30T18:11:36.415886Z","shell.execute_reply.started":"2025-01-30T18:09:29.142110Z","shell.execute_reply":"2025-01-30T18:11:36.415170Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_final_df = {\n    'images' :[],\n    'instance_number' :[],\n    'study_id' :[],\n    'series_id' :[],\n    'condition' :[],\n    'level' :[],\n    'series_description' :[]\n}\nlv = ['L1/L2', 'L2/L3', 'L3/L4', 'L4/L5', 'L5/S1']\ncon = ['Spinal Canal Stenosis', 'Right Neural Foraminal Narrowing', 'Left Neural Foraminal Narrowing',\n       'Left Subarticular Stenosis', 'Right Subarticular Stenosis']\n\n##### process image path to get study id, seriese id, instance number,\nfor _, img in test_img_path.iterrows():\n    for col, val in img.items():\n        if col == 'images':\n            l = str(val).split('/')\n            for c in con:\n                for j in lv:\n                    test_final_df['images'].append(val)\n                    test_final_df['study_id'].append(l[-3])\n                    test_final_df['series_id'].append(l[-2])\n                    test_final_df['instance_number'].append(l[-1][0:-4])\n                    test_final_df['condition'].append(c)\n                    test_final_df['level'].append(j)\n\n##### add description for each image\ndescription = []\nfor ser in test_final_df['series_id']:\n    #print(test_description_df[test_description_df['series_id'] == int(ser)]['series_description'])\n    description.append(test_description_df[int(ser) == test_description_df['series_id']]['series_description'].values[0])\n\ntest_final_df['series_description'] = description\n\ntest_df = pd.DataFrame(test_final_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T18:11:36.416798Z","iopub.execute_input":"2025-01-30T18:11:36.417028Z","iopub.status.idle":"2025-01-30T18:11:37.048876Z","shell.execute_reply.started":"2025-01-30T18:11:36.417005Z","shell.execute_reply":"2025-01-30T18:11:37.048173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"structured_df = []\nfor _, row in train_df.iterrows():\n\n    df = {\n        'study_id' : [],\n        'condition' : [],\n        'level' : [], \n        'severity' : []\n    }\n\n    for col, val in row.items():\n        if col != 'study_id':\n            parts = col.split('_')\n            condition = ' '.join([word.capitalize() for word in parts[0:-2]])\n            level = f\"{parts[-2].capitalize()}/{parts[-1].capitalize()}\"\n\n            df['study_id'].append(row['study_id'])\n            df['condition'].append(condition)\n            df['level'].append(level)\n            df['severity'].append(val)\n\n    structured_df.append(pd.DataFrame(df))\n\nstructured_df = pd.concat(structured_df, ignore_index=True)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T18:11:37.049822Z","iopub.execute_input":"2025-01-30T18:11:37.050054Z","iopub.status.idle":"2025-01-30T18:11:38.094968Z","shell.execute_reply.started":"2025-01-30T18:11:37.050032Z","shell.execute_reply":"2025-01-30T18:11:38.094307Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"merged_df = pd.merge(structured_df, label_df, on=['study_id', 'condition', 'level'], how='inner')\n\nmerged_df_2 = pd.merge(merged_df, img_path_df, on=['series_id', 'study_id'], how='inner')\n\nfinal_df = pd.merge(merged_df_2, train_description_df, on=['series_id', 'study_id'], how='inner')\n\n# Dropping rows with NaN values \nfinal_df = final_df.dropna()\nfinal_df = final_df.drop(columns=['x','y'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T18:11:38.096048Z","iopub.execute_input":"2025-01-30T18:11:38.096351Z","iopub.status.idle":"2025-01-30T18:11:39.051388Z","shell.execute_reply.started":"2025-01-30T18:11:38.096324Z","shell.execute_reply":"2025-01-30T18:11:39.050669Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_df['row_id'] = (\n    final_df['study_id'].astype(str) + '_' +\n    final_df['condition'].str.lower().str.replace(' ', '_') + '_' +\n    final_df['level'].str.lower().str.replace('/', '_')\n)\n\ntest_df['row_id'] = (\n    test_df['study_id'].astype(str) + '_' +\n    test_df['condition'].str.lower().str.replace(' ', '_') + '_' +\n    test_df['level'].str.lower().str.replace('/', '_')\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T18:11:39.052321Z","iopub.execute_input":"2025-01-30T18:11:39.052572Z","iopub.status.idle":"2025-01-30T18:11:40.903557Z","shell.execute_reply.started":"2025-01-30T18:11:39.052547Z","shell.execute_reply":"2025-01-30T18:11:40.902634Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_df['severity'] = final_df['severity'].map({\n    'Normal/Mild': 'normal_mild', \n    'Moderate': 'moderate', \n    'Severe': 'severe'\n})\n\nencoder = LabelEncoder()\nfinal_df['condition'] = encoder.fit_transform(final_df['condition'])\nfinal_df['level'] = encoder.fit_transform(final_df['level'])\nfinal_df['series_description'] = encoder.fit_transform(final_df['series_description'])\n\ntest_df['series_description'] = encoder.fit_transform(test_df['series_description'])\ntest_df['condition'] = encoder.fit_transform(test_df['condition'])\ntest_df['level'] = encoder.fit_transform(test_df['level'])\n\nlabel_encoder = LabelEncoder()\nfinal_df['severity'] = label_encoder.fit_transform(final_df['severity'])\nfinal_df['severity'] = final_df['severity'].fillna('unknown') \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T18:11:40.904674Z","iopub.execute_input":"2025-01-30T18:11:40.904950Z","iopub.status.idle":"2025-01-30T18:11:41.619089Z","shell.execute_reply.started":"2025-01-30T18:11:40.904924Z","shell.execute_reply":"2025-01-30T18:11:41.618435Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# print(test_df.head(20))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T18:11:41.622466Z","iopub.execute_input":"2025-01-30T18:11:41.622946Z","iopub.status.idle":"2025-01-30T18:11:41.626394Z","shell.execute_reply.started":"2025-01-30T18:11:41.622918Z","shell.execute_reply":"2025-01-30T18:11:41.625482Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# !pip install pydicom","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T18:11:41.627313Z","iopub.execute_input":"2025-01-30T18:11:41.627550Z","iopub.status.idle":"2025-01-30T18:11:41.639233Z","shell.execute_reply.started":"2025-01-30T18:11:41.627519Z","shell.execute_reply":"2025-01-30T18:11:41.638426Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data, val_data = train_test_split(final_df, test_size=0.2, random_state=42)\n\nclass DicomDataGenerator(Sequence):\n    def __init__(self, df, label_encoder, metadata_columns, batch_size, image_size=(224, 224), shuffle=True):\n        self.df = df\n        self.batch_size = batch_size\n        self.label_encoder = label_encoder\n        self.metadata_columns = metadata_columns\n        self.image_size = image_size\n        self.shuffle = shuffle\n        self.indices = np.arange(len(self.df))  # Indices for batching\n        self.on_epoch_end()  # Initialize the generator\n        \n    def __len__(self):\n        \"\"\"Returns the number of batches per epoch.\"\"\"\n        return int(np.ceil(len(self.df) / self.batch_size))\n\n    def __getitem__(self, index):\n        \"\"\"Generates one batch of data.\"\"\"\n        start = index * self.batch_size\n        end = min((index + 1) * self.batch_size, len(self.df))\n        batch_df = self.df.iloc[start:end]\n\n        if len(batch_df) < self.batch_size:\n            num_missing = self.batch_size - len(batch_df)\n            \n            if len(batch_df) > 0:\n                batch_df = pd.concat([batch_df, batch_df.sample(num_missing, replace=True)], ignore_index=True)\n            else:\n                raise StopIteration(\"Empty batch encountered; stopping iteration.\")\n\n        if start >= len(self.df):\n            raise StopIteration(\"No more data to yield.\")\n\n        images = []\n        metadata = []\n        labels = []\n\n        for _, row in batch_df.iterrows():\n            # Load DICOM image and resize it\n            img = self.load_dicom_image(str(row['images']))\n            images.append(img)\n\n            # Get the metadata (excluding 'images', 'severity', and 'row_id')\n            metadata.append(row[self.metadata_columns].values)\n            \n            # Encode severity label using the label encoder\n            \n            if 'severity' in batch_df.columns:\n                labels.append(row['severity'])\n\n        # Convert to NumPy arrays\n        images = np.array(images, dtype='float32')\n        metadata = np.array(metadata, dtype='float32')\n    \n        # If labels are available, return them\n        if labels:            \n            labels = to_categorical(np.array(labels, dtype='int32'), num_classes=len(self.label_encoder.classes_))\n            return (images, metadata), labels\n        \n        # Otherwise, return only input data\n        return ((images, metadata),)\n\n    def on_epoch_end(self):\n        \"\"\"Shuffles the data after each epoch if shuffle is True.\"\"\"\n        if self.shuffle:\n            np.random.shuffle(self.indices)\n\n    def load_dicom_image(self, image_path):\n        \"\"\"Load and preprocess the DICOM image.\"\"\"\n        dicom_data = pydicom.dcmread(image_path)\n        img = dicom_data.pixel_array\n        img = cv2.resize(img, self.image_size)\n        img = np.expand_dims(img, axis=-1)  # Add channel dimension (for grayscale images)\n        img = img.astype('float32') / np.max(img)  # Normalize to [0, 1]\n        img_rgb = np.repeat(img, 3, axis=-1)  # Convert grayscale to RGB (224, 224, 3)\n        return img_rgb\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T18:11:41.640483Z","iopub.execute_input":"2025-01-30T18:11:41.640736Z","iopub.status.idle":"2025-01-30T18:11:42.071561Z","shell.execute_reply.started":"2025-01-30T18:11:41.640711Z","shell.execute_reply":"2025-01-30T18:11:42.070518Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define your metadata columns (excluding 'images', 'severity', and 'row_id')\nmetadata_columns = final_df.columns.difference(['images', 'severity', 'row_id'])\ntest_meta_columns = test_df.columns.difference(['images', 'severity', 'row_id'])\n\n# Define the generator for training and validation\ntrain_generator = DicomDataGenerator(train_data, batch_size=128, label_encoder=label_encoder, metadata_columns=metadata_columns)\nval_generator = DicomDataGenerator(val_data, batch_size=128, label_encoder=label_encoder, metadata_columns=metadata_columns, shuffle=False)\ntest_generator = DicomDataGenerator(test_df, batch_size=128, label_encoder=label_encoder, metadata_columns=test_meta_columns, shuffle=False)\n\nnum_classes = 3\n\n# Correct the generator calls by removing the unnecessary argument\ntrain_dataset = tf.data.Dataset.from_generator(\n    lambda: train_generator,  \n    output_signature=(\n        (tf.TensorSpec(shape=(128, 224, 224, 3), dtype=tf.float32),\n         tf.TensorSpec(shape=(128, len(metadata_columns)), dtype=tf.float32)),\n        tf.TensorSpec(shape=(128, num_classes), dtype=tf.int32)\n    )\n)\n\nval_dataset = tf.data.Dataset.from_generator(\n    lambda: val_generator,  \n    output_signature=(\n        (tf.TensorSpec(shape=(128, 224, 224, 3), dtype=tf.float32),\n         tf.TensorSpec(shape=(128, len(metadata_columns)), dtype=tf.float32)),\n        tf.TensorSpec(shape=(128, num_classes), dtype=tf.int32)\n    )\n)\n\ntest_dataset = tf.data.Dataset.from_generator(\n    lambda: test_generator, \n    output_signature=(\n        (tf.TensorSpec(shape=(128, 224, 224, 3), dtype=tf.float32),\n         tf.TensorSpec(shape=(128, len(test_meta_columns)), dtype=tf.float32)),\n    )\n)\n\nsteps_per_epoch = len(train_generator) // 128\nvalidation_steps = len(train_generator) // 128\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T18:11:42.072674Z","iopub.execute_input":"2025-01-30T18:11:42.072943Z","iopub.status.idle":"2025-01-30T18:11:42.894123Z","shell.execute_reply.started":"2025-01-30T18:11:42.072918Z","shell.execute_reply":"2025-01-30T18:11:42.893186Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import kagglehub\n\n# # Download latest version\n# path = kagglehub.model_download(\"google/efficientnet-v2/tensorFlow2/imagenet1k-b0-classification\")\n\n# print(\"Path to model files:\", path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T18:11:42.895083Z","iopub.execute_input":"2025-01-30T18:11:42.895356Z","iopub.status.idle":"2025-01-30T18:11:42.898933Z","shell.execute_reply.started":"2025-01-30T18:11:42.895330Z","shell.execute_reply":"2025-01-30T18:11:42.898170Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nweight_ = '/kaggle/input/densenet-keras/DenseNet-BC-121-32-no-top.h5'\n\n# Image Input\nimage_input = Input(shape=(224, 224, 3), name='image_input')\ndensenet = DenseNet121(include_top=False, weights=weight_, input_tensor=image_input)\nx = densenet.output\nx = GlobalAveragePooling2D()(x)\nx = Flatten()(x)\n\n# Metadata Input\nmetadata_input = Input(shape=(final_df.shape[1] - 3,), name='metadata_input')\n\n# Metadata Processing\ny = Dense(64, activation='relu')(metadata_input)\ny = Dense(64, activation='relu')(y)\ny = Dense(64, activation='relu')(y)\ny = Dropout(0.5)(y)\n\n# Merge Both Paths\nmerged = Concatenate()([x, y])\n\n# Final Classification Layers\nz = Dense(512, activation='relu')(merged)\nz = Dropout(0.5)(z)\nz = Dense(224, activation='relu')(z)\nz = Dropout(0.5)(z)\nz = Dense(len(label_encoder.classes_), activation='softmax')(z)\n\n# Create and Compile the Model\nmodel = Model(inputs=[image_input, metadata_input], outputs=z)\nmodel.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T18:11:42.900098Z","iopub.execute_input":"2025-01-30T18:11:42.900441Z","iopub.status.idle":"2025-01-30T18:11:46.042975Z","shell.execute_reply.started":"2025-01-30T18:11:42.900405Z","shell.execute_reply":"2025-01-30T18:11:46.042270Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"history = model.fit(\n    train_dataset,\n    validation_data=val_dataset,\n    epochs=20,\n    steps_per_epoch=steps_per_epoch,\n    validation_steps=validation_steps\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T18:11:46.043976Z","iopub.execute_input":"2025-01-30T18:11:46.044237Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predictions = model.predict(test_dataset, batch_size=128)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_df = {\n    'row_id' :[],\n    'normal_mild' :[],\n    'moderate' :[],\n    'severe' :[]\n}\n\nfor r, val in test_df.iterrows():\n    submission_df['row_id'].append(val['row_id'])\n    submission_df['normal_mild'].append(predictions[r][0])\n    submission_df['moderate'].append(predictions[r][1])\n    submission_df['severe'].append(predictions[r][2])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.DataFrame(submission_df)\n# [submission['row_id']=='44036939_left_neural_foraminal_narrowing_l2_l3']['normal_mild'].mean()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = submission.groupby('row_id')[['normal_mild', 'moderate', 'severe']].mean().reset_index()\nsubmission.head(25)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}