{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":130014,"sourceType":"modelInstanceVersion","modelInstanceId":109558,"modelId":133854}],"dockerImageVersionId":30732,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"'''\n# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n'''","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-14T14:06:01.594256Z","iopub.execute_input":"2024-10-14T14:06:01.594648Z","iopub.status.idle":"2024-10-14T14:06:01.602825Z","shell.execute_reply.started":"2024-10-14T14:06:01.594620Z","shell.execute_reply":"2024-10-14T14:06:01.601937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport glob\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport cv2\nimport warnings\nimport time\nimport json\nimport collections\nimport pydicom as dicom\nimport matplotlib.patches as patches\n\n\nfrom tqdm import tqdm\nfrom matplotlib import animation, rc\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:01.604668Z","iopub.execute_input":"2024-10-14T14:06:01.605074Z","iopub.status.idle":"2024-10-14T14:06:01.613370Z","shell.execute_reply.started":"2024-10-14T14:06:01.605043Z","shell.execute_reply":"2024-10-14T14:06:01.612570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loading the data\n\ndir_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/'  # Directory path\n\ntrain_df = pd.read_csv(dir_path + 'train.csv')   # Reading train CSV file \ntrain_label = pd.read_csv(dir_path + 'train_label_coordinates.csv')   # Reading LABEL CSV file for train \ntrain_desc = pd.read_csv(dir_path + 'train_series_descriptions.csv')  # Reading Train Description CSV file (Meta Data)\ntest_desc = pd.read_csv(dir_path + 'test_series_descriptions.csv')    # Reading VALIDATION Description CSV file (Meta Data)\n\nsubmi = pd.read_csv(dir_path + 'sample_submission.csv')     # Reading Sample Submission file for reference","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:01.614371Z","iopub.execute_input":"2024-10-14T14:06:01.614691Z","iopub.status.idle":"2024-10-14T14:06:01.784497Z","shell.execute_reply.started":"2024-10-14T14:06:01.614661Z","shell.execute_reply":"2024-10-14T14:06:01.783688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Length of the train.csv file = ',len(train_df))\ntrain_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:01.798000Z","iopub.execute_input":"2024-10-14T14:06:01.798290Z","iopub.status.idle":"2024-10-14T14:06:01.834871Z","shell.execute_reply.started":"2024-10-14T14:06:01.798264Z","shell.execute_reply":"2024-10-14T14:06:01.834046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n# List out all of the Studies we have on patients.\npart_1 = os.listdir('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images')\npart_1 = list(filter(lambda x: x.find('.DS') == -1, part_1))\n#part_1\n'''","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:01.836516Z","iopub.execute_input":"2024-10-14T14:06:01.837126Z","iopub.status.idle":"2024-10-14T14:06:01.842421Z","shell.execute_reply.started":"2024-10-14T14:06:01.837093Z","shell.execute_reply":"2024-10-14T14:06:01.841529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Length of the Training LABELS = ', len(train_label))\ntrain_label.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:02.003108Z","iopub.execute_input":"2024-10-14T14:06:02.003354Z","iopub.status.idle":"2024-10-14T14:06:02.016723Z","shell.execute_reply.started":"2024-10-14T14:06:02.003333Z","shell.execute_reply":"2024-10-14T14:06:02.015686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Descripton of the train data\nprint('Length of train description file = ', len(train_desc))\ntrain_desc","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:02.201208Z","iopub.execute_input":"2024-10-14T14:06:02.201487Z","iopub.status.idle":"2024-10-14T14:06:02.212495Z","shell.execute_reply.started":"2024-10-14T14:06:02.201464Z","shell.execute_reply":"2024-10-14T14:06:02.211619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation csv file \ntest_desc ","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:02.397559Z","iopub.execute_input":"2024-10-14T14:06:02.397844Z","iopub.status.idle":"2024-10-14T14:06:02.407516Z","shell.execute_reply.started":"2024-10-14T14:06:02.397819Z","shell.execute_reply":"2024-10-14T14:06:02.406400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('This is the sample submission')\nsubmi","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:02.594671Z","iopub.execute_input":"2024-10-14T14:06:02.594973Z","iopub.status.idle":"2024-10-14T14:06:02.609825Z","shell.execute_reply.started":"2024-10-14T14:06:02.594942Z","shell.execute_reply":"2024-10-14T14:06:02.608899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to generate image paths using list comprehension\ndef generate_image_paths(df, data_dir):\n    return df.apply(lambda row: [os.path.join(data_dir, str(row['study_id']), str(row['series_id']), img) \n                                 for img in os.listdir(os.path.join(data_dir, str(row['study_id']), str(row['series_id'])))], axis=1).explode().tolist()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:02.611647Z","iopub.execute_input":"2024-10-14T14:06:02.611914Z","iopub.status.idle":"2024-10-14T14:06:02.618478Z","shell.execute_reply.started":"2024-10-14T14:06:02.611890Z","shell.execute_reply":"2024-10-14T14:06:02.617465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generate image paths for train and test data\ntrain_img_paths = generate_image_paths(train_desc, f'{dir_path}train_images')\ntest_img_paths = generate_image_paths(test_desc, f'{dir_path}test_images')","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:02.619596Z","iopub.execute_input":"2024-10-14T14:06:02.619883Z","iopub.status.idle":"2024-10-14T14:06:57.028043Z","shell.execute_reply.started":"2024-10-14T14:06:02.619858Z","shell.execute_reply":"2024-10-14T14:06:57.027053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"The number of TRAINING images are = \",len(train_img_paths))\nprint(\"The number of TEST images are = \",len(test_img_paths))","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:57.030513Z","iopub.execute_input":"2024-10-14T14:06:57.031213Z","iopub.status.idle":"2024-10-14T14:06:57.036065Z","shell.execute_reply.started":"2024-10-14T14:06:57.031178Z","shell.execute_reply":"2024-10-14T14:06:57.035220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nprint(train_img_paths[113])","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:57.037091Z","iopub.execute_input":"2024-10-14T14:06:57.037323Z","iopub.status.idle":"2024-10-14T14:06:57.048530Z","shell.execute_reply.started":"2024-10-14T14:06:57.037302Z","shell.execute_reply":"2024-10-14T14:06:57.047662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to open an image\ndef show_dicom(path):\n    img = dicom.dcmread(path)\n\n    plt.imshow(img.pixel_array,cmap = 'gray')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:57.049356Z","iopub.execute_input":"2024-10-14T14:06:57.049669Z","iopub.status.idle":"2024-10-14T14:06:57.061290Z","shell.execute_reply.started":"2024-10-14T14:06:57.049647Z","shell.execute_reply":"2024-10-14T14:06:57.060584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_img = show_dicom(train_img_paths[103])","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:57.062210Z","iopub.execute_input":"2024-10-14T14:06:57.062539Z","iopub.status.idle":"2024-10-14T14:06:57.354191Z","shell.execute_reply.started":"2024-10-14T14:06:57.062494Z","shell.execute_reply":"2024-10-14T14:06:57.353247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_1 = dicom.dcmread(train_img_paths[501])\nval = np.array(img_1.pixel_array)\nval.shape\n#val.astype(np.float64)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:57.355300Z","iopub.execute_input":"2024-10-14T14:06:57.355663Z","iopub.status.idle":"2024-10-14T14:06:57.379419Z","shell.execute_reply.started":"2024-10-14T14:06:57.355631Z","shell.execute_reply":"2024-10-14T14:06:57.378610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to open and display DICOM images\ndef display_dicom_images(image_paths):\n    plt.figure(figsize=(15, 5))  # Adjust figure size if needed\n    for i, path in enumerate(image_paths[:5]):\n        ds = dicom.dcmread(path)\n        plt.subplot(1, 5, i+1)\n        plt.imshow(ds.pixel_array, cmap=plt.cm.bone)\n        plt.title(f\"Image {i+1}\")\n        plt.axis('off')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:57.380323Z","iopub.execute_input":"2024-10-14T14:06:57.380613Z","iopub.status.idle":"2024-10-14T14:06:57.386468Z","shell.execute_reply.started":"2024-10-14T14:06:57.380590Z","shell.execute_reply":"2024-10-14T14:06:57.385466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_dicom_images(train_img_paths)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:57.389849Z","iopub.execute_input":"2024-10-14T14:06:57.390122Z","iopub.status.idle":"2024-10-14T14:06:58.249144Z","shell.execute_reply.started":"2024-10-14T14:06:57.390099Z","shell.execute_reply":"2024-10-14T14:06:58.248235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load DICOM files from a folder\ndef load_dicom_files(path_to_folder):\n    files = [os.path.join(path_to_folder, f) for f in os.listdir(path_to_folder) if f.endswith('.dcm')]\n    files.sort(key=lambda x: int(os.path.splitext(os.path.basename(x))[0].split('-')[-1]))\n    return files","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:58.250287Z","iopub.execute_input":"2024-10-14T14:06:58.250590Z","iopub.status.idle":"2024-10-14T14:06:58.256644Z","shell.execute_reply.started":"2024-10-14T14:06:58.250565Z","shell.execute_reply":"2024-10-14T14:06:58.255582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preprocessing","metadata":{}},{"cell_type":"code","source":"#Define function to reshape a single row of the DataFrame\ndef reshape_row(row):\n    data = {'study_id':[], 'condition':[], 'level':[], 'severity':[]}\n    for column, value in row.items():\n        if column not in ['study_id', 'series_id', 'instance_number', 'x', 'y', 'series_description']:\n            parts = column.split('_')\n            condition = ' '.join([word.capitalize() for word in parts[:-2]])\n            level = parts[-2].capitalize() + '/' + parts[-1].capitalize()\n            data['study_id'].append(row['study_id'])\n            data['condition'].append(condition)\n            data['level'].append(level)\n            data['severity'].append(value)\n    \n    return pd.DataFrame(data)\n\n#Reshape the DataFrame for all rows\nnew_train_df = pd.concat([reshape_row(row) for _, row in train_df.iterrows()], ignore_index = True)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:58.257745Z","iopub.execute_input":"2024-10-14T14:06:58.258010Z","iopub.status.idle":"2024-10-14T14:06:59.567569Z","shell.execute_reply.started":"2024-10-14T14:06:58.257988Z","shell.execute_reply":"2024-10-14T14:06:59.566559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display the first few rows of the reshaped dataframe\nnew_train_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:59.568767Z","iopub.execute_input":"2024-10-14T14:06:59.569042Z","iopub.status.idle":"2024-10-14T14:06:59.579265Z","shell.execute_reply.started":"2024-10-14T14:06:59.569019Z","shell.execute_reply":"2024-10-14T14:06:59.578392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Print columns in a neat way\nprint(\"\\nColumns in new_train_df:\")\nprint(\",\".join(new_train_df.columns))\n\nprint(\"\\nColumns in label:\")\nprint(\",\".join(train_label.columns))\n\nprint(\"\\nColumns in test_desc:\")\nprint(\",\".join(test_desc.columns))\n\nprint(\"\\nColumns in sub:\")\nprint(\",\".join(submi.columns))","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:59.580548Z","iopub.execute_input":"2024-10-14T14:06:59.580864Z","iopub.status.idle":"2024-10-14T14:06:59.590207Z","shell.execute_reply.started":"2024-10-14T14:06:59.580839Z","shell.execute_reply":"2024-10-14T14:06:59.589281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge the dataframes on the common columns\nmerged_DF = pd.merge(new_train_df, train_label, on=['study_id', 'condition', 'level'], how = 'inner')\n\n# Merge the dataframes on the common column 'series_id'\nfinal_merged_df = pd.merge(merged_DF, train_desc, on='series_id', how = 'inner')","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:59.591164Z","iopub.execute_input":"2024-10-14T14:06:59.591453Z","iopub.status.idle":"2024-10-14T14:06:59.673397Z","shell.execute_reply.started":"2024-10-14T14:06:59.591409Z","shell.execute_reply":"2024-10-14T14:06:59.672656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge the dataframes on the common column 'Series_id'\nfinal_merged_DF = pd.merge(merged_DF, train_desc, on =['series_id', 'study_id'], how = 'inner')\nfinal_merged_DF.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:59.674420Z","iopub.execute_input":"2024-10-14T14:06:59.674731Z","iopub.status.idle":"2024-10-14T14:06:59.700533Z","shell.execute_reply.started":"2024-10-14T14:06:59.674705Z","shell.execute_reply":"2024-10-14T14:06:59.699631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_merged_DF[final_merged_DF['study_id'] == 100206310].sort_values(['x','y'], ascending = True)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:59.701593Z","iopub.execute_input":"2024-10-14T14:06:59.701853Z","iopub.status.idle":"2024-10-14T14:06:59.725169Z","shell.execute_reply.started":"2024-10-14T14:06:59.701830Z","shell.execute_reply":"2024-10-14T14:06:59.724322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_merged_DF[final_merged_DF['series_id'] == 1012284084].sort_values(\"instance_number\")","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:59.726124Z","iopub.execute_input":"2024-10-14T14:06:59.726396Z","iopub.status.idle":"2024-10-14T14:06:59.742385Z","shell.execute_reply.started":"2024-10-14T14:06:59.726373Z","shell.execute_reply":"2024-10-14T14:06:59.741592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, we can see what the data represents,\nseries ID 1012284084 contains 60 images, and how each image maps to each level and condition.","metadata":{}},{"cell_type":"code","source":"# Filter the dataframe for the given study_id and sort by instance_number\nfiltered_DF = final_merged_DF[final_merged_DF['study_id'] == 1013589491].sort_values(\"instance_number\")\nfiltered_DF","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:59.743400Z","iopub.execute_input":"2024-10-14T14:06:59.743681Z","iopub.status.idle":"2024-10-14T14:06:59.764798Z","shell.execute_reply.started":"2024-10-14T14:06:59.743653Z","shell.execute_reply":"2024-10-14T14:06:59.763797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sort the final_merged_DF by study_id, series_id and series_description\nsorted_final_merged_DF = final_merged_DF[final_merged_DF['study_id'] == 1013589491].sort_values(by = ['series_id', 'series_description', 'instance_number'])\nsorted_final_merged_DF","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:59.766134Z","iopub.execute_input":"2024-10-14T14:06:59.766528Z","iopub.status.idle":"2024-10-14T14:06:59.789989Z","shell.execute_reply.started":"2024-10-14T14:06:59.766495Z","shell.execute_reply":"2024-10-14T14:06:59.789166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that,\n\nsaggital T1 images map to Neural Foraminal Narrowing\nAxial T2 images map to subarticular Stenosis\nSaggital T2/STIR map to Canal Stenosis","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\n# create the row_id column\nfinal_merged_DF['row_id'] = (final_merged_DF['study_id'].astype(str)+'_'+final_merged_DF['condition'].str.lower().str.replace(' ','_')+'_'+final_merged_DF['level'].str.lower().str.replace('/','_'))\n\n# Create the image_path column\nfinal_merged_DF['image_path'] = (f'{dir_path}train_images/' + final_merged_DF['study_id'].astype(str)+'/' + final_merged_DF['series_id'].astype(str) + '/' + final_merged_DF['instance_number'].astype(str) + '.dcm')\n\n\n# Display the updated dataframe\nfinal_merged_DF.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:06:59.791242Z","iopub.execute_input":"2024-10-14T14:06:59.791565Z","iopub.status.idle":"2024-10-14T14:07:00.027320Z","shell.execute_reply.started":"2024-10-14T14:06:59.791541Z","shell.execute_reply":"2024-10-14T14:07:00.026388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Note:\nCheck image path, since there's 1 instance id, for 1 image, but there's many more images other than the ones labelled in the instance ID.","metadata":{}},{"cell_type":"code","source":"final_merged_DF[final_merged_DF[\"severity\"] == \"Normal/Mild\"].value_counts().sum()","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:00.028579Z","iopub.execute_input":"2024-10-14T14:07:00.028892Z","iopub.status.idle":"2024-10-14T14:07:00.177855Z","shell.execute_reply.started":"2024-10-14T14:07:00.028866Z","shell.execute_reply":"2024-10-14T14:07:00.176948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_merged_DF[final_merged_DF[\"severity\"] == \"Moderate\"].value_counts().sum()","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:00.178991Z","iopub.execute_input":"2024-10-14T14:07:00.179273Z","iopub.status.idle":"2024-10-14T14:07:00.227186Z","shell.execute_reply.started":"2024-10-14T14:07:00.179247Z","shell.execute_reply":"2024-10-14T14:07:00.226374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the base path for test images\nbase_path = dir_path + 'test_images/'\n\n# Function to get image paths for a series\ndef get_img_paths(row):\n    series_path = os.path.join(base_path, str(row['study_id']), str(row['series_id']))\n    if os.path.exists(series_path):\n        return[os.path.join(series_path, f) for f in os.listdir(series_path) if os.path.isfile(os.path.join(series_path, f))]\n    return []\n\n# Mapping of series_description to conditions\ncondition_mapping = {'Sagittal T1': {'left': 'left_neural_foraminal_narrowing', 'right': 'right_neural_forminal_narrowing'}, 'Axial T2': {'left':'left_subarticular_stenosis' , 'right':'right_subarticular_stenosis'}, 'Sagittal T2/STIR': 'spinal_canal_stenosis'}\n\n# Create a list to store the expanded rows\nexpanded_rows = []\n\n# Expand the dataframe by adding new rows for each file path\nfor index, row in test_desc.iterrows():\n    img_paths = get_img_paths(row)\n    conditions = condition_mapping.get(row['series_description'],{})\n    if isinstance(conditions, str): # Single condition\n        conditions = {'left': conditions, 'right': conditions}\n    \n    for side, condition in conditions.items():\n        for img_path in img_paths:\n            expanded_rows.append({'study_id': row['study_id'], 'series_id': row['series_id'], 'series_description': row['series_description'], 'image_path': img_path, 'condition': condition, 'row_id': f\"{row['study_id']}_{condition}\"})\n            \n# Create a new dataframe from the expanded rows\nexpanded_test_desc = pd.DataFrame(expanded_rows)\n\nexpanded_test_desc.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:00.228316Z","iopub.execute_input":"2024-10-14T14:07:00.228934Z","iopub.status.idle":"2024-10-14T14:07:00.296029Z","shell.execute_reply.started":"2024-10-14T14:07:00.228898Z","shell.execute_reply":"2024-10-14T14:07:00.295147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Change severity column lables.\n# Normal/Mild to 'normal_mild', Moderate to 'moderate', Severe to 'severe'.\nfinal_merged_DF['severity'] = final_merged_DF['severity'].map({'Normal/Mild':'normal_mild', 'Moderate':'moderate', 'Severe':'severe'})","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:00.297144Z","iopub.execute_input":"2024-10-14T14:07:00.297394Z","iopub.status.idle":"2024-10-14T14:07:00.307963Z","shell.execute_reply.started":"2024-10-14T14:07:00.297372Z","shell.execute_reply":"2024-10-14T14:07:00.307115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_merged_DF['severity']","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:00.308887Z","iopub.execute_input":"2024-10-14T14:07:00.309157Z","iopub.status.idle":"2024-10-14T14:07:00.321051Z","shell.execute_reply.started":"2024-10-14T14:07:00.309125Z","shell.execute_reply":"2024-10-14T14:07:00.320247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = expanded_test_desc\ntrain_data = final_merged_DF","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:00.322097Z","iopub.execute_input":"2024-10-14T14:07:00.322357Z","iopub.status.idle":"2024-10-14T14:07:00.333214Z","shell.execute_reply.started":"2024-10-14T14:07:00.322335Z","shell.execute_reply":"2024-10-14T14:07:00.332439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data['series_description'].unique()\ntest_data","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:00.341622Z","iopub.execute_input":"2024-10-14T14:07:00.341931Z","iopub.status.idle":"2024-10-14T14:07:00.355225Z","shell.execute_reply.started":"2024-10-14T14:07:00.341910Z","shell.execute_reply":"2024-10-14T14:07:00.354226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\n# Define a function to check if the path exists \ndef check_exists(path):\n    return os.path.exists(path)\n\n# Check if the stydy id exists by a function\ndef check_study_id(row):\n    study_id = row['study_id']\n    path = f'{dir_path}/train_images/{study_id}'\n    return check_exists(path)\n\n# Function to check if a series ID directory exists\ndef check_series_id(row):\n    study_id = row['study_id']\n    series_id = row['series_id']\n    path = f'{dir_path}/train_images/{study_id}/{series_id}'\n    return check_exists(path)\n\n# Function to check if an image exists\ndef check_image_exists(row):\n    image_path = row['image_path']\n    return check_exists(image_path)\n\n# Applying the functions to the train_data frame\ntrain_data['study_id_exists'] = train_data.apply(check_study_id, axis = 1)\ntrain_data['series_id_exists'] = train_data.apply(check_series_id, axis = 1)\ntrain_data['image_exists'] = train_data.apply(check_image_exists, axis = 1)\n\n# Filter train data\ntrain_data = train_data[(train_data['study_id_exists']) & (train_data['series_id_exists']) & (train_data['image_exists'])]","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:00.356498Z","iopub.execute_input":"2024-10-14T14:07:00.356891Z","iopub.status.idle":"2024-10-14T14:07:18.116321Z","shell.execute_reply.started":"2024-10-14T14:07:00.356860Z","shell.execute_reply":"2024-10-14T14:07:18.115230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:18.117697Z","iopub.execute_input":"2024-10-14T14:07:18.118499Z","iopub.status.idle":"2024-10-14T14:07:18.133545Z","shell.execute_reply.started":"2024-10-14T14:07:18.118464Z","shell.execute_reply":"2024-10-14T14:07:18.132693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data['series_description'].unique()","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:18.134807Z","iopub.execute_input":"2024-10-14T14:07:18.135691Z","iopub.status.idle":"2024-10-14T14:07:18.150184Z","shell.execute_reply.started":"2024-10-14T14:07:18.135663Z","shell.execute_reply":"2024-10-14T14:07:18.149450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dicom(path):\n    dcm = dicom.read_file(path)\n    data = dcm.pixel_array\n    data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data/np.max(data)\n    #data = (data*255).astype(np.uint8)\n    return data","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:18.151325Z","iopub.execute_input":"2024-10-14T14:07:18.151622Z","iopub.status.idle":"2024-10-14T14:07:18.158153Z","shell.execute_reply.started":"2024-10-14T14:07:18.151599Z","shell.execute_reply":"2024-10-14T14:07:18.157438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load images randomly \nimport random \nimages = []\nrow_ids = []\nselected_indices = random.sample(range(len(train_data)),2)\nfor i in selected_indices:\n    image = load_dicom(train_data['image_path'][i])\n    images.append(image)\n    row_ids.append(train_data['row_id'][i])\n    \n# Plot images\nfig, ax = plt.subplots(1, 2, figsize = (8,4))\nfor i in range(2):    \n    ax[i].imshow(images[i],cmap='gray') #plt.cm.bone\n    ax[i].set_title(f'Row ID:{row_ids[i]}', fontsize = 8)\n    ax[i].axis('off')\nplt.tight_layout()\nplt.show()    ","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:18.159168Z","iopub.execute_input":"2024-10-14T14:07:18.159454Z","iopub.status.idle":"2024-10-14T14:07:18.488173Z","shell.execute_reply.started":"2024-10-14T14:07:18.159424Z","shell.execute_reply":"2024-10-14T14:07:18.487326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nulls = train_data.isnull().sum()\nprint(f'The NULL or BLANK values in the final dataset by column-wise \\n{nulls}')","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:18.489475Z","iopub.execute_input":"2024-10-14T14:07:18.489829Z","iopub.status.idle":"2024-10-14T14:07:18.524277Z","shell.execute_reply.started":"2024-10-14T14:07:18.489797Z","shell.execute_reply":"2024-10-14T14:07:18.523448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Removed the null values\ntrain_data_set = train_data.dropna()\nprint(type(train_data_set), len(train_data_set))\ntrain_data_set.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:18.525600Z","iopub.execute_input":"2024-10-14T14:07:18.525989Z","iopub.status.idle":"2024-10-14T14:07:18.583167Z","shell.execute_reply.started":"2024-10-14T14:07:18.525950Z","shell.execute_reply":"2024-10-14T14:07:18.582320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"exist_img = train_data_set['condition'].unique()\nexist_img","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:18.584303Z","iopub.execute_input":"2024-10-14T14:07:18.584666Z","iopub.status.idle":"2024-10-14T14:07:18.595718Z","shell.execute_reply.started":"2024-10-14T14:07:18.584634Z","shell.execute_reply":"2024-10-14T14:07:18.594688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filtering DataFrame by series (condition) description\nsagittal_t2_stir_data = train_data_set[train_data_set['series_description'] == 'Sagittal T2/STIR']\nsagittal_t1_data = train_data_set[train_data_set['series_description'] == 'Sagittal T1']\naxial_t2_data = train_data_set[train_data_set['series_description'] == 'Axial T2']","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:18.596961Z","iopub.execute_input":"2024-10-14T14:07:18.597266Z","iopub.status.idle":"2024-10-14T14:07:18.634108Z","shell.execute_reply.started":"2024-10-14T14:07:18.597241Z","shell.execute_reply":"2024-10-14T14:07:18.633251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"The length of the sagittal T2 / STIR data is\", len(sagittal_t2_stir_data))\nprint(\"Severity count of unique values\", sagittal_t2_stir_data['severity'].value_counts())\nsagittal_t2_stir_data","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:18.635342Z","iopub.execute_input":"2024-10-14T14:07:18.635730Z","iopub.status.idle":"2024-10-14T14:07:18.659352Z","shell.execute_reply.started":"2024-10-14T14:07:18.635696Z","shell.execute_reply":"2024-10-14T14:07:18.658476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"The length of the sagittal T2 / STIR data is\", len(sagittal_t1_data))\nprint(\"Severity count of unique values\", sagittal_t1_data['severity'].value_counts())\nsagittal_t1_data","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:18.660597Z","iopub.execute_input":"2024-10-14T14:07:18.660860Z","iopub.status.idle":"2024-10-14T14:07:18.688093Z","shell.execute_reply.started":"2024-10-14T14:07:18.660828Z","shell.execute_reply":"2024-10-14T14:07:18.687124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"The length of the sagittal T2 / STIR data is\", len(axial_t2_data))\nprint(\"Severity count of unique values\\n\",axial_t2_data['severity'].value_counts())\naxial_t2_data","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:18.689232Z","iopub.execute_input":"2024-10-14T14:07:18.689530Z","iopub.status.idle":"2024-10-14T14:07:18.714319Z","shell.execute_reply.started":"2024-10-14T14:07:18.689507Z","shell.execute_reply":"2024-10-14T14:07:18.713362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedShuffleSplit\n\n# Function to split the dataset proportionally for each type of severity\ndef split_dataset(df):\n\n    # Define the stratified shuffle split\n    strat_split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)\n\n    # Split the dataset\n    for train_idx, val_idx in strat_split.split(df, df['severity']):\n        df_train = df.iloc[train_idx].reset_index(drop=True)\n        df_val = df.iloc[val_idx].reset_index(drop=True)\n        \n    return df_train, df_val","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:18.715648Z","iopub.execute_input":"2024-10-14T14:07:18.715949Z","iopub.status.idle":"2024-10-14T14:07:18.723796Z","shell.execute_reply.started":"2024-10-14T14:07:18.715925Z","shell.execute_reply":"2024-10-14T14:07:18.722755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def split_check(df, df_train, df_val):\n    # Calculate percentage distribution\n    original_percent = (df['severity'].value_counts() / len(df)) * 100\n    train_percent = (df_train['severity'].value_counts() / len(df_train)) * 100\n    val_percent = (df_val['severity'].value_counts() / len(df_val)) * 100\n\n    # Print percentage distribution for comparison\n    print(\"Original Dataset Percentage Distribution:\\n\", original_percent)\n    print(\"Training Set Percentage Distribution:\\n\", train_percent)\n    print(\"Validation Set Percentage Distribution:\\n\", val_percent)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:18.725037Z","iopub.execute_input":"2024-10-14T14:07:18.725502Z","iopub.status.idle":"2024-10-14T14:07:18.734436Z","shell.execute_reply.started":"2024-10-14T14:07:18.725470Z","shell.execute_reply":"2024-10-14T14:07:18.733524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nsagittal_t2_stir_data\nsagittal_t1_data\naxial_t2_data\n'''\n# Splitting the datasets for training and validation\nsag_t2STIR_train, sag_t2STIR_val = split_dataset(sagittal_t2_stir_data)\nsag_t1_train, sag_t1_val = split_dataset(sagittal_t1_data)\naxi_t2_train, axi_t2_val = split_dataset(axial_t2_data)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:18.735562Z","iopub.execute_input":"2024-10-14T14:07:18.735835Z","iopub.status.idle":"2024-10-14T14:07:18.825961Z","shell.execute_reply.started":"2024-10-14T14:07:18.735812Z","shell.execute_reply":"2024-10-14T14:07:18.824979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checking the percentage of training and validation splits\nprint(split_check(sagittal_t2_stir_data, sag_t2STIR_train, sag_t2STIR_val),'sagittal_t2_stir_data SPLIT\\n')\nprint('sagittal_t1_data SPLIT\\n', split_check(sagittal_t1_data, sag_t1_train, sag_t1_val))\nprint('axial_t2_data SPLIT\\n', split_check(axial_t2_data, axi_t2_train, axi_t2_val))","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:18.827111Z","iopub.execute_input":"2024-10-14T14:07:18.827392Z","iopub.status.idle":"2024-10-14T14:07:18.854395Z","shell.execute_reply.started":"2024-10-14T14:07:18.827369Z","shell.execute_reply":"2024-10-14T14:07:18.853471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndef show_array(array,title=None):\n    plt.imshow(array,cmap = 'gray')\n    plt.title(title)\n    plt.show()\n'''    \ndef show_array(array, title=None, coordinates=None):\n    plt.imshow(array, cmap='gray')\n    if title:\n        plt.title(title)\n    if coordinates:\n        x,y = coordinates\n        plt.plot(x, y, 'ro')  # 'ro' means red color, circle marker\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:18.855673Z","iopub.execute_input":"2024-10-14T14:07:18.856000Z","iopub.status.idle":"2024-10-14T14:07:18.863261Z","shell.execute_reply.started":"2024-10-14T14:07:18.855972Z","shell.execute_reply":"2024-10-14T14:07:18.862582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"im_coor_x = axi_t2_train['x'][1337]\nim_coor_y = axi_t2_train['y'][1337]\ncoord = [(im_coor_x,im_coor_y)]\nprint(im_coor_x)\nprint(im_coor_y)\ncoord","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:18.864332Z","iopub.execute_input":"2024-10-14T14:07:18.864597Z","iopub.status.idle":"2024-10-14T14:07:18.876908Z","shell.execute_reply.started":"2024-10-14T14:07:18.864575Z","shell.execute_reply":"2024-10-14T14:07:18.876210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imt1 = load_dicom(sag_t1_train['image_path'][7359])  # sag_t2STIR_train, sag_t1_train, axi_t2_train\nprint(imt1.shape)\nshow_array(imt1)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:18.877925Z","iopub.execute_input":"2024-10-14T14:07:18.878149Z","iopub.status.idle":"2024-10-14T14:07:19.175979Z","shell.execute_reply.started":"2024-10-14T14:07:18.878129Z","shell.execute_reply":"2024-10-14T14:07:19.174621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"im_path = axi_t2_train['image_path'][1337]\nim_array = load_dicom(im_path)/255.0\nim_title = axi_t2_train['severity'][1337]\n#plt_array = show_array(im_array,im_title,coord)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:19.177213Z","iopub.execute_input":"2024-10-14T14:07:19.177574Z","iopub.status.idle":"2024-10-14T14:07:19.273420Z","shell.execute_reply.started":"2024-10-14T14:07:19.177542Z","shell.execute_reply":"2024-10-14T14:07:19.272640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"im_df = pd.DataFrame(im_array.squeeze())\n#im_df.style.background_gradient('Greys')","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:19.274583Z","iopub.execute_input":"2024-10-14T14:07:19.274868Z","iopub.status.idle":"2024-10-14T14:07:19.279806Z","shell.execute_reply.started":"2024-10-14T14:07:19.274844Z","shell.execute_reply":"2024-10-14T14:07:19.278867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading the data to feed into model","metadata":{}},{"cell_type":"code","source":"# Label Mapping - Should be applied in Dataset class\nlabel_map = {\n    'normal_mild': 0,\n    'moderate': 1,\n    'severe': 2  \n}","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:19.281131Z","iopub.execute_input":"2024-10-14T14:07:19.281522Z","iopub.status.idle":"2024-10-14T14:07:19.290254Z","shell.execute_reply.started":"2024-10-14T14:07:19.281480Z","shell.execute_reply":"2024-10-14T14:07:19.289455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\n\n# Custom Dataset with Coordinates\nclass CustomDatasetWithCoords(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        img_path = self.df.iloc[idx]['image_path']\n        label = self.df.iloc[idx]['severity']\n        x_coord = self.df.iloc[idx]['x']\n        y_coord = self.df.iloc[idx]['y']\n\n        # Load DICOM image as array (assuming load_dicom is defined)\n        image = load_dicom(img_path)\n\n        # Convert image to tensor if necessary\n        image_tensor = torch.tensor(image, dtype=torch.float32) / 255.0\n        \n        # If the image has 2 dimensions, add a channel dimension\n        if image_tensor.ndim == 2:\n            image_tensor = image_tensor.unsqueeze(0)\n            \n        # Get original image size (height, width)\n        original_size = image_tensor.shape[1:3]  # Assuming shape is (C, H, W)\n\n        # Apply transformations if any\n        if self.transform:\n            trans_image = self.transform(image_tensor)\n        else:\n            trans_image = image_tensor\n\n        # Get resized image size (after applying the Resize transformation)\n        new_size = trans_image.shape[1:3]  # (height, width) after transformation\n\n        # Calculate scaling factors if the image was resized\n        if new_size != original_size:\n            scaling_factor_x = new_size[1] / original_size[1]\n            scaling_factor_y = new_size[0] / original_size[0]\n\n            # Scale the coordinates\n            x_coord = x_coord * scaling_factor_x\n            y_coord = y_coord * scaling_factor_y\n\n        # Encoding label\n        label = label_map[label]  # Map string label to integer\n        label = torch.tensor(label, dtype=torch.long)\n        # Coordinates tensor\n        coords = torch.tensor([x_coord,y_coord], dtype=torch.float32)\n\n        return trans_image, coords, label  # Return image, coordinates, and label","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:19.291771Z","iopub.execute_input":"2024-10-14T14:07:19.292285Z","iopub.status.idle":"2024-10-14T14:07:19.303793Z","shell.execute_reply.started":"2024-10-14T14:07:19.292253Z","shell.execute_reply":"2024-10-14T14:07:19.302934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchvision import transforms\n# Defining the \"Transforms\"\nimage_transforms = transforms.Compose([\n    transforms.Resize((319, 319),antialias=True)\n])","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:19.305073Z","iopub.execute_input":"2024-10-14T14:07:19.305672Z","iopub.status.idle":"2024-10-14T14:07:19.318111Z","shell.execute_reply.started":"2024-10-14T14:07:19.305639Z","shell.execute_reply":"2024-10-14T14:07:19.317387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating datasets for the sagittal t2 STIR training and validation.\nsag_t2STIR_train_set = CustomDatasetWithCoords(sag_t2STIR_train,image_transforms)\nsag_t2STIR_val_set = CustomDatasetWithCoords(sag_t2STIR_val,image_transforms)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:19.319009Z","iopub.execute_input":"2024-10-14T14:07:19.319249Z","iopub.status.idle":"2024-10-14T14:07:19.328913Z","shell.execute_reply.started":"2024-10-14T14:07:19.319228Z","shell.execute_reply":"2024-10-14T14:07:19.328139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Access the first element of the dataset\nfirst_element = sag_t2STIR_train_set[0]\n\n# Access the coordinates tensor\ncoordinates_tensor = first_element[1]\n\n# Print the coordinates tensor\nprint(coordinates_tensor)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:19.330133Z","iopub.execute_input":"2024-10-14T14:07:19.330565Z","iopub.status.idle":"2024-10-14T14:07:19.421851Z","shell.execute_reply.started":"2024-10-14T14:07:19.330535Z","shell.execute_reply":"2024-10-14T14:07:19.420938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ten = sag_t2STIR_train_set[90][0]\ncoor= sag_t2STIR_train_set[90][1]\ntitle = sag_t2STIR_train_set[90][2]\nprint(ten,coor,title)\n#show_array(ten.squeeze(),title)\n#ten","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:19.422946Z","iopub.execute_input":"2024-10-14T14:07:19.423227Z","iopub.status.idle":"2024-10-14T14:07:19.473998Z","shell.execute_reply.started":"2024-10-14T14:07:19.423203Z","shell.execute_reply":"2024-10-14T14:07:19.473125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n# Stacking all the normal_mild images for t2STIR train \nstacked_normal_t2STIR = torch.stack([I_c[0] for I_c, label in sag_t2STIR_train_set if label == 'normal_mild'])\nstacked_moderate_t2STIR = torch.stack( [ I_c[0] for I_c,label in sag_t2STIR_train_set if label =='moderate'])\nstacked_severe_t2STIR = torch.stack( [ I_c[0] for I_c,label in sag_t2STIR_train_set if label =='severe'])\n'''","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:19.475205Z","iopub.execute_input":"2024-10-14T14:07:19.475652Z","iopub.status.idle":"2024-10-14T14:07:19.481813Z","shell.execute_reply.started":"2024-10-14T14:07:19.475619Z","shell.execute_reply":"2024-10-14T14:07:19.480989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nmean_normal_t2STIR = stacked_normal_t2STIR.mean(0)\nmean_moderate_t2STIR = stacked_moderate_t2STIR.mean(0)\nmean_severe_t2STIR = stacked_moderate_t2STIR.mean(0)\n'''","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:19.482915Z","iopub.execute_input":"2024-10-14T14:07:19.483228Z","iopub.status.idle":"2024-10-14T14:07:19.493950Z","shell.execute_reply.started":"2024-10-14T14:07:19.483199Z","shell.execute_reply":"2024-10-14T14:07:19.493089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"The train dataset length:\",len(sag_t2STIR_train_set) ,\"The validation dataset length:\",len(sag_t2STIR_val_set) )","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:19.494931Z","iopub.execute_input":"2024-10-14T14:07:19.495270Z","iopub.status.idle":"2024-10-14T14:07:19.504358Z","shell.execute_reply.started":"2024-10-14T14:07:19.495237Z","shell.execute_reply":"2024-10-14T14:07:19.503563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating a training and validation dataloaders for t2STIRs\ntrain_loader_t2STIR = DataLoader(sag_t2STIR_train_set,batch_size = 64, shuffle = True, num_workers = 2)\nvalid_loader_t2STIR = DataLoader(sag_t2STIR_val_set,batch_size = 64, shuffle = False, num_workers = 2)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:19.505376Z","iopub.execute_input":"2024-10-14T14:07:19.505687Z","iopub.status.idle":"2024-10-14T14:07:19.519780Z","shell.execute_reply.started":"2024-10-14T14:07:19.505662Z","shell.execute_reply":"2024-10-14T14:07:19.518946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, coordinates, labels = next(iter(train_loader_t2STIR))\ncoor = coordinates[7].tolist()\nprint(labels,type(labels))","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:19.520862Z","iopub.execute_input":"2024-10-14T14:07:19.521162Z","iopub.status.idle":"2024-10-14T14:07:22.923021Z","shell.execute_reply.started":"2024-10-14T14:07:19.521140Z","shell.execute_reply":"2024-10-14T14:07:22.921809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"coordinates","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:22.924482Z","iopub.execute_input":"2024-10-14T14:07:22.924788Z","iopub.status.idle":"2024-10-14T14:07:22.934066Z","shell.execute_reply.started":"2024-10-14T14:07:22.924759Z","shell.execute_reply":"2024-10-14T14:07:22.933222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_array(images[7].squeeze(),labels[7],coor)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:22.935289Z","iopub.execute_input":"2024-10-14T14:07:22.935756Z","iopub.status.idle":"2024-10-14T14:07:23.220613Z","shell.execute_reply.started":"2024-10-14T14:07:22.935723Z","shell.execute_reply":"2024-10-14T14:07:23.219689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating a Baseline model which is convolution network","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\n\n'''\nclass SimpleCNN(nn.Module):\n    def __init__(self, coord_size=None, dropout_prob=0.5):\n        super(SimpleCNN, self).__init__()\n        self.conv1 = nn.Conv2d(1, 32, kernel_size=5, padding=2)\n        self.bn1 = nn.BatchNorm2d(32)  # Batch Normalization for the first conv layer\n        self.conv2 = nn.Conv2d(32, 64, kernel_size=5, padding=2)\n        self.bn2 = nn.BatchNorm2d(64)  # Batch Normalization for the second conv layer\n        self.conv3 = nn.Conv2d(64, 128, kernel_size=5, padding=2)\n        self.bn3 = nn.BatchNorm2d(128)  # Batch Normalization for the third conv layer\n        \n        self.pool = nn.MaxPool2d(2, 2)\n        self.relu = nn.ReLU()  # ReLU activation function\n        self.dropout = nn.Dropout(dropout_prob)  # Dropout layer\n\n        # Flattened size after the convolutional layers\n        self.flattened_size = 128 * 39 * 39  # Adjust based on your image size and pooling\n        self.coord_size = coord_size if coord_size is not None else 0\n        \n        # Fully connected layers\n        self.fc1_with_coords = nn.Linear(self.flattened_size + self.coord_size, 256)\n        self.fc1_without_coords = nn.Linear(self.flattened_size, 256)\n        self.bn_fc = nn.BatchNorm1d(256)  # Batch Normalization for the fully connected layer\n        self.fc2 = nn.Linear(256, 3)\n\n    def forward(self, x, coords=None):\n        # Convolutional layers with Batch Normalization, ReLU, and Dropout\n        x = self.pool(self.relu(self.bn1(self.conv1(x))))\n        #x = self.dropout(x)  # Apply dropout after ReLU and before next conv layer\n        \n        x = self.pool(self.relu(self.bn2(self.conv2(x))))\n        #x = self.dropout(x)  # Dropout after second conv layer\n\n        x = self.pool(self.relu(self.bn3(self.conv3(x))))\n        x = self.dropout(x)  # Dropout after third conv layer\n\n        x = x.view(x.size(0), -1)  # Flatten\n\n        if coords is not None:\n            # Concatenate the coordinates and pass through fc1_with_coords\n            x = torch.cat((x, coords), dim=1)  # Concatenate image features and coordinates\n            x = self.relu(self.bn_fc(self.fc1_with_coords(x)))\n        else:\n            # If no coordinates, pass through fc1_without_coords\n            x = self.relu(self.bn_fc(self.fc1_without_coords(x)))\n\n        x = self.dropout(x)  # Dropout before the final fully connected layer\n        x = self.fc2(x)\n        return x\n'''","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:23.222017Z","iopub.execute_input":"2024-10-14T14:07:23.222455Z","iopub.status.idle":"2024-10-14T14:07:23.232935Z","shell.execute_reply.started":"2024-10-14T14:07:23.222422Z","shell.execute_reply":"2024-10-14T14:07:23.232017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\n'''\n# Define the CNN model\nclass SimpleCNN(nn.Module):\n    def __init__(self):\n        super(SimpleCNN, self).__init__()\n        self.conv1 = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=3, stride=1, padding=1)  # 1 channel for grayscale\n        self.bn1 = nn.BatchNorm2d(16)  # Batch Normalization\n        self.conv2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=3, stride=1, padding=1)\n        self.bn2 = nn.BatchNorm2d(32)\n        self.conv3 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3, stride=1, padding=1)\n        self.bn3 = nn.BatchNorm2d(64)\n        \n        self.fc1 = nn.Linear(64 * 39 * 39, 128)  # Fully connected layer\n        self.fc2 = nn.Linear(128, 3)  # Output for 3 classes\n        \n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = torch.relu(x)\n        x = torch.max_pool2d(x, 2)  # Pooling layer\n        \n        x = self.conv2(x)\n        x = self.bn2(x)\n        x = torch.relu(x)\n        x = torch.max_pool2d(x, 2)\n        \n        x = self.conv3(x)\n        x = self.bn3(x)\n        x = torch.relu(x)\n        x = torch.max_pool2d(x, 2)\n        \n        x = x.view(x.size(0), -1)  # Flatten the tensor\n        \n        x = torch.relu(self.fc1(x))\n        x = self.fc2(x)\n        return x\n'''","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:23.234087Z","iopub.execute_input":"2024-10-14T14:07:23.234330Z","iopub.status.idle":"2024-10-14T14:07:23.248787Z","shell.execute_reply.started":"2024-10-14T14:07:23.234309Z","shell.execute_reply":"2024-10-14T14:07:23.247877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n# Device configuration\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n       \n# Training loop\ndef train_model(model, optimizer, train_loader, valid_loader, num_epochs=10):\n    # Loss function and optimizer\n    criterion = nn.CrossEntropyLoss()\n    \n    # Training the model\n    for epoch in range(num_epochs):\n        model.train()  # Set the model to training mode\n        running_loss = 0.0\n        batch_idx = 0\n        \n        for batch in train_loader:\n            images, coords, labels = batch\n            images, labels = images.cuda(), labels.cuda()  # Move data to GPU if available\n\n            # Zero the parameter gradients\n            optimizer.zero_grad()\n            \n            # Forward pass\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n            \n            # Backward pass and optimization\n            loss.backward()\n            optimizer.step()  #optimizer.step()\n\n\n            # Print statistics\n            running_loss += loss.item()\n            if batch_idx % 10 == 9:  # Print every 10 batches\n                print(f'Epoch [{epoch+1}/{num_epochs}], Step [{batch_idx+1}/{len(train_loader)}], Loss: {running_loss / 10:.4f}')\n                running_loss = 0.0\n\n            \n            # Print statistics\n            running_loss += loss.item()\n            if batch_idx % 10 == 9:  # Print every 10 batches\n                print(f'Epoch [{epoch+1}/{num_epochs}], Step [{batch_idx+1}/{len(train_loader)}], Loss: {running_loss / 10:.4f}')\n                running_loss = 0.0\n            \n            batch_idx += 1\n        \n        # Print epoch statistics\n        print(f'Epoch [{epoch+1}/{num_epochs}] completed with average loss: {running_loss / len(train_loader):.4f}')\n            \n            \n        # Validation phase\n        model.eval()  # Set the model to evaluation mode\n        val_loss = 0.0\n        correct = 0\n        total = 0\n        with torch.no_grad():\n            for images, coords, labels in valid_loader:\n                images, labels = images.cuda(), labels.cuda()\n                outputs = model(images)\n                loss = criterion(outputs, labels)\n                val_loss += loss.item()\n                \n                # Calculate accuracy\n                _, predicted = torch.max(outputs.data, 1)\n                total += labels.size(0)\n                correct += (predicted == labels).sum().item()\n        \n        val_accuracy = 100 * correct / total\n        print(f'Epoch [{epoch+1}/{num_epochs}], Validation Loss: {val_loss / len(valid_loader):.4f}, Validation Accuracy: {val_accuracy:.2f}%')\n        \n'''","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:23.249988Z","iopub.execute_input":"2024-10-14T14:07:23.250476Z","iopub.status.idle":"2024-10-14T14:07:23.264032Z","shell.execute_reply.started":"2024-10-14T14:07:23.250451Z","shell.execute_reply":"2024-10-14T14:07:23.263201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ResNet18 ","metadata":{}},{"cell_type":"code","source":"import kagglehub\nmodel_path = kagglehub.model_download('prabhakarnimmagadda/resnet_18/PyTorch/default/1', path='resnet18.pth')\nprint(model_path)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:23.265155Z","iopub.execute_input":"2024-10-14T14:07:23.265385Z","iopub.status.idle":"2024-10-14T14:07:23.735796Z","shell.execute_reply.started":"2024-10-14T14:07:23.265365Z","shell.execute_reply":"2024-10-14T14:07:23.734848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torchvision.models as models\n\n# Device configuration\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Load the pre-trained ResNet-18 model\nmodeL = models.resnet18()\n\nmodeL.load_state_dict(torch.load('/kaggle/input/resnet_18/pytorch/default/1/resnet18.pth'))\n\n# Modify the first convolutional layer to accept grayscale images\nmodeL.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False)\nmodeL.bn1 = nn.BatchNorm2d(64, eps=1e-05, momentum=0.9, affine=True, track_running_stats=True)\n\n# Modify the final fully connected layer to output 3 classes\nnum_ftrs = modeL.fc.in_features\nmodeL.fc = nn.Linear(num_ftrs, 3)\n\nmodel_Resnet = modeL.to(device)\n\n# Print the modified model architecture\nprint(model_Resnet)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:23.736969Z","iopub.execute_input":"2024-10-14T14:07:23.737252Z","iopub.status.idle":"2024-10-14T14:07:24.601043Z","shell.execute_reply.started":"2024-10-14T14:07:23.737228Z","shell.execute_reply":"2024-10-14T14:07:24.600150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Device configuration\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Loss function\ncriterion = nn.CrossEntropyLoss()\n\n# Training loop\ndef train_model(model, optimizer, train_loader, valid_loader, num_epochs=10):\n    for epoch in range(num_epochs):\n        model.train()  # Set the model to training mode\n        running_loss = 0.0\n        batch_idx = 0\n        \n        for batch in train_loader:\n            images, coords, labels = batch\n            images, labels = images.to(device), labels.to(device)  # Move data to GPU if available\n\n            # Zero the parameter gradients\n            optimizer.zero_grad()\n            \n            # Forward pass\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n            \n            # Backward pass and optimization\n            loss.backward()\n            optimizer.step()\n            \n            # Print statistics\n            running_loss += loss.item()\n            if batch_idx % 10 == 9:  # Print every 10 batches\n                print(f'Epoch [{epoch+1}/{num_epochs}], Step [{batch_idx+1}/{len(train_loader)}], Loss: {running_loss / 10:.4f}')\n                running_loss = 0.0\n            \n            batch_idx += 1\n        \n        # Print epoch statistics\n        print(f'Epoch [{epoch+1}/{num_epochs}] completed with average loss: {running_loss / len(train_loader):.4f}')\n        \n        # Validation phase\n        model.eval()  # Set the model to evaluation mode\n        val_loss = 0.0\n        correct = 0\n        total = 0\n        with torch.no_grad():\n            for images, coords, labels in valid_loader:\n                images, labels = images.to(device), labels.to(device)\n                outputs = model(images)\n                loss = criterion(outputs, labels)\n                val_loss += loss.item()\n                \n                # Calculate accuracy\n                _, predicted = torch.max(outputs.data, 1)\n                total += labels.size(0)\n                correct += (predicted == labels).sum().item()\n        \n        val_accuracy = 100 * correct / total\n        print(f'Epoch [{epoch+1}/{num_epochs}], Validation Loss: {val_loss / len(valid_loader):.4f}, Validation Accuracy: {val_accuracy:.2f}%')\n        \n        # Step the scheduler\n        lr_scheduler.step()","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:07:24.602335Z","iopub.execute_input":"2024-10-14T14:07:24.602634Z","iopub.status.idle":"2024-10-14T14:07:24.614302Z","shell.execute_reply.started":"2024-10-14T14:07:24.602610Z","shell.execute_reply":"2024-10-14T14:07:24.613395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize the model, loss function, and optimizer\nModel = model_Resnet\noptimizer = optim.SGD(Model.parameters(), lr=0.01, momentum=0.9)\nlr_scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.1)\n# Train the model for Sagittal T2/STIR\ntrain_model(Model, optimizer, train_loader_t2STIR, valid_loader_t2STIR, num_epochs=10)\n\n# Save the model weights for this condition\ntorch.save(Model.state_dict(), \"model_WEIGHTS_t2STIR.pth\")","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:09:04.063268Z","iopub.execute_input":"2024-10-14T14:09:04.063670Z","iopub.status.idle":"2024-10-14T14:13:48.296277Z","shell.execute_reply.started":"2024-10-14T14:09:04.063634Z","shell.execute_reply":"2024-10-14T14:13:48.294980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndef validate_model(model, valid_loader, criterion):\n    model.eval()  # Set the model to evaluation mode\n    total_loss = 0.0\n    correct = 0\n    running_loss = 0.0\n    total = 0\n\n    with torch.no_grad():  # Disable gradient calculation for validation\n        for batch in valid_loader:\n            images, coords, labels = batch\n            \n            # Convert string labels to integers\n            labels = labels.to(device)  # Convert to tensor and move to device\n            \n            # Move data to the device\n            images, coords = images.to(device), coords.to(device)\n            \n            outputs = model(images, coords)\n            loss = criterion(outputs, labels)\n            running_loss += loss.item()\n            \n            # Calculate accuracy\n            _, predicted = torch.max(outputs.data, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n\n    avg_loss = running_loss / len(valid_loader)\n    accuracy = correct / total\n    print(f'Validation Loss: {avg_loss:.4f}, Accuracy: {accuracy:.4f}')\n    '''","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:13:48.298330Z","iopub.execute_input":"2024-10-14T14:13:48.298679Z","iopub.status.idle":"2024-10-14T14:13:48.306271Z","shell.execute_reply.started":"2024-10-14T14:13:48.298646Z","shell.execute_reply":"2024-10-14T14:13:48.305302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation for Sagittal T2/STIR\n#validate_model(model, valid_loader_t2STIR, criterion)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:13:48.307396Z","iopub.execute_input":"2024-10-14T14:13:48.307692Z","iopub.status.idle":"2024-10-14T14:13:48.318000Z","shell.execute_reply.started":"2024-10-14T14:13:48.307668Z","shell.execute_reply":"2024-10-14T14:13:48.317111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sagittal t1 images training and validation","metadata":{}},{"cell_type":"code","source":"# Defining image transformations for the sagittal t1 \nimg_transforms = transforms.Compose([\n    \n    transforms.Resize((319, 319),antialias=True),\n    #transforms.RandomHorizontalFlip(p=0.5),  # 50% chance to flip the image horizontally\n])","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:13:48.319921Z","iopub.execute_input":"2024-10-14T14:13:48.320192Z","iopub.status.idle":"2024-10-14T14:13:48.330299Z","shell.execute_reply.started":"2024-10-14T14:13:48.320168Z","shell.execute_reply":"2024-10-14T14:13:48.329357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sag_t1_train_set = CustomDatasetWithCoords(sag_t1_train,img_transforms)\nsag_t1_valid_set = CustomDatasetWithCoords(sag_t1_val,img_transforms)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:13:48.331480Z","iopub.execute_input":"2024-10-14T14:13:48.331803Z","iopub.status.idle":"2024-10-14T14:13:48.341729Z","shell.execute_reply.started":"2024-10-14T14:13:48.331780Z","shell.execute_reply":"2024-10-14T14:13:48.340886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(sag_t1_train_set)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:13:48.343103Z","iopub.execute_input":"2024-10-14T14:13:48.343779Z","iopub.status.idle":"2024-10-14T14:13:48.357981Z","shell.execute_reply.started":"2024-10-14T14:13:48.343747Z","shell.execute_reply":"2024-10-14T14:13:48.357143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating a training and validation dataloaders for sagittal t1 \ntrain_loader_t1 = DataLoader(sag_t1_train_set,batch_size = 64, shuffle = True, num_workers = 2)\nvalid_loader_t1 = DataLoader(sag_t1_valid_set,batch_size = 64, shuffle = False, num_workers = 2)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:13:48.359160Z","iopub.execute_input":"2024-10-14T14:13:48.359612Z","iopub.status.idle":"2024-10-14T14:13:48.370249Z","shell.execute_reply.started":"2024-10-14T14:13:48.359582Z","shell.execute_reply":"2024-10-14T14:13:48.369343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the pre-trained ResNet-18 model\nmodel = models.resnet18()\n\nmodel.load_state_dict(torch.load('/kaggle/input/resnet_18/pytorch/default/1/resnet18.pth'))\n\n# Modify the first convolutional layer to accept grayscale images\nmodel.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False)\nmodel.bn1 = nn.BatchNorm2d(64, eps=1e-05, momentum=0.9, affine=True, track_running_stats=True)\n\n# Modify the final fully connected layer to output 3 classes\nnum_ftrs = model.fc.in_features\nmodel.fc = nn.Linear(num_ftrs, 3)\n\nnew_model_Resnet = model.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:13:48.371363Z","iopub.execute_input":"2024-10-14T14:13:48.371960Z","iopub.status.idle":"2024-10-14T14:13:48.658771Z","shell.execute_reply.started":"2024-10-14T14:13:48.371929Z","shell.execute_reply":"2024-10-14T14:13:48.657786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Optimizer and learning rate scheduler\n#optimizer = optim.Adam(model.parameters(), lr=0.001)\noptimizer = optim.SGD(model.parameters(), lr= 0.01, momentum=0.9)\nlr_scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.1)\nModel = new_model_Resnet\n\n# Assuming train_loader_axial_t2 and valid_loader_axial_t2 are defined\ntrain_model(Model, optimizer, train_loader_t1, valid_loader_t1, num_epochs=10)\ntorch.save(Model.state_dict(), \"model_weights_t1.pth\")","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:13:48.659909Z","iopub.execute_input":"2024-10-14T14:13:48.660191Z","iopub.status.idle":"2024-10-14T14:24:46.455132Z","shell.execute_reply.started":"2024-10-14T14:13:48.660167Z","shell.execute_reply":"2024-10-14T14:24:46.453858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n# For Sagittal T1\nmodel = SimpleCNN().to(device)  # Re-initialize model\noptimizer = optim.Adam(model.parameters(), lr=0.01)\n#optimizer = optim.SGD(model.parameters(), lr= 0.01, momentum=0.9)  # Re-initialize optimizer\n\n# Train the model for Sagittal T1\ntrain_model(model, optimizer,train_loader_t1, valid_loader_t1, num_epochs=7)\ntorch.save(model.state_dict(), \"model_weights_t1.pth\")\n'''","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:24:46.459714Z","iopub.execute_input":"2024-10-14T14:24:46.460039Z","iopub.status.idle":"2024-10-14T14:24:46.466859Z","shell.execute_reply.started":"2024-10-14T14:24:46.460007Z","shell.execute_reply":"2024-10-14T14:24:46.465936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation for Sagittal T1\n#validate_model(model, valid_loader_t1, criterion)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:24:46.468052Z","iopub.execute_input":"2024-10-14T14:24:46.468388Z","iopub.status.idle":"2024-10-14T14:24:46.478847Z","shell.execute_reply.started":"2024-10-14T14:24:46.468358Z","shell.execute_reply":"2024-10-14T14:24:46.478021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Axial t2 images training and validation","metadata":{}},{"cell_type":"code","source":"# Defining image transformations for the sagittal t1 \nimg_transforms = transforms.Compose([\n\n    transforms.Resize((319, 319),antialias=True),\n])","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:24:46.479835Z","iopub.execute_input":"2024-10-14T14:24:46.480094Z","iopub.status.idle":"2024-10-14T14:24:46.490788Z","shell.execute_reply.started":"2024-10-14T14:24:46.480072Z","shell.execute_reply":"2024-10-14T14:24:46.489979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"axial_t2_train_set = CustomDatasetWithCoords(axi_t2_train,img_transforms)  # axi_t2_train, axi_t2_val\naxial_t2_valid_set = CustomDatasetWithCoords(axi_t2_val,img_transforms)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:24:46.491770Z","iopub.execute_input":"2024-10-14T14:24:46.492048Z","iopub.status.idle":"2024-10-14T14:24:46.501352Z","shell.execute_reply.started":"2024-10-14T14:24:46.492020Z","shell.execute_reply":"2024-10-14T14:24:46.500636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(axial_t2_train_set)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:24:46.502522Z","iopub.execute_input":"2024-10-14T14:24:46.502938Z","iopub.status.idle":"2024-10-14T14:24:46.517297Z","shell.execute_reply.started":"2024-10-14T14:24:46.502908Z","shell.execute_reply":"2024-10-14T14:24:46.516467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating a training and validation dataloaders for the condition axial_t2\ntrain_loader_axial_t2 = DataLoader(axial_t2_train_set,batch_size = 128, shuffle = True, num_workers = 2)\nvalid_loader_axial_t2 = DataLoader(axial_t2_valid_set,batch_size = 128, shuffle = False, num_workers = 2)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:24:46.518357Z","iopub.execute_input":"2024-10-14T14:24:46.518661Z","iopub.status.idle":"2024-10-14T14:24:46.526174Z","shell.execute_reply.started":"2024-10-14T14:24:46.518640Z","shell.execute_reply":"2024-10-14T14:24:46.525374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the pre-trained ResNet-18 model\nMOdel = models.resnet18()\n\nMOdel.load_state_dict(torch.load('/kaggle/input/resnet_18/pytorch/default/1/resnet18.pth'))\n\n# Modify the first convolutional layer to accept grayscale images\nMOdel.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False)\nMOdel.bn1 = nn.BatchNorm2d(64, eps=1e-05, momentum=0.9, affine=True, track_running_stats=True)\n\n# Modify the final fully connected layer to output 3 classes\nnum_ftrs = MOdel.fc.in_features\nMOdel.fc = nn.Linear(num_ftrs, 3)\n\nNew_model_Resnet = MOdel.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:25:03.700077Z","iopub.execute_input":"2024-10-14T14:25:03.700462Z","iopub.status.idle":"2024-10-14T14:25:03.990917Z","shell.execute_reply.started":"2024-10-14T14:25:03.700432Z","shell.execute_reply":"2024-10-14T14:25:03.990121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Optimizer and learning rate scheduler\n#optimizer = optim.Adam(model.parameters(), lr=0.001)\noptimizer = optim.SGD(model.parameters(), lr= 0.01, momentum=0.9)\nlr_scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.1)\nmodel=New_model_Resnet\n\n# Assuming train_loader_axial_t2 and valid_loader_axial_t2 are defined\ntrain_model(model, optimizer, train_loader_axial_t2, valid_loader_axial_t2, num_epochs=15)\ntorch.save(model.state_dict(), \"model_weights_axial_t2.pth\")","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:29:14.715818Z","iopub.execute_input":"2024-10-14T14:29:14.716543Z","iopub.status.idle":"2024-10-14T14:36:55.617272Z","shell.execute_reply.started":"2024-10-14T14:29:14.716507Z","shell.execute_reply":"2024-10-14T14:36:55.616088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, _, labels = next(iter(train_loader_axial_t2))\nimages.shape","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:36:55.619253Z","iopub.execute_input":"2024-10-14T14:36:55.619625Z","iopub.status.idle":"2024-10-14T14:36:59.556172Z","shell.execute_reply.started":"2024-10-14T14:36:55.619589Z","shell.execute_reply":"2024-10-14T14:36:59.555093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n# For Axial T2\nfrom torch.optim import lr_scheduler\n\nmodel = SimpleCNN().to(device)  # Re-initialize model\n#optimizer = optim.SGD(model.parameters(), lr = 0.01, momentum=0.9)  # Re-initialize optimizer\noptimizer = optim.Adam(model.parameters(), lr=lr_scheduler)\ncriterion = nn.CrossEntropyLoss()\n\ntrain_model(model, optimizer,train_loader_axial_t2, valid_loader_axial_t2, num_epochs=7)\ntorch.save(model.state_dict(), \"model_weights_axial_t2.pth\")\n'''","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:37:03.210840Z","iopub.execute_input":"2024-10-14T14:37:03.211683Z","iopub.status.idle":"2024-10-14T14:37:03.218178Z","shell.execute_reply.started":"2024-10-14T14:37:03.211643Z","shell.execute_reply":"2024-10-14T14:37:03.217228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation for Axial T2\n#validate_model(model, valid_loader_axial_t2, criterion)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:37:03.851446Z","iopub.execute_input":"2024-10-14T14:37:03.852299Z","iopub.status.idle":"2024-10-14T14:37:03.856143Z","shell.execute_reply.started":"2024-10-14T14:37:03.852269Z","shell.execute_reply":"2024-10-14T14:37:03.855099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"levels = ['l1_l2', 'l2_l3', 'l3_l4', 'l4_l5', 'l5_s1']\n\n# Function to update row_id with levels\ndef update_row_id(row, levels):\n    level = levels[row.name % len(levels)]\n    return f\"{row['study_id']}_{row['condition']}_{level}\"\n\n# Update row_id in expanded_test_desc to include levels\ntest_data['row_id'] = test_data.apply(lambda row: update_row_id(row, levels), axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:37:06.158905Z","iopub.execute_input":"2024-10-14T14:37:06.159603Z","iopub.status.idle":"2024-10-14T14:37:06.172819Z","shell.execute_reply.started":"2024-10-14T14:37:06.159566Z","shell.execute_reply":"2024-10-14T14:37:06.171659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:37:08.806778Z","iopub.execute_input":"2024-10-14T14:37:08.807128Z","iopub.status.idle":"2024-10-14T14:37:08.819826Z","shell.execute_reply.started":"2024-10-14T14:37:08.807103Z","shell.execute_reply":"2024-10-14T14:37:08.818860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating a custome dataset class for test data\n\nclass Test_Dataset(Dataset):\n    def __init__(self, dataframe, transform = None):\n        self.dataframe = dataframe\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.dataframe)\n    \n    def __getitem__(self, index):\n        image_path = self.dataframe['image_path'][index]\n        image = load_dicom(image_path)\n        \n        # Convert image to tensor\n        image_tensor = torch.tensor(image, dtype=torch.float32) / 255.0\n        \n        # If the image has 2 dimensions, add a channel dimension\n        if image_tensor.ndim == 2:\n            image_tensor = image_tensor.unsqueeze(0)\n            \n        if self.transform:\n            image_tensor = self.transform(image_tensor)\n\n        return image_tensor\n    \n\n# Define the transforms\ntest_transform = transforms.Compose([\n    transforms.Resize((319, 319),antialias=True),\n])","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:37:09.761995Z","iopub.execute_input":"2024-10-14T14:37:09.762757Z","iopub.status.idle":"2024-10-14T14:37:09.770949Z","shell.execute_reply.started":"2024-10-14T14:37:09.762724Z","shell.execute_reply":"2024-10-14T14:37:09.769863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating Dataset and Dataloader for test data\ntest_data_set = Test_Dataset(test_data,test_transform)\ntest_data_loader = DataLoader(test_data_set, batch_size = 1, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:37:11.901085Z","iopub.execute_input":"2024-10-14T14:37:11.901461Z","iopub.status.idle":"2024-10-14T14:37:11.906960Z","shell.execute_reply.started":"2024-10-14T14:37:11.901425Z","shell.execute_reply":"2024-10-14T14:37:11.905936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for image in test_data_loader:\n    print(image.shape, image)\n    break","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:37:12.287754Z","iopub.execute_input":"2024-10-14T14:37:12.288091Z","iopub.status.idle":"2024-10-14T14:37:12.338920Z","shell.execute_reply.started":"2024-10-14T14:37:12.288063Z","shell.execute_reply":"2024-10-14T14:37:12.337989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_set[0]","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:37:14.596166Z","iopub.execute_input":"2024-10-14T14:37:14.596652Z","iopub.status.idle":"2024-10-14T14:37:14.631981Z","shell.execute_reply.started":"2024-10-14T14:37:14.596619Z","shell.execute_reply":"2024-10-14T14:37:14.631141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom tqdm import tqdm\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Function to make predictions on the test data\ndef predict_test_data(testloader):\n    # Initialize lists to store probabilities for each class\n    normal_mild_probs = []\n    moderate_probs = []\n    severe_probs = []\n    predictions = []\n    \n    # Preload the models with their corresponding weights\n    model_t2STIR = models.resnet18()\n    model_t2STIR.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False)\n    model_t2STIR.bn1 = nn.BatchNorm2d(64, eps=1e-05, momentum=0.9)\n    num_ftrs_t2STIR = model_t2STIR.fc.in_features\n    model_t2STIR.fc = nn.Linear(num_ftrs_t2STIR, 3)\n    model_t2STIR.load_state_dict(torch.load(\"/kaggle/working/model_WEIGHTS_t2STIR.pth\"))\n    model_t2STIR.to(device)\n    model_t2STIR.eval()\n\n    model_t1 = models.resnet18()\n    model_t1.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False)\n    model_t1.bn1 = nn.BatchNorm2d(64, eps=1e-05, momentum=0.9)\n    num_ftrs_t1 = model_t1.fc.in_features\n    model_t1.fc = nn.Linear(num_ftrs_t1, 3)\n    model_t1.load_state_dict(torch.load(\"/kaggle/working/model_weights_t1.pth\"))\n    model_t1.to(device)\n    model_t1.eval()\n\n    model_axial_t2 = models.resnet18()\n    model_axial_t2.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False)\n    model_axial_t2.bn1 = nn.BatchNorm2d(64, eps=1e-05, momentum=0.9)\n    num_ftrs_axial_t2 = model_axial_t2.fc.in_features\n    model_axial_t2.fc = nn.Linear(num_ftrs_axial_t2, 3)\n    model_axial_t2.load_state_dict(torch.load(\"/kaggle/working/model_weights_axial_t2.pth\"))\n    model_axial_t2.to(device)\n    model_axial_t2.eval()\n    with torch.no_grad():  # Disable gradient calculations for inference\n        for images in tqdm(testloader):\n            images = images.to(device)\n\n            # Forward pass through each model (images only, no labels during inference)\n            outputs_t2STIR = model_t2STIR(images)\n            outputs_t1 = model_t1(images)\n            outputs_axial_t2 = model_axial_t2(images)\n\n            # Average the outputs (ensemble technique)\n            avg_outputs = (outputs_t2STIR + outputs_t1 + outputs_axial_t2) / 3\n            \n            # Apply softmax to get class probabilities\n            probs = torch.softmax(avg_outputs, dim=1)\n            \n            # Store probabilities for each class\n            normal_mild_probs.append(probs[:, 0].cpu().numpy())  # Class 0\n            moderate_probs.append(probs[:, 1].cpu().numpy())     # Class 1\n            severe_probs.append(probs[:, 2].cpu().numpy())       # Class 2\n            predictions.append(probs.cpu().numpy())\n    \n    return normal_mild_probs, moderate_probs, severe_probs, predictions","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:41:55.456099Z","iopub.execute_input":"2024-10-14T14:41:55.456826Z","iopub.status.idle":"2024-10-14T14:41:55.472184Z","shell.execute_reply.started":"2024-10-14T14:41:55.456796Z","shell.execute_reply":"2024-10-14T14:41:55.470977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make predictions on test data\nnormal_mild_probs, moderate_probs, severe_probs, predictions = predict_test_data(test_data_loader)\n\n# Print the first few predictions\nprint(predictions[:5])  # Print the first 5 predictions\n","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:42:07.490064Z","iopub.execute_input":"2024-10-14T14:42:07.490478Z","iopub.status.idle":"2024-10-14T14:42:15.262180Z","shell.execute_reply.started":"2024-10-14T14:42:07.490445Z","shell.execute_reply":"2024-10-14T14:42:15.261270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Run predictions\n#normal_mild_probs, moderate_probs, severe_probs, predictions = predict_test_data(test_data_loader)\n\n# Example of printing the probabilities\n#for i in range(len(normal_mild_probs)):\n    #print(f'Image {i}: Normal Mild Prob: {normal_mild_probs[i]}, Moderate Prob: {moderate_probs[i]}, Severe Prob: {severe_probs[i]}')","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:42:23.580372Z","iopub.execute_input":"2024-10-14T14:42:23.580775Z","iopub.status.idle":"2024-10-14T14:42:23.585348Z","shell.execute_reply.started":"2024-10-14T14:42:23.580746Z","shell.execute_reply":"2024-10-14T14:42:23.584355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions[0]","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:42:24.450383Z","iopub.execute_input":"2024-10-14T14:42:24.450777Z","iopub.status.idle":"2024-10-14T14:42:24.457214Z","shell.execute_reply.started":"2024-10-14T14:42:24.450748Z","shell.execute_reply":"2024-10-14T14:42:24.456235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add predictions and probabilities to the test DataFrame\ntest_data['normal_mild'] = normal_mild_probs\ntest_data['moderate'] = moderate_probs\ntest_data['severe'] = severe_probs","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:42:28.070171Z","iopub.execute_input":"2024-10-14T14:42:28.070548Z","iopub.status.idle":"2024-10-14T14:42:28.077553Z","shell.execute_reply.started":"2024-10-14T14:42:28.070520Z","shell.execute_reply":"2024-10-14T14:42:28.076559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert the lists to scalar values using apply and lambda function\ntest_data['normal_mild'] = test_data['normal_mild'].apply(lambda x: x[0] if isinstance(x, (list, np.ndarray)) else x)\ntest_data['moderate'] = test_data['moderate'].apply(lambda x: x[0] if isinstance(x, (list, np.ndarray)) else x)\ntest_data['severe'] = test_data['severe'].apply(lambda x: x[0] if isinstance(x, (list, np.ndarray)) else x)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:42:30.243702Z","iopub.execute_input":"2024-10-14T14:42:30.244399Z","iopub.status.idle":"2024-10-14T14:42:30.254523Z","shell.execute_reply.started":"2024-10-14T14:42:30.244365Z","shell.execute_reply":"2024-10-14T14:42:30.253524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = test_data[[\"row_id\",\"normal_mild\",\"moderate\",\"severe\"]]","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:42:31.264150Z","iopub.execute_input":"2024-10-14T14:42:31.264713Z","iopub.status.idle":"2024-10-14T14:42:31.270574Z","shell.execute_reply.started":"2024-10-14T14:42:31.264676Z","shell.execute_reply":"2024-10-14T14:42:31.269610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:42:32.243654Z","iopub.execute_input":"2024-10-14T14:42:32.244695Z","iopub.status.idle":"2024-10-14T14:42:32.256769Z","shell.execute_reply.started":"2024-10-14T14:42:32.244660Z","shell.execute_reply":"2024-10-14T14:42:32.255826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Group by 'row_id' and sum the values\ngrouped_submission = submission.groupby('row_id').max().reset_index()","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:42:38.449238Z","iopub.execute_input":"2024-10-14T14:42:38.449888Z","iopub.status.idle":"2024-10-14T14:42:38.459384Z","shell.execute_reply.started":"2024-10-14T14:42:38.449854Z","shell.execute_reply":"2024-10-14T14:42:38.458440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_submission","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:42:40.323582Z","iopub.execute_input":"2024-10-14T14:42:40.324370Z","iopub.status.idle":"2024-10-14T14:42:40.338347Z","shell.execute_reply.started":"2024-10-14T14:42:40.324338Z","shell.execute_reply":"2024-10-14T14:42:40.337437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(grouped_submission)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:42:47.312213Z","iopub.execute_input":"2024-10-14T14:42:47.313022Z","iopub.status.idle":"2024-10-14T14:42:47.318835Z","shell.execute_reply.started":"2024-10-14T14:42:47.312990Z","shell.execute_reply":"2024-10-14T14:42:47.317857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submi[['normal_mild', 'moderate', 'severe']] = grouped_submission[['normal_mild', 'moderate', 'severe']]","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:42:49.141647Z","iopub.execute_input":"2024-10-14T14:42:49.142638Z","iopub.status.idle":"2024-10-14T14:42:49.153392Z","shell.execute_reply.started":"2024-10-14T14:42:49.142599Z","shell.execute_reply":"2024-10-14T14:42:49.151519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\n# Save the DataFrame to \"submission.csv\" in the desired directory\nsubmi.to_csv(\"/kaggle/working/submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:42:50.713308Z","iopub.execute_input":"2024-10-14T14:42:50.714146Z","iopub.status.idle":"2024-10-14T14:42:50.722033Z","shell.execute_reply.started":"2024-10-14T14:42:50.714113Z","shell.execute_reply":"2024-10-14T14:42:50.721192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submi.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-10-14T14:42:52.288994Z","iopub.execute_input":"2024-10-14T14:42:52.289334Z","iopub.status.idle":"2024-10-14T14:42:52.301524Z","shell.execute_reply.started":"2024-10-14T14:42:52.289307Z","shell.execute_reply":"2024-10-14T14:42:52.300664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}