{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":30746,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport cv2\nimport pydicom\nimport numpy as np\nimport os\nimport glob\nfrom tqdm import tqdm\nfrom tqdm.auto import tqdm \nimport warnings\ntqdm.pandas()  # Enable the pandas extension\n","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:26:26.730240Z","iopub.execute_input":"2024-09-09T20:26:26.731073Z","iopub.status.idle":"2024-09-09T20:26:28.417296Z","shell.execute_reply.started":"2024-09-09T20:26:26.731039Z","shell.execute_reply":"2024-09-09T20:26:28.416258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the datasets into pandas DataFrames\nlabel_coordinates_df = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_label_coordinates.csv')\ntrain_series = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_series_descriptions.csv')\ndf_train = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv')\n","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:26:28.419297Z","iopub.execute_input":"2024-09-09T20:26:28.419794Z","iopub.status.idle":"2024-09-09T20:26:28.599165Z","shell.execute_reply.started":"2024-09-09T20:26:28.419761Z","shell.execute_reply":"2024-09-09T20:26:28.598046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:26:28.600391Z","iopub.execute_input":"2024-09-09T20:26:28.602396Z","iopub.status.idle":"2024-09-09T20:26:28.648318Z","shell.execute_reply.started":"2024-09-09T20:26:28.602362Z","shell.execute_reply":"2024-09-09T20:26:28.647188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:26:28.650718Z","iopub.execute_input":"2024-09-09T20:26:28.651013Z","iopub.status.idle":"2024-09-09T20:26:28.663219Z","shell.execute_reply.started":"2024-09-09T20:26:28.650988Z","shell.execute_reply":"2024-09-09T20:26:28.662184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_coordinates_df","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:26:28.665033Z","iopub.execute_input":"2024-09-09T20:26:28.665374Z","iopub.status.idle":"2024-09-09T20:26:28.687204Z","shell.execute_reply.started":"2024-09-09T20:26:28.665345Z","shell.execute_reply":"2024-09-09T20:26:28.686185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_outer_df = pd.merge(label_coordinates_df, train_series, on=['study_id', 'series_id'], how='outer')\nmerged_outer_df","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:26:28.688373Z","iopub.execute_input":"2024-09-09T20:26:28.688664Z","iopub.status.idle":"2024-09-09T20:26:28.740041Z","shell.execute_reply.started":"2024-09-09T20:26:28.688639Z","shell.execute_reply":"2024-09-09T20:26:28.738758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a new column 'filepath' for each row in 'merged_outer_df' and check if the file exists\nmain_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images'\n# Create a new column 'filepath' only if all necessary values are present\ndef generate_filepath(row):\n    if pd.notna(row['study_id']) and pd.notna(row['series_id']) and pd.notna(row['instance_number']):\n        file_path = f\"{main_path}/{row['study_id']}/{row['series_id']}/{int(row['instance_number'])}.dcm\"\n        return file_path if os.path.exists(file_path) else None\n    return None\n\nmerged_outer_df['filepath'] = merged_outer_df.progress_apply(generate_filepath, axis=1)\n\nmerged_outer_df","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:26:28.741351Z","iopub.execute_input":"2024-09-09T20:26:28.741682Z","iopub.status.idle":"2024-09-09T20:27:05.647248Z","shell.execute_reply.started":"2024-09-09T20:26:28.741656Z","shell.execute_reply":"2024-09-09T20:27:05.646140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nan_rows = merged_outer_df[merged_outer_df.isna().any(axis=1)]\nprint(\"\\nRows with NaN values in merged_outer_df:\")\nnan_rows\n#The study-id 3008676218 doesn't have an samples in the trainnig data and there are labels and some of nan \n#let's just drop it\n# for the other case no data for these series id but it have other data that can complete the predictions","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:27:05.648526Z","iopub.execute_input":"2024-09-09T20:27:05.648857Z","iopub.status.idle":"2024-09-09T20:27:05.687826Z","shell.execute_reply.started":"2024-09-09T20:27:05.648829Z","shell.execute_reply":"2024-09-09T20:27:05.686755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_outer_df = merged_outer_df.dropna()\nmerged_outer_df","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:27:05.689175Z","iopub.execute_input":"2024-09-09T20:27:05.689495Z","iopub.status.idle":"2024-09-09T20:27:05.744552Z","shell.execute_reply.started":"2024-09-09T20:27:05.689469Z","shell.execute_reply":"2024-09-09T20:27:05.743341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nan_rows = df_train[df_train.isna().any(axis=1)]\nprint(\"\\nRows with NaN values in merged_outer_df:\")\ndisplay(nan_rows)","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:27:05.748836Z","iopub.execute_input":"2024-09-09T20:27:05.749244Z","iopub.status.idle":"2024-09-09T20:27:05.791736Z","shell.execute_reply.started":"2024-09-09T20:27:05.749213Z","shell.execute_reply":"2024-09-09T20:27:05.790533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to dynamically find the correct severity column name\ndef get_severity_column(condition, level):\n    # Convert the condition and level to match the column names in df_train\n    condition_formatted = condition.lower().replace(' ', '_')\n    level_formatted = level.lower().replace('/', '_')\n    return f\"{condition_formatted}_{level_formatted}\"\n\n# Add a new 'severity' column to merged_outer_df\ndef map_severity(row):\n    # Get the severity column name for the row\n    severity_column = get_severity_column(row['condition'], row['level'])\n    # Fetch the severity value from df_train using study_id\n    severity_value = df_train.loc[df_train['study_id'] == row['study_id'], severity_column]\n    # Return the severity value if available, otherwise return None\n    return severity_value.values[0] if not severity_value.empty else None\n\n# Apply the mapping function to each row in merged_outer_df\nmerged_outer_df.loc[:, 'severity'] = merged_outer_df.progress_apply(map_severity, axis=1)\n\n# Display the updated DataFrame\nprint(\"Merged DataFrame with Severity:\")\nmerged_outer_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:27:05.793754Z","iopub.execute_input":"2024-09-09T20:27:05.794585Z","iopub.status.idle":"2024-09-09T20:27:27.953327Z","shell.execute_reply.started":"2024-09-09T20:27:05.794545Z","shell.execute_reply":"2024-09-09T20:27:27.952360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_outer_df['severity'].unique()","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:27:27.954457Z","iopub.execute_input":"2024-09-09T20:27:27.954712Z","iopub.status.idle":"2024-09-09T20:27:27.965098Z","shell.execute_reply.started":"2024-09-09T20:27:27.954690Z","shell.execute_reply":"2024-09-09T20:27:27.964185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Group by 'study_id', 'series_id', and 'series_description' and aggregate the columns into lists\ngrouped_df = merged_outer_df.groupby(['study_id', 'series_id', 'series_description']).agg({\n    'instance_number': lambda x: list(x),\n    'condition': lambda x: list(x),\n    'level': lambda x: list(x),\n    'x': lambda x: list(x),\n    'y': lambda x: list(x),\n    'filepath': lambda x: list(x),\n    'severity': lambda x: list(x)\n}).reset_index()\ngrouped_df","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:37:06.637159Z","iopub.execute_input":"2024-09-09T20:37:06.638231Z","iopub.status.idle":"2024-09-09T20:37:08.228327Z","shell.execute_reply.started":"2024-09-09T20:37:06.638185Z","shell.execute_reply":"2024-09-09T20:37:08.227179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So that means not all series have heir data available let's clean the merged dataframe then","metadata":{}},{"cell_type":"code","source":"# Define the columns that are lists\ncolumns_with_lists = ['instance_number', 'condition', 'level', 'x', 'y', 'filepath']\n\n# Function to clean NaN values within lists\ndef clean_nan_in_lists(row):\n    for col in columns_with_lists:\n        if isinstance(row[col], list):\n            # Remove NaN values from the list\n            row[col] = [item for item in row[col] if pd.notna(item)]\n            # If the list becomes empty after removing NaNs, consider it as None\n            if not row[col]:\n                row[col] = None\n    return row\n\n# Apply the cleaning function to each row\ncleaned_grouped_df = grouped_df.apply(clean_nan_in_lists, axis=1)\n\n# Remove rows where any column has become None (if required)\ncleaned_grouped_df = cleaned_grouped_df.dropna(subset=columns_with_lists, how='any')\n\n# Display the cleaned DataFrame\nprint(\"Cleaned DataFrame:\")\ncleaned_grouped_df","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:37:09.011714Z","iopub.execute_input":"2024-09-09T20:37:09.012099Z","iopub.status.idle":"2024-09-09T20:37:11.406866Z","shell.execute_reply.started":"2024-09-09T20:37:09.012069Z","shell.execute_reply":"2024-09-09T20:37:11.405760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_outer_df['series_description'].unique()","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:37:11.408712Z","iopub.execute_input":"2024-09-09T20:37:11.409016Z","iopub.status.idle":"2024-09-09T20:37:11.419393Z","shell.execute_reply.started":"2024-09-09T20:37:11.408989Z","shell.execute_reply":"2024-09-09T20:37:11.418488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Axial T2 ->['Left Subarticular Stenosis', 'Right Subarticular Stenosis']\nprint('Axial T2')\ndisplay(merged_outer_df[merged_outer_df['series_description']==\"Axial T2\"][\"condition\"].value_counts())\nprint('================')\nprint('Sagittal T1')\n#Sagittal T1 ->['Right Neural Foraminal Narrowing', 'Left Neural Foraminal Narrowing', 'Spinal Canal Stenosis']\ndisplay(merged_outer_df[merged_outer_df['series_description']==\"Sagittal T1\"][\"condition\"].value_counts())\nprint('================')\nprint('Sagittal T2/STIR')\n#Sagittal T2/STIR ->['Spinal Canal Stenosis']\ndisplay(merged_outer_df[merged_outer_df['series_description']==\"Sagittal T2/STIR\"][\"condition\"].value_counts())\nprint('================')\n\n","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:37:11.420657Z","iopub.execute_input":"2024-09-09T20:37:11.420941Z","iopub.status.idle":"2024-09-09T20:37:11.484025Z","shell.execute_reply.started":"2024-09-09T20:37:11.420917Z","shell.execute_reply":"2024-09-09T20:37:11.483131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Read DICOM images","metadata":{}},{"cell_type":"code","source":"import pydicom\nimport matplotlib.pyplot as plt\n\ndef read_dicom(file_path):\n    \"\"\"\n    Reads a DICOM file from the given file path and returns the pixel data.\n    \n    Parameters:\n    file_path (str): The path to the DICOM file.\n    \n    Returns:\n    numpy.ndarray: The pixel data from the DICOM file, or None if an error occurs.\n    \"\"\"\n    try:\n        # Read the DICOM file\n        dicom_image = pydicom.dcmread(file_path)\n        \n        # Extract and return the pixel data\n        return dicom_image.pixel_array\n    \n    except Exception as e:\n        print(f\"An error occurred while reading the DICOM file {file_path}: {e}\")\n        return None\n\n\ndef visualize_dicom(pixel_array, title='DICOM Image'):\n    \"\"\"\n    Visualizes the pixel data of a DICOM image using matplotlib.\n    \n    Parameters:\n    pixel_array (numpy.ndarray): The pixel data of the DICOM image.\n    title (str): Title for the plot.\n    \n    Returns:\n    None\n    \"\"\"\n    try:\n        # Visualize the image using matplotlib\n        plt.figure(figsize=(6, 6))\n        plt.imshow(pixel_array, cmap='gray')\n        plt.title(title)\n        plt.axis('off')\n        plt.show()\n        \n    except Exception as e:\n        print(f\"An error occurred while visualizing the DICOM image: {e}\")\n\ndef read_multiple_dicoms(file_paths):\n    \"\"\"\n    Reads multiple DICOM files from a list of file paths and returns their pixel data.\n    \n    Parameters:\n    file_paths (list of str): A list of paths to DICOM files.\n    \n    Returns:\n    dict: A dictionary with file paths as keys and pixel data (numpy.ndarray) as values.\n    \"\"\"\n    dicom_images = {}\n    \n    for file_path in file_paths:\n        pixel_data = read_dicom(file_path)\n        if pixel_data is not None:\n            dicom_images[file_path] = pixel_data\n        else:\n            print(f\"Failed to read DICOM file: {file_path}\")\n    \n    return dicom_images\ndef visualize_multiple_dicoms(dicom_images):\n    \"\"\"\n    Visualizes multiple DICOM images from a dictionary of pixel data.\n    \n    Parameters:\n    dicom_images (dict): A dictionary with file paths as keys and pixel data (numpy.ndarray) as values.\n    \n    Returns:\n    None\n    \"\"\"\n    for file_path, pixel_array in dicom_images.items():\n        visualize_dicom(pixel_array, title=f'DICOM Image: {file_path.split(\"/\")[-1]}')\n","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:37:11.486250Z","iopub.execute_input":"2024-09-09T20:37:11.486583Z","iopub.status.idle":"2024-09-09T20:37:11.497020Z","shell.execute_reply.started":"2024-09-09T20:37:11.486559Z","shell.execute_reply":"2024-09-09T20:37:11.495834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"read_dicom(cleaned_grouped_df.iloc[0]['filepath'][0]).shape","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:37:11.498485Z","iopub.execute_input":"2024-09-09T20:37:11.498827Z","iopub.status.idle":"2024-09-09T20:37:11.559288Z","shell.execute_reply.started":"2024-09-09T20:37:11.498796Z","shell.execute_reply":"2024-09-09T20:37:11.558364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_dicoms_grid_with_annotations(df, study_id, series_id):\n    \"\"\"\n    Visualizes DICOM images in a grid format for a specific study and series ID,\n    including additional information such as series description, conditions, levels, and severity,\n    with annotations for each condition using the x and y coordinates.\n    \n    Parameters:\n    df (DataFrame): The DataFrame containing DICOM data details.\n    study_id (int): The study ID to filter the data.\n    series_id (int): The series ID to filter the data.\n    \n    Returns:\n    None\n    \"\"\"\n    # Filter the DataFrame based on the given study_id and series_id\n    filtered_df = df[(df['study_id'] == study_id) & (df['series_id'] == series_id)]\n\n    if filtered_df.empty:\n        print(f\"No images found for study ID {study_id} and series ID {series_id}.\")\n        return\n\n    # Extract all filepaths and related info from the filtered DataFrame\n    filepaths = []\n    xy_annotations = {}  # Dictionary to store x, y, conditions, levels, and severities for each file\n\n    for index, row in filtered_df.iterrows():\n        if isinstance(row['filepath'], list):\n            for i, file_path in enumerate(row['filepath']):\n                filepaths.append(file_path)\n                if file_path not in xy_annotations:\n                    xy_annotations[file_path] = {'x': [], 'y': [], 'condition': [], 'level': [], 'severity': []}\n                xy_annotations[file_path]['x'].append(row['x'][i])\n                xy_annotations[file_path]['y'].append(row['y'][i])\n                xy_annotations[file_path]['condition'].append(row['condition'][i])\n                xy_annotations[file_path]['level'].append(row['level'][i])\n                xy_annotations[file_path]['severity'].append(row['severity'][i])\n        else:\n            filepaths.append(row['filepath'])\n            if row['filepath'] not in xy_annotations:\n                xy_annotations[row['filepath']] = {'x': [row['x']], 'y': [row['y']], \n                                                   'condition': [row['condition']], \n                                                   'level': [row['level']], \n                                                   'severity': [row['severity']]}\n\n    # Read all DICOM files from the extracted filepaths\n    dicom_images = read_multiple_dicoms(filepaths)\n\n    # Determine the number of images\n    num_images = len(dicom_images)\n    cols = 3  # Number of columns in the grid\n    rows = (num_images + cols - 1) // cols  # Calculate the number of rows needed\n\n    # Extract additional information to display\n    series_description = filtered_df['series_description'].values[0]\n\n    # Create a grid to visualize the DICOM images\n    fig, axes = plt.subplots(rows, cols, figsize=(15, 5 * rows))\n    axes = axes.flatten()\n\n    # Iterate through the DICOM images and display them in the grid\n    for idx, (file_path, pixel_array) in enumerate(dicom_images.items()):\n        axes[idx].imshow(pixel_array, cmap='gray')\n        \n        # Add the image information in the title\n        title = f\"Series: {series_description}\\nImage: {file_path.split('/')[-1]}\"\n        axes[idx].set_title(title, fontsize=8)\n        axes[idx].axis('off')\n\n        # Plot each condition with its x, y coordinates, level, and severity\n        if file_path in xy_annotations:\n            for x, y, condition, level, severity in zip(xy_annotations[file_path]['x'], \n                                                        xy_annotations[file_path]['y'], \n                                                        xy_annotations[file_path]['condition'],\n                                                        xy_annotations[file_path]['level'],\n                                                        xy_annotations[file_path]['severity']):\n                # Draw a larger circle around the point\n                circle = plt.Circle((x, y), radius=10, color='red', fill=False, linewidth=1.5)\n                axes[idx].add_patch(circle)\n\n                # Improved text annotation with better color and size\n                annotation_text = f\"Condition: {condition}\\nLevel: {level}\\nSeverity: {severity}\"\n                axes[idx].text(x, y, annotation_text, color='white', fontsize=8, fontweight='bold',\n                               ha='left', va='top', bbox=dict(facecolor='black', alpha=0.6, edgecolor='none'))\n\n    # Hide any unused subplots\n    for idx in range(num_images, len(axes)):\n        axes[idx].axis('off')\n\n    plt.tight_layout()\n    plt.show()\n\n# Example usage\nstudy_id = 4003253  # Replace with the desired study_id\nseries_id = 702807833  # Replace with the desired series_id\nvisualize_dicoms_grid_with_annotations(cleaned_grouped_df, study_id, series_id)","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:37:11.560706Z","iopub.execute_input":"2024-09-09T20:37:11.561223Z","iopub.status.idle":"2024-09-09T20:37:12.281056Z","shell.execute_reply.started":"2024-09-09T20:37:11.561188Z","shell.execute_reply":"2024-09-09T20:37:12.280167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cleaned_grouped_df","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:37:12.282761Z","iopub.execute_input":"2024-09-09T20:37:12.283218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cleaned_grouped_df.dtypes","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare DF fo training","metadata":{}},{"cell_type":"code","source":"expanded_df = cleaned_grouped_df.explode(['instance_number', 'condition', 'level', 'x', 'y', 'filepath', 'severity'])\n\n# Reset the index after expanding\nexpanded_df = expanded_df.reset_index(drop=True)\nexpanded_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_encoded = pd.get_dummies(expanded_df, columns=['condition', 'level'])\ndf_encoded","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"severity_mapping = {'Normal/Mild': 0, 'Moderate': 1, 'Severe': 2}\ndf_encoded['severity'] = df_encoded['severity'].map(severity_mapping)\ndf_encoded","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Find the most frequent value in the 'severity' column\nmost_frequent_value = df_encoded['severity'].mode()[0]\n\n# Print the number of NaN values before filling\nprint(\"Number of NaN values in 'severity' column before filling:\")\nprint(df_encoded['severity'].isna().sum())\n\n# Fill NaN values in the 'severity' column with the most frequent value\ndf_encoded['severity'] = df_encoded['severity'].fillna(most_frequent_value)\n\n# Print the most frequent value used to fill NaN\nprint(f\"The most frequent value in 'severity' is: {most_frequent_value}\")\n\n# Verify that there are no more NaN values in the 'severity' column\nprint(\"Number of NaN values in 'severity' column after filling:\")\nprint(df_encoded['severity'].isna().sum())\n\n# Print unique values in the 'severity' column to ensure all labels are numeric\nprint(df_encoded['severity'].unique())\n\n# Ensure all labels are numeric after mapping\nprint(df_encoded['severity'].dtype)","metadata":{"execution":{"iopub.status.idle":"2024-09-09T20:37:12.575763Z","shell.execute_reply.started":"2024-09-09T20:37:12.562207Z","shell.execute_reply":"2024-09-09T20:37:12.574785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as transforms\nimport torch\nimport pydicom\nimport numpy as np\n\n# Define the function to read DICOM images\ndef read_dicom(file_path):\n    \"\"\"\n    Reads a DICOM file from the given file path and returns the pixel data.\n    \n    Parameters:\n    file_path (str): The path to the DICOM file.\n    \n    Returns:\n    numpy.ndarray: The pixel data from the DICOM file, or None if an error occurs.\n    \"\"\"\n    try:\n        # Read the DICOM file\n        dicom_image = pydicom.dcmread(file_path)\n        \n        # Extract the pixel data and convert to float32\n        image_array = dicom_image.pixel_array.astype(np.float32)\n        \n        return image_array\n    \n    except Exception as e:\n        print(f\"An error occurred while reading the DICOM file {file_path}: {e}\")\n        return None\n\n# Define a custom dataset class\nclass CustomDataset(Dataset):\n    def __init__(self, dataframe, features_list, y_label=None, transform=None):\n        \"\"\"\n        Initializes the CustomDataset.\n        \n        Parameters:\n        dataframe (DataFrame): The dataframe containing data.\n        features_list (list of str): List of column names to be used as features.\n        y_label (str, optional): Column name of the target label. Defaults to None for inference.\n        transform (callable, optional): A function/transform to apply to the images.\n        \"\"\"\n        self.dataframe = dataframe\n        self.features_list = features_list\n        self.y_label = y_label\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.dataframe)\n\n    def __getitem__(self, index):\n        # Load the DICOM image\n        image_path = self.dataframe['filepath'].iloc[index]\n        image = read_dicom(image_path)\n\n        # Convert the image to a PyTorch tensor\n        image_tensor = torch.tensor(image, dtype=torch.float32).unsqueeze(0)  # Add a channel dimension for grayscale\n\n        # Apply the transform if provided\n        if self.transform:\n            image_tensor = self.transform(image_tensor)\n\n        # Extract the features\n        features = self.dataframe[self.features_list].iloc[index].values.astype(np.float32)\n\n        # If y_label is provided, extract the label, otherwise return None\n        if self.y_label:\n            label = self.dataframe[self.y_label].iloc[index]\n            return image_tensor, torch.tensor(features), torch.tensor(label, dtype=torch.long)\n        else:\n            return image_tensor, torch.tensor(features)\n# Define the features and label columns\nfeatures_list = [\n    'condition_Left Neural Foraminal Narrowing',\n    'condition_Left Subarticular Stenosis',\n    'condition_Right Neural Foraminal Narrowing',\n    'condition_Right Subarticular Stenosis',\n    'condition_Spinal Canal Stenosis',\n    'level_L1/L2',\n    'level_L2/L3',\n    'level_L3/L4',\n    'level_L4/L5',\n    'level_L5/S1'\n]\n\ny_label = 'severity'\n\n# Define the transforms\ntransform = transforms.Compose([\n    transforms.Lambda(lambda x: (x - x.min()) / (x.max() - x.min())),  # Normalize to range [0, 1]\n    transforms.ToPILImage(),\n    transforms.Resize((224, 224)),  # Resize to the model's expected input size\n    transforms.RandomRotation(degrees=5),  # Small rotation\n    transforms.RandomHorizontalFlip(),  # Horizontal flip\n    transforms.ColorJitter(brightness=0.1, contrast=0.1),  # Adjust brightness and contrast slightly\n    transforms.Grayscale(num_output_channels=3),  # Convert to grayscale with 3 channels\n    transforms.ToTensor(),\n])\n\n# Create the dataset and data loader\ndataset = CustomDataset(df_encoded, features_list, y_label, transform=transform)\ndata_loader = DataLoader(dataset, batch_size=32, shuffle=True)\n\n# Example: Iterate over data\nfor images, features, labels in data_loader:\n    print(images.shape, features.shape, labels.shape)  # Display the shape of images, features, and labels\n    break  # Just to display one batch","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:37:12.577929Z","iopub.execute_input":"2024-09-09T20:37:12.578283Z","iopub.status.idle":"2024-09-09T20:37:13.685093Z","shell.execute_reply.started":"2024-09-09T20:37:12.578256Z","shell.execute_reply":"2024-09-09T20:37:13.684120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Split data","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n# Define the percentage of data to use for a quick run\nQUICK_RUN_PERCENTAGE = 1  # Use 20% of the data for a quick run\n\n# Sample the data for a quick run\nquick_run_df = df_encoded.sample(frac=QUICK_RUN_PERCENTAGE, random_state=42).reset_index(drop=True)\n\n# Define the test size percentage\nTEST_SIZE = 0.2  # 20% of the data will be used for testing\n\n# Split the sampled data into training and testing sets\ntrain_df, val_df = train_test_split(df_encoded, test_size=TEST_SIZE, random_state=42)\n\n# Reset indices after splitting\ntrain_df = train_df.reset_index(drop=True)\nval_df = val_df.reset_index(drop=True)\n\nprint(f\"Total data size: {len(df_encoded)}\")\nprint(f\"Total data size for quick run: {len(quick_run_df)}\")\nprint(f\"Training set size: {len(train_df)}\")\nprint(f\"Testing set size: {len(val_df)}\")","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:37:14.291727Z","iopub.execute_input":"2024-09-09T20:37:14.292031Z","iopub.status.idle":"2024-09-09T20:37:14.334767Z","shell.execute_reply.started":"2024-09-09T20:37:14.292006Z","shell.execute_reply":"2024-09-09T20:37:14.333790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define PyTorch datasets for training and testing\ntrain_dataset = CustomDataset(train_df, features_list, y_label, transform=transform)\nval_dataset = CustomDataset(val_df, features_list, y_label, transform=transform)\n\n# Define DataLoaders\ntrain_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=32, shuffle=False)\n\nprint(f\"Number of batches in training set: {len(train_loader)}\")\nprint(f\"Number of batches in testing set: {len(val_loader)}\")","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:37:14.336498Z","iopub.execute_input":"2024-09-09T20:37:14.337066Z","iopub.status.idle":"2024-09-09T20:37:14.344070Z","shell.execute_reply.started":"2024-09-09T20:37:14.337029Z","shell.execute_reply":"2024-09-09T20:37:14.343044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\ndef visualize_batch(data_loader):\n    \"\"\"\n    Visualizes a batch of images and their corresponding labels from the DataLoader in a grid format.\n    \n    Parameters:\n    data_loader (DataLoader): The DataLoader containing the dataset.\n    \n    Returns:\n    None\n    \"\"\"\n    # Get one batch of data\n    images, features, labels = next(iter(data_loader))\n    \n    # Determine the number of images in the batch\n    batch_size = images.shape[0]\n    cols = 4  # Number of columns in the grid\n    rows = (batch_size + cols - 1) // cols  # Calculate the number of rows needed\n\n    fig, axes = plt.subplots(rows, cols, figsize=(15, 5 * rows))\n    axes = axes.flatten()  # Flatten the axes array for easy iteration\n\n    for i in range(batch_size):\n        img = images[i].permute(1, 2, 0).numpy()  # Convert tensor to NumPy array with HWC format\n        \n        # Plot image\n        axes[i].imshow(img, cmap='gray')\n        axes[i].set_title(f\"Label: {labels[i].item()}\\nFeatures: {features[i].tolist()}\", fontsize=8)\n        axes[i].axis('off')\n\n    # Hide any unused subplots\n    for i in range(batch_size, len(axes)):\n        axes[i].axis('off')\n\n    plt.tight_layout()\n    plt.show()\n\n# Visualize a batch from the training set\nvisualize_batch(train_loader)","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:37:19.991535Z","iopub.execute_input":"2024-09-09T20:37:19.992314Z","iopub.status.idle":"2024-09-09T20:37:26.064796Z","shell.execute_reply.started":"2024-09-09T20:37:19.992281Z","shell.execute_reply":"2024-09-09T20:37:26.063652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\ndef plot_label_distribution(train_df, test_df, y_label):\n    \"\"\"\n    Plots the distribution of labels in the training and validation sets.\n    \n    Parameters:\n    train_df (DataFrame): The training dataframe.\n    test_df (DataFrame): The validation dataframe.\n    y_label (str): The column name of the target label.\n    \n    Returns:\n    None\n    \"\"\"\n    # Calculate the label distribution in training and testing sets\n    train_distribution = train_df[y_label].value_counts().sort_index()\n    test_distribution = test_df[y_label].value_counts().sort_index()\n    \n    # Plot the distribution\n    fig, ax = plt.subplots(1, 2, figsize=(12, 5))\n    \n    train_distribution.plot(kind='bar', ax=ax[0], color='skyblue')\n    ax[0].set_title('Training Set Label Distribution')\n    ax[0].set_xlabel('Label')\n    ax[0].set_ylabel('Count')\n    \n    test_distribution.plot(kind='bar', ax=ax[1], color='lightcoral')\n    ax[1].set_title('Validation Set Label Distribution')\n    ax[1].set_xlabel('Label')\n    ax[1].set_ylabel('Count')\n    \n    plt.tight_layout()\n    plt.show()\n\n# Plot the distribution of labels in the training and validation sets\nplot_label_distribution(train_df, val_df, y_label)","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:40:04.391039Z","iopub.execute_input":"2024-09-09T20:40:04.392009Z","iopub.status.idle":"2024-09-09T20:40:04.851758Z","shell.execute_reply.started":"2024-09-09T20:40:04.391969Z","shell.execute_reply":"2024-09-09T20:40:04.850802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"# Set the device to GPU if available; otherwise, use CPU\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# Print the device being used\nprint(f\"Using device: {device}\")","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:40:08.158730Z","iopub.execute_input":"2024-09-09T20:40:08.159936Z","iopub.status.idle":"2024-09-09T20:40:08.190916Z","shell.execute_reply.started":"2024-09-09T20:40:08.159889Z","shell.execute_reply":"2024-09-09T20:40:08.189931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision.models as models\n\nclass EfficientNetWithFeatures(nn.Module):\n    def __init__(self, num_classes, num_features):\n        super(EfficientNetWithFeatures, self).__init__()\n\n        # Initialize EfficientNetV2 with pre-trained weights\n        self.efficientnet = models.efficientnet_v2_s(weights='IMAGENET1K_V1')\n\n        # Freeze all layers first\n        for param in self.efficientnet.parameters():\n            param.requires_grad = False\n            \n        # Get the list of all parameters in EfficientNet\n        all_layers = list(self.efficientnet.parameters())\n\n        # Unfreeze the last 20 layers\n        for param in all_layers[-20:]:\n            param.requires_grad = True\n\n        # Extract the number of input features from the last layer of the classifier\n        num_features_eff = self.efficientnet.classifier[-1].in_features\n\n        # Replace the classifier with an identity function to get the embeddings\n        self.efficientnet.classifier = nn.Identity()\n\n        # Define a more complex fully connected layer to combine EfficientNet embeddings with numerical features\n        self.fc1 = nn.Linear(num_features_eff + num_features, 256)\n        self.fc2 = nn.Linear(256, 128)\n        self.fc3 = nn.Linear(128, num_classes)\n        self.dropout = nn.Dropout(p=0.5)  # Dropout layer\n\n    def forward(self, image, features):\n        # Forward pass through EfficientNet to get embeddings\n        image_embedding = self.efficientnet(image)\n\n        # Concatenate EfficientNet embeddings with numerical features\n        combined_input = torch.cat((image_embedding, features), dim=1)\n\n        # Pass through the fully connected layers with dropout\n        x = torch.relu(self.fc1(combined_input))\n        x = self.dropout(x)  # Dropout after the first fully connected layer\n        x = torch.relu(self.fc2(x))\n        x = self.fc3(x)\n\n        return x\n\n\n# Define the number of classes and the number of numerical features\nnum_classes = 3  # Replace with the number of classes in your dataset\nnum_features = 10  # Replace with the number of numerical features\n\n# Initialize the custom model\nmodel = EfficientNetWithFeatures(num_classes=num_classes, num_features=num_features).to(device)","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:43:21.374435Z","iopub.execute_input":"2024-09-09T20:43:21.375391Z","iopub.status.idle":"2024-09-09T20:43:22.275967Z","shell.execute_reply.started":"2024-09-09T20:43:21.375345Z","shell.execute_reply":"2024-09-09T20:43:22.275144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import datetime\nimport torch\nfrom tqdm import tqdm\n\n# Function to train the model\ndef train_model(model, train_loader, criterion, optimizer, device):\n    model.train()  # Set the model to training mode\n    running_loss = 0.0\n    correct = 0\n    total = 0\n\n    for images, features, labels in tqdm(train_loader, desc=\"Training\"):\n        # Move data to the appropriate device\n        images, features, labels = images.to(device), features.to(device), labels.to(device)\n        \n        # Zero the parameter gradients\n        optimizer.zero_grad()\n\n        # Forward pass\n        outputs = model(images, features)\n        loss = criterion(outputs, labels)\n\n        # Backward pass and optimize\n        loss.backward()\n        optimizer.step()\n\n        # Update running loss and accuracy\n        running_loss += loss.item()\n        _, predicted = torch.max(outputs, 1)\n        total += labels.size(0)\n        correct += (predicted == labels).sum().item()\n\n    epoch_loss = running_loss / len(train_loader)\n    epoch_acc = 100 * correct / total\n    return epoch_loss, epoch_acc\n\n# Function to validate the model\ndef validate_model(model, val_loader, criterion, device):\n    model.eval()  # Set the model to evaluation mode\n    running_loss = 0.0\n    correct = 0\n    total = 0\n\n    with torch.no_grad():  # Disable gradient calculation for validation\n        for images, features, labels in tqdm(val_loader, desc=\"Validation\"):\n            # Move data to the appropriate device\n            images, features, labels = images.to(device), features.to(device), labels.to(device)\n\n            # Forward pass\n            outputs = model(images, features)\n            loss = criterion(outputs, labels)\n\n            # Update running loss and accuracy\n            running_loss += loss.item()\n            _, predicted = torch.max(outputs, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n\n    epoch_loss = running_loss / len(val_loader)\n    epoch_acc = 100 * correct / total\n    return epoch_loss, epoch_acc\n\n# Main training loop with early stopping and learning rate scheduler\ndef train_and_validate(model, train_loader, val_loader, criterion, optimizer, num_epochs, device, patience=5):\n    best_val_loss = float('inf')  # Initialize best validation loss to infinity\n    epochs_without_improvement = 0  # Counter for early stopping\n    save_path = f\"/kaggle/working/model_{datetime.now().strftime('%Y%m%d_%H%M%S')}.pth\"\n    \n    # Learning rate scheduler: Reduce LR when a metric has stopped improving\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=3, verbose=True)\n    \n    for epoch in range(num_epochs):\n        print(f\"Epoch {epoch+1}/{num_epochs}\")\n\n        # Train the model\n        train_loss, train_acc = train_model(model, train_loader, criterion, optimizer, device)\n        print(f\"Train Loss: {train_loss:.4f} | Train Acc: {train_acc:.2f}%\")\n\n        # Validate the model\n        val_loss, val_acc = validate_model(model, val_loader, criterion, device)\n        print(f\"Validation Loss: {val_loss:.4f} | Validation Acc: {val_acc:.2f}%\")\n\n        # Step with the scheduler\n        scheduler.step(val_loss)\n\n        # Check for improvement\n        if val_loss < best_val_loss:\n            print(f\"Validation loss decreased ({best_val_loss:.4f} --> {val_loss:.4f}). Saving model...\")\n            best_val_loss = val_loss\n            epochs_without_improvement = 0  # Reset the counter\n            torch.save(model.state_dict(), save_path)  # Save the best model\n        else:\n            epochs_without_improvement += 1\n            print(f\"No improvement in validation loss for {epochs_without_improvement} epochs.\")\n            model.load_state_dict(torch.load(save_path))  # Load the best model state\n\n        # Check if early stopping should be applied\n        if epochs_without_improvement >= patience:\n            print(f\"Early stopping triggered after {patience} epochs without improvement.\")\n            break\n\n    print(\"Training complete. Best validation loss:\", best_val_loss)\n    print(f\"Best model saved to: {save_path}\")\n    return save_path  # Return the path of the saved model\n\n# Example usage\nclass_weights = torch.tensor([1.0, 2.0, 4.0], device=device)  # Example weights for each class severity\ncriterion = nn.CrossEntropyLoss(weight=class_weights)\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n\n# Assuming train_loader, val_loader, num_epochs, and device are defined\n#saved_model_path = train_and_validate(model, train_loader, val_loader, criterion, optimizer, num_epochs, device)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:43:29.685618Z","iopub.execute_input":"2024-09-09T20:43:29.686004Z","iopub.status.idle":"2024-09-09T20:43:29.711338Z","shell.execute_reply.started":"2024-09-09T20:43:29.685973Z","shell.execute_reply":"2024-09-09T20:43:29.710419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set the number of epochs\nnum_epochs = 20\npatience = 5\nsaved_model_path = train_and_validate(model, train_loader, val_loader, criterion, optimizer, num_epochs, device, patience)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-09T20:48:30.823116Z","iopub.execute_input":"2024-09-09T20:48:30.823891Z","iopub.status.idle":"2024-09-09T20:48:36.880564Z","shell.execute_reply.started":"2024-09-09T20:48:30.823856Z","shell.execute_reply":"2024-09-09T20:48:36.878991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prepare test file for submission","metadata":{}},{"cell_type":"code","source":"test_df   = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_series_descriptions.csv')\ntest_df","metadata":{"execution":{"iopub.status.busy":"2024-09-08T20:09:56.065671Z","iopub.status.idle":"2024-09-08T20:09:56.065996Z","shell.execute_reply.started":"2024-09-08T20:09:56.065839Z","shell.execute_reply":"2024-09-08T20:09:56.065853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom tqdm import tqdm\n\n# Define mappings from series descriptions to conditions\ncondition_mapping = {\n    'Axial T2': ['Left Subarticular Stenosis', 'Right Subarticular Stenosis'],\n    'Sagittal T1': ['Right Neural Foraminal Narrowing', 'Left Neural Foraminal Narrowing', 'Spinal Canal Stenosis'],\n    'Sagittal T2/STIR': ['Spinal Canal Stenosis']\n}\n\n# Define all possible levels\nlevels = ['L1/L2', 'L2/L3', 'L3/L4', 'L4/L5', 'L5/S1']\n\n# Initialize a list to store each row's features\nexpanded_features = []\n\n# Main path for test images\nmain_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_images'\n\n# Iterate over each row in the test file\nfor idx, row in test_df.iterrows():\n    study_id = row['study_id']\n    series_id = row['series_id']\n    series_description = row['series_description']\n    \n    # Define the directory path for the current study_id and series_id\n    dir_path = os.path.join(main_path, str(study_id), str(series_id))\n    \n    # Check if the directory exists\n    if os.path.exists(dir_path):\n        # List all DICOM files (instances) in the directory\n        instance_files = [f for f in os.listdir(dir_path) if f.endswith('.dcm')]\n        \n        # Get instance numbers from the file names\n        instance_numbers = [int(f.split('.')[0]) for f in instance_files]\n        \n        # Generate rows for each instance\n        for instance_number in instance_numbers:\n            # Get the conditions corresponding to the series description\n            applicable_conditions = condition_mapping[series_description]\n\n            # Generate combinations for each condition and level\n            for condition in condition_mapping['Axial T2'] + condition_mapping['Sagittal T1'] + condition_mapping['Sagittal T2/STIR']:\n                for level in levels:\n                    features = {\n                        'study_id': study_id,\n                        'series_id': series_id,\n                        'instance_number': instance_number,\n                        'series_description': series_description,\n                        'condition': condition,\n                        'level': level\n                    }\n\n                    # Set True/False for the condition-level combination\n                    if condition in applicable_conditions:\n                        features['is_applicable'] = True\n                    else:\n                        features['is_applicable'] = False\n\n                    # Append features to the list\n                    expanded_features.append(features)\n\n# Convert to DataFrame\nexpanded_test_df = pd.DataFrame(expanded_features)\n\n# One-hot encode 'condition' and 'level' columns\nencoded_df = pd.get_dummies(expanded_test_df[['condition', 'level']], prefix=['condition', 'level'])\n\n# Concatenate the original DataFrame with the encoded columns\nexpanded_test_df = pd.concat([expanded_test_df, encoded_df], axis=1)\n\n# Keep only rows where 'is_applicable' is True\nexpanded_test_df = expanded_test_df[expanded_test_df['is_applicable']].reset_index(drop=True)\n\n# Generate the filepath for each row\ndef generate_filepath(row):\n    file_path = f\"{main_path}/{row['study_id']}/{row['series_id']}/{int(row['instance_number'])}.dcm\"\n    return file_path if os.path.exists(file_path) else None\n\n# Apply the filepath generation function\nexpanded_test_df['filepath'] = expanded_test_df.progress_apply(generate_filepath, axis=1)\n\nprint(\"Expanded Test DataFrame:\")\nexpanded_test_df","metadata":{"execution":{"iopub.status.busy":"2024-09-08T20:09:56.067466Z","iopub.status.idle":"2024-09-08T20:09:56.067766Z","shell.execute_reply.started":"2024-09-08T20:09:56.067616Z","shell.execute_reply":"2024-09-08T20:09:56.067628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"expanded_test_df['condition'].unique()","metadata":{"execution":{"iopub.status.busy":"2024-09-08T20:09:56.068783Z","iopub.status.idle":"2024-09-08T20:09:56.069099Z","shell.execute_reply.started":"2024-09-08T20:09:56.068943Z","shell.execute_reply":"2024-09-08T20:09:56.068957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create the dataset and data loader for the test set\ntest_dataset = CustomDataset(expanded_test_df, features_list, transform=transform)\ntest_loader = DataLoader(test_dataset, batch_size=32, shuffle=False)  # No shuffling for inference","metadata":{"execution":{"iopub.status.busy":"2024-09-08T20:09:56.070524Z","iopub.status.idle":"2024-09-08T20:09:56.070824Z","shell.execute_reply.started":"2024-09-08T20:09:56.070676Z","shell.execute_reply":"2024-09-08T20:09:56.070688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# Initialize results storage\nresults = {\n    'row_id': [],\n    'normal_mild': [],\n    'moderate': [],\n    'severe': []\n}\n\n# Display initial message\ndisplay(\"Starting inference on the test set...\")\n\n# Use tqdm to create a progress bar for the test_loader\nwith torch.no_grad():  # Disable gradient computation for inference\n    for batch_idx, (images, features) in enumerate(tqdm(test_loader, desc=\"Processing batches\")):\n        \n        # Move data to the appropriate device\n        images, features = images.to(device), features.to(device)\n\n        # Forward pass through the model\n        outputs = model(images, features)\n\n        # Get the predicted probabilities using softmax\n        probs = torch.softmax(outputs, dim=1)\n\n        # Iterate through the probabilities and corresponding rows in the batch\n        for i in range(len(probs)):\n            # Calculate the index in the DataFrame corresponding to this batch\n            df_index = batch_idx * test_loader.batch_size + i\n            \n            # Check if the index is within the bounds of the DataFrame\n            if df_index >= len(expanded_test_df):\n                continue  # Skip if index is out of bounds\n\n            # Extract study_id, condition, and level directly from the DataFrame\n            study_id = expanded_test_df.iloc[df_index]['study_id']\n            condition = expanded_test_df.iloc[df_index]['condition']\n            level = expanded_test_df.iloc[df_index]['level']\n\n            # Generate row_id\n            row_id = f\"{study_id}_{condition}_{level.replace('/', '_')}\".lower().replace(' ', '_')\n\n            # Append results\n            results['row_id'].append(row_id)\n            results['normal_mild'].append(probs[i, 0].item())  # Probability for 'Normal/Mild'\n            results['moderate'].append(probs[i, 1].item())    # Probability for 'Moderate'\n            results['severe'].append(probs[i, 2].item())      # Probability for 'Severe'\n\n# Convert the results to a DataFrame\nresults_df = pd.DataFrame(results)\n\n# Display the final results\ndisplay(results_df.head())\n","metadata":{"execution":{"iopub.status.busy":"2024-09-08T20:09:56.072424Z","iopub.status.idle":"2024-09-08T20:09:56.072755Z","shell.execute_reply.started":"2024-09-08T20:09:56.072594Z","shell.execute_reply":"2024-09-08T20:09:56.072609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_df['row_id'].unique()","metadata":{"execution":{"iopub.status.busy":"2024-09-08T20:09:56.073875Z","iopub.status.idle":"2024-09-08T20:09:56.074177Z","shell.execute_reply.started":"2024-09-08T20:09:56.074022Z","shell.execute_reply":"2024-09-08T20:09:56.074034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\n# Define all possible conditions and levels\nconditions = [\n    'Left Neural Foraminal Narrowing', 'Left Subarticular Stenosis',\n    'Right Neural Foraminal Narrowing', 'Right Subarticular Stenosis',\n    'Spinal Canal Stenosis'\n]\nlevels = ['L1/L2', 'L2/L3', 'L3/L4', 'L4/L5', 'L5/S1']\n\n# Function to check and generate missing combinations\ndef ensure_complete_results(results_df):\n    # List to store new rows for missing combinations\n    new_rows = []\n    \n    # Extract study_id from row_id by splitting on underscores\n    results_df['study_id'] = results_df['row_id'].apply(lambda x: x.split('_')[0])\n\n    # Group by 'study_id' to check each group separately\n    grouped = results_df.groupby('study_id')\n    \n    for study_id, group in grouped:\n        # Get current combinations for this study_id\n        current_combinations = set(group['row_id'])\n        \n        # Generate all possible combinations for this study_id\n        all_combinations = {\n            f\"{study_id}_{condition.lower().replace(' ', '_')}_{level.replace('/', '_')}\".lower()\n            for condition in conditions\n            for level in levels\n        }\n        \n        # Find missing combinations\n        missing_combinations = all_combinations - current_combinations\n        \n        # Generate rows for missing combinations\n        for missing_row_id in missing_combinations:\n            # Extract condition and level from the missing row_id\n            condition = '_'.join(missing_row_id.split('_')[1:-1])\n            level = missing_row_id.split('_')[-1].replace('_', '/')\n            \n            # Create a new row with default or statistical values\n            new_row = {\n                'row_id': missing_row_id,\n                'normal_mild': 1/3,  # Assign equal probabilities or adjust based on your strategy\n                'moderate': 1/3,\n                'severe': 1/3\n            }\n            \n            new_rows.append(new_row)\n\n    # Convert new rows to DataFrame\n    new_rows_df = pd.DataFrame(new_rows)\n    \n    # Combine with the original results\n    complete_results_df = pd.concat([results_df, new_rows_df], ignore_index=True)\n    \n    # Return sorted complete results\n    return complete_results_df.sort_values(by='row_id').reset_index(drop=True)\n\n# Example: Ensure the results are complete for submission\nresults_df = ensure_complete_results(results_df)\n\n# Verify that each study_id has exactly 25 rows\nstudy_id_counts = results_df['row_id'].str.split('_').str[0].value_counts()\ndisplay(study_id_counts)","metadata":{"execution":{"iopub.status.busy":"2024-09-08T20:09:56.075515Z","iopub.status.idle":"2024-09-08T20:09:56.075814Z","shell.execute_reply.started":"2024-09-08T20:09:56.075664Z","shell.execute_reply":"2024-09-08T20:09:56.075676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-08T20:09:56.077116Z","iopub.status.idle":"2024-09-08T20:09:56.077458Z","shell.execute_reply.started":"2024-09-08T20:09:56.077291Z","shell.execute_reply":"2024-09-08T20:09:56.077305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"averaged_results_df = results_df[['row_id','normal_mild', 'moderate', 'severe']].groupby('row_id', as_index=False).mean()\nsum_probs = averaged_results_df[['normal_mild', 'moderate', 'severe']].sum(axis=1)\n\n# Normalize the columns so that each row sums to 1\naveraged_results_df['normal_mild'] = averaged_results_df['normal_mild'] / sum_probs\naveraged_results_df['moderate'] = averaged_results_df['moderate'] / sum_probs\naveraged_results_df['severe'] = averaged_results_df['severe'] / sum_probs\n\n# Verify that the sum of the three columns is 1 for each row\naveraged_results_df['sum_check'] = averaged_results_df[['normal_mild', 'moderate', 'severe']].sum(axis=1).apply(lambda x: round(x,2))\naveraged_results_df","metadata":{"execution":{"iopub.status.busy":"2024-09-08T20:09:56.079547Z","iopub.status.idle":"2024-09-08T20:09:56.079981Z","shell.execute_reply.started":"2024-09-08T20:09:56.079754Z","shell.execute_reply":"2024-09-08T20:09:56.079773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_df = averaged_results_df[['row_id', 'normal_mild', 'moderate', 'severe']]\nfinal_df","metadata":{"execution":{"iopub.status.busy":"2024-09-08T20:09:56.081023Z","iopub.status.idle":"2024-09-08T20:09:56.081478Z","shell.execute_reply.started":"2024-09-08T20:09:56.081239Z","shell.execute_reply":"2024-09-08T20:09:56.081259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_df.to_csv(\"/kaggle/working/submission.csv\", index=False)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-08T20:09:56.082712Z","iopub.status.idle":"2024-09-08T20:09:56.083130Z","shell.execute_reply.started":"2024-09-08T20:09:56.082912Z","shell.execute_reply":"2024-09-08T20:09:56.082930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}