{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":114436,"sourceType":"modelInstanceVersion","modelInstanceId":92093,"modelId":116297}],"dockerImageVersionId":30762,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport cv2\nimport pydicom\nimport numpy as np\nimport os\nimport glob\nfrom tqdm import tqdm\nfrom tqdm.auto import tqdm \nimport warnings\ntqdm.pandas()  # Enable the pandas extension\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-16T17:08:07.249257Z","iopub.execute_input":"2024-09-16T17:08:07.249546Z","iopub.status.idle":"2024-09-16T17:08:08.043862Z","shell.execute_reply.started":"2024-09-16T17:08:07.249513Z","shell.execute_reply":"2024-09-16T17:08:08.042757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as transforms\nimport torch\nimport pydicom\nimport numpy as np\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\n\n# Define the function to read DICOM images\ndef read_dicom(file_path):\n    \"\"\"\n    Reads a DICOM file from the given file path and returns the pixel data.\n    \n    Parameters:\n    file_path (str): The path to the DICOM file.\n    \n    Returns:\n    numpy.ndarray: The pixel data from the DICOM file, or None if an error occurs.\n    \"\"\"\n    try:\n        # Read the DICOM file\n        dicom_image = pydicom.dcmread(file_path)\n        \n        # Extract the pixel data and convert to float32\n        image_array = dicom_image.pixel_array.astype(np.float32)\n        \n        return image_array\n    \n    except Exception as e:\n        print(f\"An error occurred while reading the DICOM file {file_path}: {e}\")\n        return None\n\n# Define a custom dataset class\nclass CustomDataset(Dataset):\n    def __init__(self, dataframe, features_list, y_label=None, transform=None):\n        \"\"\"\n        Initializes the CustomDataset.\n        \n        Parameters:\n        dataframe (DataFrame): The dataframe containing data.\n        features_list (list of str): List of column names to be used as features.\n        y_label (str, optional): Column name of the target label. Defaults to None for inference.\n        transform (callable, optional): A function/transform to apply to the images.\n        \"\"\"\n        self.dataframe = dataframe\n        self.features_list = features_list\n        self.y_label = y_label\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.dataframe)\n\n\n    def __getitem__(self, index):\n        # Load the DICOM image\n        image_path = self.dataframe['filepath'].iloc[index]\n        image = read_dicom(image_path)\n\n        # Normalize the image to range [0, 1]\n        if image.max() != image.min():\n            image = (image - image.min()) / (image.max() - image.min())\n        else:\n            image = np.zeros_like(image, dtype=np.float32)\n        image = image.astype(np.float32)\n\n        # Convert to 3-channel image by stacking\n        image = np.stack([image, image, image], axis=-1)  # Shape: (H, W, 3)\n\n        # Apply the transform if provided\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        else:\n            image = torch.tensor(image).permute(2, 0, 1)  # Convert to tensor and rearrange dimensions\n\n        # Extract the features\n        features = self.dataframe[self.features_list].iloc[index].values.astype(np.float32)\n        features = torch.tensor(features)\n\n        # If y_label is provided, extract the label\n        if self.y_label:\n            label = self.dataframe[self.y_label].iloc[index]\n            label = torch.tensor(label, dtype=torch.long)\n            return image, features, label\n        else:\n            return image, features\n        \n# Define the features and label columns\nfeatures_list = [\n    'condition_Left Neural Foraminal Narrowing',\n    'condition_Left Subarticular Stenosis',\n    'condition_Right Neural Foraminal Narrowing',\n    'condition_Right Subarticular Stenosis',\n    'condition_Spinal Canal Stenosis',\n    'level_L1/L2',\n    'level_L2/L3',\n    'level_L3/L4',\n    'level_L4/L5',\n    'level_L5/S1'\n]\n\ny_label = 'severity'\n# Visualization function\nnormalize_mean = (0.485, 0.456, 0.406)\nnormalize_std = (0.229, 0.224, 0.225)\n# Define the transforms\ntransform = A.Compose([\n    A.Resize(384, 384),\n    A.Normalize(mean=normalize_mean, std=normalize_std, max_pixel_value=1.0),\n    ToTensorV2(),\n])","metadata":{"execution":{"iopub.status.busy":"2024-09-16T17:08:08.045544Z","iopub.execute_input":"2024-09-16T17:08:08.045969Z","iopub.status.idle":"2024-09-16T17:08:34.433407Z","shell.execute_reply.started":"2024-09-16T17:08:08.045935Z","shell.execute_reply":"2024-09-16T17:08:34.432275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set the device to GPU if available; otherwise, use CPU\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# Print the device being used\nprint(f\"Using device: {device}\")","metadata":{"execution":{"iopub.status.busy":"2024-09-16T17:08:34.434769Z","iopub.execute_input":"2024-09-16T17:08:34.435422Z","iopub.status.idle":"2024-09-16T17:08:34.471253Z","shell.execute_reply.started":"2024-09-16T17:08:34.435371Z","shell.execute_reply":"2024-09-16T17:08:34.469914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision.models as models\n\nclass EfficientNetWithFeatures(nn.Module):\n    def __init__(self, num_classes, num_features):\n        super(EfficientNetWithFeatures, self).__init__()\n\n        # Initialize EfficientNetV2 with pre-trained weights\n        self.efficientnet = models.efficientnet_v2_s(weights=None)\n\n        # Freeze all layers first\n        for param in self.efficientnet.parameters():\n            param.requires_grad = False\n            \n        # Get the list of all parameters in EfficientNet\n        all_layers = list(self.efficientnet.parameters())\n\n        # Unfreeze the last 20 layers\n        for param in all_layers[-20:]:\n            param.requires_grad = True\n\n        # Extract the number of input features from the last layer of the classifier\n        num_features_eff = self.efficientnet.classifier[-1].in_features\n\n        # Replace the classifier with an identity function to get the embeddings\n        self.efficientnet.classifier = nn.Identity()\n\n        # Define a more complex fully connected layer to combine EfficientNet embeddings with numerical features\n        self.fc1 = nn.Linear(num_features_eff + num_features, 256)\n        self.fc2 = nn.Linear(256, 128)\n        self.fc3 = nn.Linear(128, num_classes)\n        self.dropout = nn.Dropout(p=0.5)  # Dropout layer\n\n    def forward(self, image, features):\n        # Forward pass through EfficientNet to get embeddings\n        image_embedding = self.efficientnet(image)\n\n        # Concatenate EfficientNet embeddings with numerical features\n        combined_input = torch.cat((image_embedding, features), dim=1)\n\n        # Pass through the fully connected layers with dropout\n        x = torch.relu(self.fc1(combined_input))\n        x = self.dropout(x)  # Dropout after the first fully connected layer\n        x = torch.relu(self.fc2(x))\n        x = self.fc3(x)\n\n        return x\n\n# Define the number of classes and the number of numerical features\nnum_classes = 3  # Replace with the number of classes in your dataset\nnum_features = 10  # Replace with the number of numerical features\n\n# Initialize the model\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = EfficientNetWithFeatures(num_classes=num_classes, num_features=num_features).to(device)\n\n# Path to the saved model weights\nweights_path = '/kaggle/input/pytorch-efficientnet-imagenet1k_v1-rsna-2024/pytorch/default/2/model_20240915_130114.pth'\n# Load the model weights\nmodel.load_state_dict(torch.load(weights_path, map_location=device))\n\n# Set the model to evaluation mode\nmodel.eval()\n\nprint(\"Model loaded successfully and ready for inference.\")","metadata":{"execution":{"iopub.status.busy":"2024-09-16T17:08:34.47339Z","iopub.execute_input":"2024-09-16T17:08:34.473698Z","iopub.status.idle":"2024-09-16T17:08:36.752878Z","shell.execute_reply.started":"2024-09-16T17:08:34.473666Z","shell.execute_reply":"2024-09-16T17:08:36.751959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df   = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_series_descriptions.csv')\ntest_df","metadata":{"execution":{"iopub.status.busy":"2024-09-16T17:08:36.754059Z","iopub.execute_input":"2024-09-16T17:08:36.754374Z","iopub.status.idle":"2024-09-16T17:08:36.782858Z","shell.execute_reply.started":"2024-09-16T17:08:36.754341Z","shell.execute_reply":"2024-09-16T17:08:36.781913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ------------------------------\n# 1. Data Preparation\n# ------------------------------\n\nimport os\nimport pandas as pd\nfrom tqdm import tqdm\nimport numpy as np\n\n# Ensure tqdm pandas integration\ntqdm.pandas()\n\n# Define mappings from series descriptions to conditions\ncondition_mapping = {\n    'Axial T2': ['Left Subarticular Stenosis', 'Right Subarticular Stenosis'],\n    'Sagittal T1': ['Right Neural Foraminal Narrowing', 'Left Neural Foraminal Narrowing'],\n    'Sagittal T2/STIR': ['Spinal Canal Stenosis']\n}\n\n# Define all possible levels\nlevels = ['L1/L2', 'L2/L3', 'L3/L4', 'L4/L5', 'L5/S1']\n\n# Initialize a list to store each row's features\nexpanded_features = []\n\n# Path to the test images\nmain_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_images'\n\n# Assuming you have a test_df loaded. If not, load it accordingly.\n# For example:\n# test_df = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test.csv')\n\n# Iterate over each row in the test file\nfor idx, row in tqdm(test_df.iterrows(), total=test_df.shape[0], desc=\"Expanding Test DataFrame\"):\n    study_id = row['study_id']\n    series_id = row['series_id']\n    series_description = row['series_description']\n    \n    # Define the directory path for the current study_id and series_id\n    dir_path = os.path.join(main_path, str(study_id), str(series_id))\n    \n    # Check if the directory exists\n    if os.path.exists(dir_path):\n        # List all DICOM files (instances) in the directory\n        instance_files = [f for f in os.listdir(dir_path) if f.endswith('.dcm')]\n        \n        # Get instance numbers from the file names\n        instance_numbers = [int(f.split('.')[0]) for f in instance_files]\n        \n        # Get the modality from the series_description\n        # This assumes that the modality can be inferred from the series_description keywords\n        # Adjust the logic as needed based on actual series_description formats\n        if 'Axial T2' in series_description:\n            modality = 'Axial T2'\n        elif 'Sagittal T1' in series_description:\n            modality = 'Sagittal T1'\n        elif 'Sagittal T2/STIR' in series_description:\n            modality = 'Sagittal T2/STIR'\n        else:\n            modality = 'Unknown'  # Handle unexpected modalities\n        \n        # Get the applicable conditions for the current modality\n        applicable_conditions = condition_mapping.get(modality, [])\n        \n        # Generate rows for each instance\n        for instance_number in instance_numbers:\n            for condition in applicable_conditions:\n                for level in levels:\n                    features = {\n                        'study_id': study_id,\n                        'series_id': series_id,\n                        'instance_number': instance_number,\n                        'series_description': series_description,\n                        'condition': condition,\n                        'level': level,\n                        'modality': modality\n                    }\n                    \n                    # Append features to the list\n                    expanded_features.append(features)\n    else:\n        # Handle missing directories if necessary\n        print(f\"Directory does not exist: {dir_path}\")\n\n# Convert to DataFrame\nexpanded_test_df = pd.DataFrame(expanded_features)\n\n# Display the expanded DataFrame\nprint(\"Expanded Test DataFrame:\")\ndisplay(expanded_test_df.head())\n\n# ------------------------------\n# 1.1 One-Hot Encoding\n# ------------------------------\n\n# One-hot encode 'condition' and 'level' columns\nencoded_conditions = pd.get_dummies(expanded_test_df['condition'], prefix='condition')\nencoded_levels = pd.get_dummies(expanded_test_df['level'], prefix='level')\n\n# Concatenate the original DataFrame with the encoded columns\nexpanded_test_df = pd.concat([expanded_test_df, encoded_conditions, encoded_levels], axis=1)\n\n# ------------------------------\n# 1.2 Generate Filepaths\n# ------------------------------\n\n# Function to generate the filepath for each row\ndef generate_filepath(row):\n    file_path = os.path.join(\n        main_path,\n        str(row['study_id']),\n        str(row['series_id']),\n        f\"{int(row['instance_number'])}.dcm\"\n    )\n    return file_path if os.path.exists(file_path) else None\n\n# Apply the filepath generation function with progress bar\nexpanded_test_df['filepath'] = expanded_test_df.progress_apply(generate_filepath, axis=1)\n\n# ------------------------------\n# 1.3 Create Condition-Modal Mapping DataFrame\n# ------------------------------\n\n# Create a DataFrame from the condition_mapping\ncondition_modal_df = pd.DataFrame([\n    {'modality': modality, 'condition': condition}\n    for modality, conditions in condition_mapping.items()\n    for condition in conditions\n])\n\n# Display the condition-modal mapping\nprint(\"Condition-Modal Mapping DataFrame:\")\ndisplay(condition_modal_df.head())\n\n# ------------------------------\n# 1.4 Remove Non-Applicable Rows (Optional)\n# ------------------------------\n\n# If you want to keep only applicable rows (where 'is_applicable' is True)\n# Since we already filtered conditions based on modality, this step might be redundant\n# However, if you have a separate 'is_applicable' flag, you can filter accordingly\n# expanded_test_df = expanded_test_df[expanded_test_df['is_applicable']].reset_index(drop=True)\n\n# ------------------------------\n# 1.5 Final Expanded Test DataFrame\n# ------------------------------\n\nprint(\"Final Expanded Test DataFrame:\")\ndisplay(expanded_test_df.head())","metadata":{"execution":{"iopub.status.busy":"2024-09-16T17:09:32.257597Z","iopub.execute_input":"2024-09-16T17:09:32.25801Z","iopub.status.idle":"2024-09-16T17:09:32.414815Z","shell.execute_reply.started":"2024-09-16T17:09:32.257972Z","shell.execute_reply":"2024-09-16T17:09:32.413949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"expanded_test_df['condition'].unique()","metadata":{"execution":{"iopub.status.busy":"2024-09-16T17:10:36.535837Z","iopub.execute_input":"2024-09-16T17:10:36.53624Z","iopub.status.idle":"2024-09-16T17:10:36.543402Z","shell.execute_reply.started":"2024-09-16T17:10:36.536173Z","shell.execute_reply":"2024-09-16T17:10:36.54254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Determine the number of workers automatically\nimport multiprocessing\n\nnum_workers = multiprocessing.cpu_count()\nprint(f\"Number of CPU cores available: {num_workers}\")","metadata":{"execution":{"iopub.status.busy":"2024-09-16T17:10:37.218699Z","iopub.execute_input":"2024-09-16T17:10:37.219049Z","iopub.status.idle":"2024-09-16T17:10:37.224244Z","shell.execute_reply.started":"2024-09-16T17:10:37.219013Z","shell.execute_reply":"2024-09-16T17:10:37.223245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create the dataset and data loader for the test set\ntest_dataset = CustomDataset(expanded_test_df, features_list, transform=transform)\ntest_loader = DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=num_workers)  # No shuffling for inference","metadata":{"execution":{"iopub.status.busy":"2024-09-16T17:10:37.947582Z","iopub.execute_input":"2024-09-16T17:10:37.947925Z","iopub.status.idle":"2024-09-16T17:10:37.953306Z","shell.execute_reply.started":"2024-09-16T17:10:37.94789Z","shell.execute_reply":"2024-09-16T17:10:37.952379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ------------------------------\n# 2. Inference\n# ------------------------------\n\nimport torch\nimport pandas as pd\nfrom tqdm import tqdm\n\n# Ensure your model is in evaluation mode and moved to the appropriate device\n# Example:\n# device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n# model = EfficientNetWithFeatures(num_classes=3, num_features=<your_num_features>, num_modalities=<num_modalities>).to(device)\n# model.load_state_dict(torch.load('path_to_best_model.pth'))\n# model.eval()\n\n# Define the condition-modal mapping DataFrame (if not already defined)\n# condition_modal_df = ... (from Data Preparation section)\n\n# Define all possible conditions and levels\nconditions = [\n    'Left Neural Foraminal Narrowing', 'Left Subarticular Stenosis',\n    'Right Neural Foraminal Narrowing', 'Right Subarticular Stenosis',\n    'Spinal Canal Stenosis'\n]\nlevels = ['L1/L2', 'L2/L3', 'L3/L4', 'L4/L5', 'L5/S1']\n\n# Function to generate masks based on modality-condition association\ndef generate_masks(row, condition_modal_df):\n    \"\"\"\n    Generates a mask for the given row based on modality-condition association.\n\n    Parameters:\n    - row (Series): A row from expanded_test_df.\n    - condition_modal_df (DataFrame): DataFrame mapping modalities to conditions.\n\n    Returns:\n    - mask (list): A list indicating valid (1) or invalid (0) for each class.\n    \"\"\"\n    modality = row['modality']\n    condition = row['condition']\n    \n    # Check if the condition is valid for the given modality\n    valid_conditions = condition_modal_df[condition_modal_df['modality'] == modality]['condition'].tolist()\n    \n    if condition not in valid_conditions:\n        # If condition is invalid for the modality, set all probabilities to zero\n        mask = [0, 0, 0]\n    else:\n        # If condition is valid, no masking needed (all classes are possible)\n        mask = [1, 1, 1]\n    \n    return mask\n\n# Function to ensure all condition-level combinations are present\ndef ensure_complete_results(results_df, condition_modal_df, conditions, levels):\n    \"\"\"\n    Ensures that all condition-level combinations are present for each study_id.\n\n    Parameters:\n    - results_df (DataFrame): The DataFrame containing inference results.\n    - condition_modal_df (DataFrame): DataFrame mapping modalities to conditions.\n    - conditions (list): List of all possible conditions.\n    - levels (list): List of all possible levels.\n\n    Returns:\n    - complete_results_df (DataFrame): The complete DataFrame with all combinations.\n    \"\"\"\n    # Extract study_id from row_id\n    results_df['study_id'] = results_df['row_id'].apply(lambda x: x.split('_')[0])\n\n    # Initialize a list to collect new rows\n    new_rows = []\n\n    # Group by study_id\n    grouped = results_df.groupby('study_id')\n\n    for study_id, group in grouped:\n        # Current combinations\n        current_combinations = set(group['row_id'])\n        \n        # Get modality for the study_id from the original expanded_test_df\n        study_modalities = expanded_test_df[expanded_test_df['study_id'] == study_id]['modality'].unique()\n        \n        # Assuming each study_id has a single modality\n        if len(study_modalities) > 1:\n            print(f\"Study ID {study_id} has multiple modalities: {study_modalities}\")\n            # Handle accordingly, e.g., skip or assign default\n            continue\n        elif len(study_modalities) == 0:\n            print(f\"No modality found for Study ID {study_id}. Skipping.\")\n            continue\n        else:\n            modality = study_modalities[0]\n        \n        # Get valid conditions for the modality\n        valid_conditions = condition_modal_df[condition_modal_df['modality'] == modality]['condition'].tolist()\n        \n        for condition in valid_conditions:\n            for level in levels:\n                row_id = f\"{study_id}_{condition.lower().replace(' ', '_')}_{level.replace('/', '_')}\".lower()\n                if row_id not in current_combinations:\n                    # Assign default probabilities\n                    new_row = {\n                        'row_id': row_id,\n                        'normal_mild': 1/3,\n                        'moderate': 1/3,\n                        'severe': 1/3\n                    }\n                    new_rows.append(new_row)\n    \n    # Create DataFrame for new rows\n    new_rows_df = pd.DataFrame(new_rows)\n    \n    # Concatenate with original results\n    complete_results_df = pd.concat([results_df, new_rows_df], ignore_index=True)\n    \n    # Drop the temporary 'study_id' column\n    complete_results_df = complete_results_df.drop(columns=['study_id'])\n    \n    return complete_results_df\n\n# Initialize results storage\nresults = {\n    'row_id': [],\n    'normal_mild': [],\n    'moderate': [],\n    'severe': []\n}\n\n# Display initial message\nprint(\"Starting inference on the test set...\")\n\n# Use tqdm to create a progress bar for the test_loader\nwith torch.no_grad():  # Disable gradient computation for inference\n    for batch_idx, (images, features) in enumerate(tqdm(test_loader, desc=\"Processing batches\")):\n        \n        # Move data to the appropriate device\n        images, features = images.to(device), features.to(device)\n\n        # Forward pass through the model\n        outputs = model(images, features)\n\n        # Get the predicted probabilities using softmax\n        probs = torch.softmax(outputs, dim=1).cpu().numpy()\n\n        # Calculate batch indices\n        batch_start = batch_idx * test_loader.batch_size\n        batch_end = batch_start + images.size(0)\n        \n        # Iterate through the batch\n        for i in range(len(probs)):\n            # Calculate the index in the DataFrame\n            df_index = batch_start + i\n            \n            if df_index >= len(expanded_test_df):\n                continue  # Skip if index is out of bounds\n\n            # Extract necessary information from the DataFrame\n            row = expanded_test_df.iloc[df_index]\n            study_id = row['study_id']\n            condition = row['condition']\n            level = row['level']\n            modality = row['modality']\n\n            # Generate row_id\n            row_id = f\"{study_id}_{condition}_{level.replace('/', '_')}\".lower().replace(' ', '_')\n\n            # Generate mask based on modality and condition\n            mask = generate_masks(row, condition_modal_df)\n            \n            # Apply mask to probabilities\n            masked_probs = probs[i] * mask\n            \n            # If all probabilities are zero (invalid condition), assign default uniform probabilities\n            if masked_probs.sum() == 0:\n                masked_probs = np.array([1/3, 1/3, 1/3])\n            else:\n                # Renormalize the probabilities so that they sum to 1\n                masked_probs = masked_probs / masked_probs.sum()\n\n            # Append results\n            results['row_id'].append(row_id)\n            results['normal_mild'].append(masked_probs[0])  # Probability for 'Normal/Mild'\n            results['moderate'].append(masked_probs[1])    # Probability for 'Moderate'\n            results['severe'].append(masked_probs[2])      # Probability for 'Severe'\n\n# Convert the results to a DataFrame\nresults_df = pd.DataFrame(results)\n\n# Display intermediate results\nprint(\"Intermediate Inference Results:\")\ndisplay(results_df.head())\n\n# Ensure completeness by generating missing combinations\nresults_df = ensure_complete_results(results_df, condition_modal_df, conditions, levels)\n\n# Display after ensuring completeness\nprint(\"Results after ensuring completeness:\")\ndisplay(results_df.head())\n\n# Verify that each study_id has exactly 25 rows (assuming 5 conditions x 5 levels per modality)\nexpected_rows_per_study = len(condition_mapping) * len(levels)  # Adjust based on actual conditions per modality\nstudy_id_counts = results_df['row_id'].str.split('_').str[0].value_counts()\nprint(\"Study ID counts (should be equal for each study_id):\")\nprint(study_id_counts.head())\n\n# Average the probabilities for duplicated row_ids (if any)\naveraged_results_df = results_df.groupby('row_id', as_index=False).mean()\n\n# Normalize the probabilities so that they sum to 1 for each row\nsum_probs = averaged_results_df[['normal_mild', 'moderate', 'severe']].sum(axis=1)\naveraged_results_df['normal_mild'] = averaged_results_df['normal_mild'] / sum_probs\naveraged_results_df['moderate'] = averaged_results_df['moderate'] / sum_probs\naveraged_results_df['severe'] = averaged_results_df['severe'] / sum_probs\n\n# Verify that the sum of the three columns is approximately 1 for each row\naveraged_results_df['sum_check'] = averaged_results_df[['normal_mild', 'moderate', 'severe']].sum(axis=1).apply(lambda x: round(x,2))\nprint(\"Normalization Check:\")\ndisplay(averaged_results_df[['row_id', 'sum_check']].head())\n\n# Final Results DataFrame ready for submission\nfinal_submission_df = averaged_results_df[['row_id', 'normal_mild', 'moderate', 'severe']]\n\n# Display the final submission DataFrame\nprint(\"Final Submission DataFrame:\")\ndisplay(final_submission_df.head())\n\n# Save the final submission to a CSV file\nfinal_submission_df.to_csv('submission.csv', index=False)\n\nprint(\"Inference complete. Submission file 'submission.csv' is ready.\")","metadata":{"execution":{"iopub.status.busy":"2024-09-16T17:10:38.923732Z","iopub.execute_input":"2024-09-16T17:10:38.924144Z","iopub.status.idle":"2024-09-16T17:10:52.097199Z","shell.execute_reply.started":"2024-09-16T17:10:38.924105Z","shell.execute_reply":"2024-09-16T17:10:52.096345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# final_submission_df","metadata":{"execution":{"iopub.status.busy":"2024-09-16T17:12:08.345138Z","iopub.execute_input":"2024-09-16T17:12:08.345534Z","iopub.status.idle":"2024-09-16T17:12:08.360927Z","shell.execute_reply.started":"2024-09-16T17:12:08.345497Z","shell.execute_reply":"2024-09-16T17:12:08.359948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import pandas as pd\n\n# # Define all possible conditions and levels\n# conditions = [\n#     'Left Neural Foraminal Narrowing', 'Left Subarticular Stenosis',\n#     'Right Neural Foraminal Narrowing', 'Right Subarticular Stenosis',\n#     'Spinal Canal Stenosis'\n# ]\n# levels = ['L1/L2', 'L2/L3', 'L3/L4', 'L4/L5', 'L5/S1']\n\n# # Function to check and generate missing combinations\n# def ensure_complete_results(results_df):\n#     # List to store new rows for missing combinations\n#     new_rows = []\n    \n#     # Extract study_id from row_id by splitting on underscores\n#     results_df['study_id'] = results_df['row_id'].apply(lambda x: x.split('_')[0])\n\n#     # Group by 'study_id' to check each group separately\n#     grouped = results_df.groupby('study_id')\n    \n#     for study_id, group in grouped:\n#         # Get current combinations for this study_id\n#         current_combinations = set(group['row_id'])\n        \n#         # Generate all possible combinations for this study_id\n#         all_combinations = {\n#             f\"{study_id}_{condition.lower().replace(' ', '_')}_{level.replace('/', '_')}\".lower()\n#             for condition in conditions\n#             for level in levels\n#         }\n        \n#         # Find missing combinations\n#         missing_combinations = all_combinations - current_combinations\n        \n#         # Generate rows for missing combinations\n#         for missing_row_id in missing_combinations:\n#             # Extract condition and level from the missing row_id\n#             condition = '_'.join(missing_row_id.split('_')[1:-1])\n#             level = missing_row_id.split('_')[-1].replace('_', '/')\n            \n#             # Create a new row with default or statistical values\n#             new_row = {\n#                 'row_id': missing_row_id,\n#                 'normal_mild': 1/3,  # Assign equal probabilities or adjust based on your strategy\n#                 'moderate': 1/3,\n#                 'severe': 1/3\n#             }\n            \n#             new_rows.append(new_row)\n\n#     # Convert new rows to DataFrame\n#     new_rows_df = pd.DataFrame(new_rows)\n    \n#     # Combine with the original results\n#     complete_results_df = pd.concat([results_df, new_rows_df], ignore_index=True)\n    \n#     # Return sorted complete results\n#     return complete_results_df.sort_values(by='row_id').reset_index(drop=True)\n\n# # Example: Ensure the results are complete for submission\n# results_df = ensure_complete_results(results_df)\n\n# # Verify that each study_id has exactly 25 rows\n# study_id_counts = results_df['row_id'].str.split('_').str[0].value_counts()\n# display(study_id_counts)","metadata":{"execution":{"iopub.status.busy":"2024-09-16T00:09:22.071474Z","iopub.status.idle":"2024-09-16T00:09:22.071878Z","shell.execute_reply.started":"2024-09-16T00:09:22.071662Z","shell.execute_reply":"2024-09-16T00:09:22.071682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# results_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-16T00:09:22.07394Z","iopub.status.idle":"2024-09-16T00:09:22.074303Z","shell.execute_reply.started":"2024-09-16T00:09:22.074121Z","shell.execute_reply":"2024-09-16T00:09:22.074139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# averaged_results_df = results_df[['row_id','normal_mild', 'moderate', 'severe']].groupby('row_id', as_index=False).mean()\n# sum_probs = averaged_results_df[['normal_mild', 'moderate', 'severe']].sum(axis=1)\n\n# # Normalize the columns so that each row sums to 1\n# averaged_results_df['normal_mild'] = averaged_results_df['normal_mild'] / sum_probs\n# averaged_results_df['moderate'] = averaged_results_df['moderate'] / sum_probs\n# averaged_results_df['severe'] = averaged_results_df['severe'] / sum_probs\n\n# # Verify that the sum of the three columns is 1 for each row\n# averaged_results_df['sum_check'] = averaged_results_df[['normal_mild', 'moderate', 'severe']].sum(axis=1).apply(lambda x: round(x,2))\n# averaged_results_df","metadata":{"execution":{"iopub.status.busy":"2024-09-16T00:09:22.075596Z","iopub.status.idle":"2024-09-16T00:09:22.076023Z","shell.execute_reply.started":"2024-09-16T00:09:22.075775Z","shell.execute_reply":"2024-09-16T00:09:22.075793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# final_df = averaged_results_df[['row_id', 'normal_mild', 'moderate', 'severe']]\n# final_df","metadata":{"execution":{"iopub.status.busy":"2024-09-16T00:09:22.077345Z","iopub.status.idle":"2024-09-16T00:09:22.07774Z","shell.execute_reply.started":"2024-09-16T00:09:22.077542Z","shell.execute_reply":"2024-09-16T00:09:22.077562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# final_df.to_csv(\"/kaggle/working/submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-09-16T00:09:22.079363Z","iopub.status.idle":"2024-09-16T00:09:22.079792Z","shell.execute_reply.started":"2024-09-16T00:09:22.079562Z","shell.execute_reply":"2024-09-16T00:09:22.079583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}