{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":9528888,"sourceType":"datasetVersion","datasetId":5788739},{"sourceId":9553895,"sourceType":"datasetVersion","datasetId":5821406},{"sourceId":9577712,"sourceType":"datasetVersion","datasetId":5839069}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np\nfrom PIL import Image\nimport pydicom\nimport cv2\nimport torch.nn as nn\nimport pandas as pd\nimport torch\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom torchvision import transforms\nfrom transformers import AutoModel\nimport torchvision.models as models\nimport requests\nfrom tensorflow.keras.models import load_model\nimport kagglehub","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-13T20:49:56.505707Z","iopub.execute_input":"2024-10-13T20:49:56.506428Z","iopub.status.idle":"2024-10-13T20:50:13.820157Z","shell.execute_reply.started":"2024-10-13T20:49:56.506383Z","shell.execute_reply":"2024-10-13T20:50:13.819211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root_directory = \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images\"\ndescription_csv = \"/kaggle/input/preprocessed-dataset/unseen_unheard/unseen_unheard_data_complete.csv\"\nsolution_csv = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv'","metadata":{"execution":{"iopub.status.busy":"2024-10-13T20:49:42.417981Z","iopub.execute_input":"2024-10-13T20:49:42.418837Z","iopub.status.idle":"2024-10-13T20:49:42.431695Z","shell.execute_reply.started":"2024-10-13T20:49:42.418784Z","shell.execute_reply":"2024-10-13T20:49:42.430695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Downloading Model**","metadata":{}},{"cell_type":"code","source":"resnet50_AT2 = models.resnet50(pretrained=False)\nresnet50_ST1 = models.resnet50(pretrained=False)\nresnet50_ST2 = models.resnet50(pretrained=False)\n\nweights_path_AT2 = '/kaggle/input/medicalnet-attention-layers-for-rsna/AT2_attention_model_hist.pth'\nweights_path_ST1 = '/kaggle/input/medicalnet-attention-layers-for-rsna/ST1_attention_model_gsl.pth'\nweights_path_ST2 = '/kaggle/input/medicalnet-attention-layers-for-rsna/ST2_attention_model_hist.pth'\n\ntry:\n    resnet50_AT2.load_state_dict(torch.load(weights_path_AT2), strict=False)\n    resnet50_ST1.load_state_dict(torch.load(weights_path_ST1), strict=False)\n    resnet50_ST2.load_state_dict(torch.load(weights_path_ST2), strict=False)\n    \n    print(\"ResNet-152 model loaded with custom weights (missing keys ignored).\")\nexcept RuntimeError as e:\n    print(f\"Failed to load weights: {e}\")","metadata":{"execution":{"iopub.status.busy":"2024-10-13T19:51:31.086750Z","iopub.execute_input":"2024-10-13T19:51:31.087075Z","iopub.status.idle":"2024-10-13T19:51:33.082890Z","shell.execute_reply.started":"2024-10-13T19:51:31.087043Z","shell.execute_reply":"2024-10-13T19:51:33.081736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Greyscale 256x256**","metadata":{}},{"cell_type":"code","source":"\nBATCH_SIZE_LIMIT = 3 * (1024 ** 3)\n\ndef get_image_size_in_bytes(img_array):\n    return img_array.nbytes\n\ndef process_dicom_to_numpy(dicom_path, target_size=(256, 256)):\n    dicom_data = pydicom.dcmread(dicom_path)\n    image_array = dicom_data.pixel_array\n    img = Image.fromarray(image_array)\n    if img.mode not in ['RGB', 'L', 'RGBA']:\n        img = img.convert('L')\n    img_resized = img.resize(target_size, Image.LANCZOS)\n    img_np = np.array(img_resized)\n    return img_np\n\ndef save_image_as_npy(image_data, target_path):\n    os.makedirs(os.path.dirname(target_path), exist_ok=True)\n    np.save(target_path, image_data)\n\ndef process_directory(source_dir, target_dir, df, target_size=(256, 256)):\n    for study_dir in os.listdir(source_dir):\n        study_dir = np.int64(study_dir)\n        if study_dir in df['study_id'].values:\n            study_path = os.path.join(source_dir, str(study_dir))\n            for series_dir in os.listdir(study_path):\n                series_dir = np.int64(series_dir)\n                if series_dir in df['series_id'].values:\n                    series_path = os.path.join(study_path, str(series_dir))\n                    for file in os.listdir(series_path):\n                        if file.endswith(\".dcm\"):\n                            dicom_path = os.path.join(series_path, file)\n                            relative_path = os.path.relpath(dicom_path, source_dir)\n                            npy_file_name = os.path.splitext(relative_path)[0] + '.npy'\n                            target_path = os.path.join(target_dir, npy_file_name)\n                            img_np = process_dicom_to_numpy(dicom_path, target_size)\n                            save_image_as_npy(img_np, target_path)","metadata":{"execution":{"iopub.status.busy":"2024-10-13T19:51:33.085069Z","iopub.execute_input":"2024-10-13T19:51:33.085506Z","iopub.status.idle":"2024-10-13T19:51:33.097666Z","shell.execute_reply.started":"2024-10-13T19:51:33.085459Z","shell.execute_reply":"2024-10-13T19:51:33.096347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Histogram Equalization**","metadata":{}},{"cell_type":"markdown","source":"## Code for attention embeddings","metadata":{}},{"cell_type":"code","source":"def apply_histogram_equalization(image_data):\n    equalized_image = cv2.equalizeHist(image_data)\n    return equalized_image\n\ndef save_image_as_npy_hist(image_data, target_path):\n    os.makedirs(os.path.dirname(target_path), exist_ok=True)\n    np.save(target_path, image_data)\n    \ndef process_directory_hist(source_dir, target_dir,df):\n    for root, dirs, files in os.walk(source_dir):\n        for file in files:\n            if file.endswith(\".npy\"):\n                npy_path = os.path.join(root, file)\n                img_np = np.load(npy_path)\n                if img_np.shape == (256, 256):\n                    img_equalized = apply_histogram_equalization(img_np)\n                    relative_path = os.path.relpath(npy_path, source_dir)\n                    target_path = os.path.join(target_dir, relative_path)\n                    save_image_as_npy_hist(img_equalized, target_path)\n                else:\n                    print(f\"Image {npy_path} is not 256x256. Skipping.\")","metadata":{"execution":{"iopub.status.busy":"2024-10-13T19:51:33.099060Z","iopub.execute_input":"2024-10-13T19:51:33.099475Z","iopub.status.idle":"2024-10-13T19:51:33.114500Z","shell.execute_reply.started":"2024-10-13T19:51:33.099396Z","shell.execute_reply":"2024-10-13T19:51:33.113629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MRIEmbeddingModel(torch.nn.Module):\n    def __init__(self, base_model, embedding_dim):\n        super(MRIEmbeddingModel, self).__init__()\n        self.base_model = base_model\n        self.attention_layer = torch.nn.Linear(embedding_dim, 1) \n        self.embedding_dim = embedding_dim\n\n    def forward(self, x):\n        attention_weights = self.attention_layer(x)\n        final_embedding = torch.sum(x * attention_weights, dim=1)\n        return final_embedding, attention_weights","metadata":{"execution":{"iopub.status.busy":"2024-10-13T19:51:33.116941Z","iopub.execute_input":"2024-10-13T19:51:33.117850Z","iopub.status.idle":"2024-10-13T19:51:33.128548Z","shell.execute_reply.started":"2024-10-13T19:51:33.117804Z","shell.execute_reply":"2024-10-13T19:51:33.127504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport torch\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom torchvision import transforms\n\ndef attention_embeddings(model, df, img_path, result_path_csv, result_path_pth):\n    model.fc = torch.nn.Linear(model.fc.in_features, 512)\n    embedding_model = MRIEmbeddingModel(model, embedding_dim=512)\n    model = model.to('cuda')\n    embedding_model = embedding_model.to('cuda')\n\n    results = []\n    for index, row in tqdm(df.iterrows()):\n        patient_id = str(row['study_id'])\n        series_id = str(row['series_id'])\n\n        series_path = os.path.join(img_path, patient_id, series_id)\n        embeddings = []\n        if os.path.exists(series_path):\n            for slice_file in os.listdir(series_path):\n                if slice_file.endswith('.npy'):\n                    slice_path = os.path.join(series_path, slice_file)\n                    slice_data = np.load(slice_path)\n\n                    if slice_data.ndim == 2:\n                        slice_data = np.stack([slice_data] * 3, axis=0)\n                    elif slice_data.ndim == 3 and slice_data.shape[0] == 1:\n                        slice_data = np.repeat(slice_data, 3, axis=0)\n\n                    input_tensor = torch.from_numpy(slice_data).float().to('cuda')\n                    input_tensor = transforms.Resize((224, 224))(input_tensor)\n                    input_tensor = (input_tensor - torch.tensor([0.485, 0.456, 0.406]).view(3, 1, 1).to('cuda')) / \\\n                                   torch.tensor([0.229, 0.224, 0.225]).view(3, 1, 1).to('cuda')\n                    input_tensor = input_tensor.unsqueeze(0)\n                    \n                    with torch.no_grad():\n                        embedding = model(input_tensor)\n                        embeddings.append(embedding)\n\n            if embeddings:\n                slice_embeddings = torch.stack(embeddings, dim=1).to('cuda')\n                with torch.no_grad():\n                    final_embedding, attention_weights = embedding_model(slice_embeddings)\n                    final_embedding = final_embedding.squeeze().cpu()\n            else:\n                final_embedding = torch.zeros(512)\n        else:\n            final_embedding = torch.zeros(512)\n\n        embedding_dict = {f'{i}': final_embedding[i].item() for i in range(final_embedding.shape[0])}\n        embedding_dict.update({'study_id': patient_id, 'series_id': series_id})\n        results.append(embedding_dict)\n\n    results_df = pd.DataFrame(results)\n    results_df.to_csv(result_path_csv, index=False)\n    \n    torch.save(embedding_model.state_dict(), result_path_pth)\n    print(f\"Embeddings with attention completed and saved to {result_path_csv}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-13T19:51:33.129818Z","iopub.execute_input":"2024-10-13T19:51:33.130105Z","iopub.status.idle":"2024-10-13T19:51:33.147075Z","shell.execute_reply.started":"2024-10-13T19:51:33.130074Z","shell.execute_reply":"2024-10-13T19:51:33.146112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fill_missing_conditions(df):\n    required_conditions = [\"Axial T2\", \"Sagittal T1\", \"Sagittal T2/STIR\"]\n    new_rows = []\n    grouped = df.groupby('study_id')\n\n    for study_id, group in grouped:\n        existing_conditions = set(group['series_description'])\n        missing_conditions = set(required_conditions) - existing_conditions\n\n        for condition in missing_conditions:\n            new_rows.append({'study_id': study_id, 'series_id': 0, 'series_description': condition})\n\n    new_df = pd.DataFrame(new_rows)\n    df_filled = pd.concat([df, new_df], ignore_index=True)\n    df_filled = df_filled.sort_values(['study_id', 'series_description']).reset_index(drop=True)\n\n    return df_filled","metadata":{"execution":{"iopub.status.busy":"2024-10-13T19:51:33.148276Z","iopub.execute_input":"2024-10-13T19:51:33.148746Z","iopub.status.idle":"2024-10-13T19:51:33.159344Z","shell.execute_reply.started":"2024-10-13T19:51:33.148705Z","shell.execute_reply":"2024-10-13T19:51:33.158421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_embeddings(csv_path,npy_dir,hist_dir):\n    df = pd.read_csv(csv_path)\n    df = fill_missing_conditions(df)\n    #attention_embeddings(df[df['series_description'] == 'Axial T2'], npy_dir, 'AT2_attention_embeddings_gsl.csv', 'AT2_attention_model_gsl.pth')\n    attention_embeddings(resnet50_AT2, df[df['series_description'] == 'Axial T2'], hist_dir, 'AT2_attention_embeddings_hist.csv', 'AT2_attention_model_hist.pth')\n    attention_embeddings(resnet50_ST1, df[df['series_description'] == 'Sagittal T1'], npy_dir, 'ST1_attention_embeddings_gsl.csv', 'ST1_attention_model_gsl.pth')\n    #attention_embeddings(df[df['series_description'] == 'Sagittal T1'], hist_dir, 'ST1_attention_embeddings_hist.csv', 'ST1_attention_model_hist.pth')\n    #attention_embeddings(df[df['series_description'] == 'Sagittal T2/STIR'], npy_dir, 'ST2_attention_embeddings_gsl.csv', 'ST2_attention_model_gsl.pth')\n    attention_embeddings(resnet50_ST2,df[df['series_description'] == 'Sagittal T2/STIR'], hist_dir, 'ST2_attention_embeddings_hist.csv', 'ST2_attention_model_hist.pth')\n    ","metadata":{"execution":{"iopub.status.busy":"2024-10-13T19:51:33.160544Z","iopub.execute_input":"2024-10-13T19:51:33.160820Z","iopub.status.idle":"2024-10-13T19:51:33.170260Z","shell.execute_reply.started":"2024-10-13T19:51:33.160789Z","shell.execute_reply":"2024-10-13T19:51:33.169422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Conditions list","metadata":{}},{"cell_type":"code","source":"AT2_conditions = ['left_neural_foraminal_narrowing_l1_l2',\n       'left_neural_foraminal_narrowing_l2_l3',\n       'left_neural_foraminal_narrowing_l3_l4',\n       'left_neural_foraminal_narrowing_l4_l5',\n       'left_neural_foraminal_narrowing_l5_s1',\n       'right_neural_foraminal_narrowing_l1_l2',\n       'right_neural_foraminal_narrowing_l2_l3',\n       'right_neural_foraminal_narrowing_l3_l4',\n       'right_neural_foraminal_narrowing_l4_l5',\n       'right_neural_foraminal_narrowing_l5_s1']\n\nST1_conditions = ['left_subarticular_stenosis_l1_l2', 'left_subarticular_stenosis_l2_l3',\n       'left_subarticular_stenosis_l3_l4', 'left_subarticular_stenosis_l4_l5',\n       'left_subarticular_stenosis_l5_s1', 'right_subarticular_stenosis_l1_l2',\n       'right_subarticular_stenosis_l2_l3',\n       'right_subarticular_stenosis_l3_l4',\n       'right_subarticular_stenosis_l4_l5',\n       'right_subarticular_stenosis_l5_s1']\n\nST2_conditions = ['spinal_canal_stenosis_l1_l2', 'spinal_canal_stenosis_l2_l3',\n       'spinal_canal_stenosis_l3_l4', 'spinal_canal_stenosis_l4_l5',\n       'spinal_canal_stenosis_l5_s1']\n\nreal_arrangement = ['left_neural_foraminal_narrowing_l1_l2',\n 'left_neural_foraminal_narrowing_l2_l3',\n 'left_neural_foraminal_narrowing_l3_l4',\n 'left_neural_foraminal_narrowing_l4_l5',\n 'left_neural_foraminal_narrowing_l5_s1',\n 'left_subarticular_stenosis_l1_l2',\n 'left_subarticular_stenosis_l2_l3',\n 'left_subarticular_stenosis_l3_l4',\n 'left_subarticular_stenosis_l4_l5',\n 'left_subarticular_stenosis_l5_s1',\n 'right_neural_foraminal_narrowing_l1_l2',\n 'right_neural_foraminal_narrowing_l2_l3',\n 'right_neural_foraminal_narrowing_l3_l4',\n 'right_neural_foraminal_narrowing_l4_l5',\n 'right_neural_foraminal_narrowing_l5_s1',\n 'right_subarticular_stenosis_l1_l2',\n 'right_subarticular_stenosis_l2_l3',\n 'right_subarticular_stenosis_l3_l4',\n 'right_subarticular_stenosis_l4_l5',\n 'right_subarticular_stenosis_l5_s1',\n 'spinal_canal_stenosis_l1_l2',\n 'spinal_canal_stenosis_l2_l3',\n 'spinal_canal_stenosis_l3_l4',\n 'spinal_canal_stenosis_l4_l5',\n 'spinal_canal_stenosis_l5_s1']","metadata":{"execution":{"iopub.status.busy":"2024-10-13T19:51:33.203905Z","iopub.execute_input":"2024-10-13T19:51:33.204194Z","iopub.status.idle":"2024-10-13T19:51:33.211434Z","shell.execute_reply.started":"2024-10-13T19:51:33.204163Z","shell.execute_reply":"2024-10-13T19:51:33.210487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_embeddings(path):\n    df = pd.read_csv(path) \n    return df\nimport numpy as np\nimport pandas as pd\n\ndef create_prediction_dataframe(model, df, conditions):\n    embeddings = df[df.columns[:512]]  \n    study_ids = df[df.columns[512]]  \n    series_ids = df[df.columns[513]]  \n\n    predictions = model.predict(embeddings)\n    all_rows = []\n    num_conditions = len(conditions)\n    \n    if predictions.shape[1] != num_conditions * 3:\n        raise ValueError(\"The number of predicted values doesn't match the number of expected conditions.\")\n    \n    for i, (study_id, series_id) in enumerate(zip(study_ids, series_ids)):\n        study_predictions = predictions[i]\n        condition_predictions = np.split(study_predictions, num_conditions)\n        \n        for condition, pred in zip(conditions, condition_predictions):\n            row_id = f\"{study_id}_{condition}\"  \n            all_rows.append([row_id, pred[0], pred[1], pred[2]])\n    \n    final_df = pd.DataFrame(all_rows, columns=['row_id', 'normal_mild', 'moderate', 'severe'])\n    \n    return final_df\n","metadata":{"execution":{"iopub.status.busy":"2024-10-13T19:51:33.464938Z","iopub.execute_input":"2024-10-13T19:51:33.465332Z","iopub.status.idle":"2024-10-13T19:51:33.474379Z","shell.execute_reply.started":"2024-10-13T19:51:33.465270Z","shell.execute_reply":"2024-10-13T19:51:33.473304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Full pipeline","metadata":{}},{"cell_type":"code","source":"def full_pipeline(dcm_dir,description_data_path):\n    #load csv description file\n    df = pd.read_csv(description_data_path)\n    \n    npy_dir = '/kaggle/working/grey_scale_test'\n    process_directory(dcm_dir, npy_dir,df)\n    \n    hist_dir = '/kaggle/working/hist_norm_test'\n    process_directory_hist(npy_dir,hist_dir,df)\n    \n    csv_path = description_data_path\n    \n    generate_embeddings(csv_path,npy_dir,hist_dir)\n\n    AT2 = load_embeddings('/kaggle/working/AT2_attention_embeddings_hist.csv')\n    ST1 = load_embeddings('/kaggle/working/ST1_attention_embeddings_gsl.csv')\n    ST2 = load_embeddings('/kaggle/working/ST2_attention_embeddings_hist.csv')\n\n    model1 = load_model('/kaggle/input/models/AT2 - HIST - Attention Network_best_model.h5',compile=False)\n    model2 = load_model('/kaggle/input/models/ST1 - GSL - Attention Network_best_model.h5',compile=False)\n    model3 = load_model('/kaggle/input/models/ST2 - HIST - Attention Network_best_model.h5',compile=False)\n\n    AT2_predictions = create_prediction_dataframe(model1,AT2,AT2_conditions)\n    ST1_predictions = create_prediction_dataframe(model2,ST1,ST1_conditions)\n    ST2_predictions = create_prediction_dataframe(model3,ST2,ST2_conditions)\n    \n    submission = pd.concat([AT2_predictions, ST1_predictions, ST2_predictions], axis=0, ignore_index=True)\n    df = submission\n    df['study_id'] = df['row_id'].apply(lambda x: x.split('_')[0])\n\n    df['condition'] = df['row_id'].apply(lambda x: '_'.join(x.split('_')[1:]))\n    \n    df['row_id'] = df['study_id'] + '_' + df['condition']\n    df = df.set_index('row_id').reindex(\n        [f\"{sid}_{cond}\" for sid in df['study_id'].unique() for cond in real_arrangement]\n    ).reset_index()\n    \n    columns_to_normalize = ['normal_mild', 'moderate', 'severe']\n\n    df[columns_to_normalize] = df[columns_to_normalize].div(df[columns_to_normalize].sum(axis=1), axis=0)\n    \n    return df.drop(columns = ['condition','study_id'])\n        \n    ","metadata":{"execution":{"iopub.status.busy":"2024-10-13T19:51:33.842530Z","iopub.execute_input":"2024-10-13T19:51:33.843215Z","iopub.status.idle":"2024-10-13T19:51:33.853919Z","shell.execute_reply.started":"2024-10-13T19:51:33.843174Z","shell.execute_reply":"2024-10-13T19:51:33.852947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def full_pipeline(dcm_dir,description_data_path):\n    #load csv description file\n    df = pd.read_csv(description_data_path)\n    \n    npy_dir = '/kaggle/working/grey_scale_test'\n    process_directory(dcm_dir, npy_dir,df)\n    \n    hist_dir = '/kaggle/working/hist_norm_test'\n    process_directory_hist(npy_dir,hist_dir,df)\n    \n    csv_path = description_data_path\n    \n    generate_embeddings(csv_path,npy_dir,hist_dir)\n\n    AT2 = load_embeddings('/kaggle/working/AT2_attention_embeddings_hist.csv')\n    ST1 = load_embeddings('/kaggle/working/ST1_attention_embeddings_gsl.csv')\n    ST2 = load_embeddings('/kaggle/working/ST2_attention_embeddings_hist.csv')\n\n    model1 = load_model('/kaggle/input/models/AT2 - HIST - Attention Network_best_model.h5',compile=False)\n    model2 = load_model('/kaggle/input/models/ST1 - GSL - Attention Network_best_model.h5',compile=False)\n    model3 = load_model('/kaggle/input/models/ST2 - HIST - Attention Network_best_model.h5',compile=False)\n\n    AT2_predictions = create_prediction_dataframe(model1,AT2,AT2_conditions)\n    ST1_predictions = create_prediction_dataframe(model2,ST1,ST1_conditions)\n    ST2_predictions = create_prediction_dataframe(model3,ST2,ST2_conditions)\n    \n    submission = pd.concat([AT2_predictions, ST1_predictions, ST2_predictions], axis=0, ignore_index=True)\n    df = submission.copy()\n    df['study_id'] = df['row_id'].apply(lambda x: x.split('_')[0])\n    study_ids = df['study_id'].unique()\n    row_ids = [f\"{s}_{condition}\" for s in study_ids for condition in real_arrangement]\n    data = {\n        'row_id': row_ids,\n        'normal_mild': [],\n        'moderate': [],\n        'severe': []\n    }\n\n    for value in data['row_id']:\n        matching_row = df[df['row_id'] == value]\n        if not matching_row.empty:\n            data['normal_mild'].append(matching_row.iloc[0]['normal_mild'])\n            data['moderate'].append(matching_row.iloc[0]['moderate'])\n            data['severe'].append(matching_row.iloc[0]['severe'])\n        else:\n            data['normal_mild'].append(None)\n            data['moderate'].append(None)\n            data['severe'].append(None)\n\n    # Create new DataFrame\n    new_df = pd.DataFrame(data)\n                \n    \n    return new_df\n        \n    ","metadata":{"execution":{"iopub.status.busy":"2024-10-13T19:51:41.694982Z","iopub.execute_input":"2024-10-13T19:51:41.695380Z","iopub.status.idle":"2024-10-13T19:51:41.707518Z","shell.execute_reply.started":"2024-10-13T19:51:41.695341Z","shell.execute_reply":"2024-10-13T19:51:41.706530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = full_pipeline(root_directory,description_csv)","metadata":{"execution":{"iopub.status.busy":"2024-10-13T19:51:42.217627Z","iopub.execute_input":"2024-10-13T19:51:42.218321Z","iopub.status.idle":"2024-10-13T20:03:18.802751Z","shell.execute_reply.started":"2024-10-13T19:51:42.218268Z","shell.execute_reply":"2024-10-13T20:03:18.801691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### saving submission file","metadata":{}},{"cell_type":"code","source":"#!rm -r /kaggle/working/*","metadata":{"execution":{"iopub.status.busy":"2024-10-13T20:03:18.804536Z","iopub.execute_input":"2024-10-13T20:03:18.804860Z","iopub.status.idle":"2024-10-13T20:03:18.809464Z","shell.execute_reply.started":"2024-10-13T20:03:18.804826Z","shell.execute_reply":"2024-10-13T20:03:18.808370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-13T20:03:18.810582Z","iopub.execute_input":"2024-10-13T20:03:18.810850Z","iopub.status.idle":"2024-10-13T20:03:18.838450Z","shell.execute_reply.started":"2024-10-13T20:03:18.810820Z","shell.execute_reply":"2024-10-13T20:03:18.837690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### METRICS","metadata":{}},{"cell_type":"code","source":"import os\nimport sys\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport random\nimport pandas as pd\nfrom glob import glob\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold\nimport polars as pl\npl.Config.set_tbl_rows(40)\npl.Config.set_fmt_str_lengths(n=40)\nimport seaborn as sns\nimport pydicom\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms","metadata":{"execution":{"iopub.status.busy":"2024-10-13T20:03:18.840315Z","iopub.execute_input":"2024-10-13T20:03:18.840653Z","iopub.status.idle":"2024-10-13T20:03:18.847458Z","shell.execute_reply.started":"2024-10-13T20:03:18.840620Z","shell.execute_reply":"2024-10-13T20:03:18.846517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pandas.api.types\nimport sklearn.metrics\n\n\nclass ParticipantVisibleError(Exception):\n    pass\n\n\ndef get_condition(full_location: str) -> str:\n    for injury_condition in ['spinal', 'foraminal', 'subarticular']:\n        if injury_condition in full_location:\n            return injury_condition\n    raise ValueError(f'condition not found in {full_location}')\n\n\ndef log_loss_safe(y_true, y_pred, sample_weight=None):\n    \"\"\"Safely computes log loss even if only one class is present.\"\"\"\n\n    \n    unique_classes = np.unique(y_true)\n    \n    # If only one unique class, explicitly provide both classes\n    if len(unique_classes) == 1:\n        prin(\"binary\")\n        labels = [0, 1]  # Assume binary labels\n    else:\n        labels = None  # Let log_loss infer the labels\n\n    return sklearn.metrics.log_loss(\n        y_true=y_true,\n        y_pred=y_pred,\n        sample_weight=sample_weight,\n        labels=labels\n    )\n\n\ndef score(\n        solution: pd.DataFrame,\n        submission: pd.DataFrame,\n        row_id_column_name: str,\n        any_severe_scalar: float\n    ) -> float:\n    target_levels = ['normal_mild', 'moderate', 'severe']\n\n    if not pandas.api.types.is_numeric_dtype(submission[target_levels].values):\n        raise ParticipantVisibleError('All submission values must be numeric')\n\n    if not np.isfinite(submission[target_levels].values).all():\n        raise ParticipantVisibleError('All submission values must be finite')\n\n    if solution[target_levels].min().min() < 0:\n        raise ParticipantVisibleError('All labels must be at least zero')\n    if submission[target_levels].min().min() < 0:\n        raise ParticipantVisibleError('All predictions must be at least zero')\n\n    solution['study_id'] = solution['row_id'].apply(lambda x: x.split('_')[0])\n    solution['location'] = solution['row_id'].apply(lambda x: '_'.join(x.split('_')[1:]))\n    solution['condition'] = solution['row_id'].apply(get_condition)\n\n    del solution[row_id_column_name]\n    del submission[row_id_column_name]\n    assert sorted(submission.columns) == sorted(target_levels)\n\n    submission['study_id'] = solution['study_id']\n    submission['location'] = solution['location']\n    submission['condition'] = solution['condition']\n\n    condition_losses = []\n    condition_weights = []\n    for condition in ['spinal', 'foraminal', 'subarticular']:\n        condition_indices = solution.loc[solution['condition'] == condition].index.values\n        condition_loss = log_loss_safe(\n            y_true=solution.loc[condition_indices, target_levels].values,\n            y_pred=submission.loc[condition_indices, target_levels].values,\n            sample_weight=solution.loc[condition_indices, 'sample_weight'].values\n        )\n        condition_losses.append(condition_loss)\n        condition_weights.append(1)\n\n    any_severe_spinal_labels = solution.loc[solution['condition'] == 'spinal'].groupby('study_id')['severe'].max()\n    any_severe_spinal_weights = solution.loc[solution['condition'] == 'spinal'].groupby('study_id')['sample_weight'].max()\n    any_severe_spinal_predictions = submission.loc[submission['condition'] == 'spinal'].groupby('study_id')['severe'].max()\n\n    any_severe_spinal_loss = log_loss_safe(\n        y_true=any_severe_spinal_labels.values,\n        y_pred=any_severe_spinal_predictions.values,\n        sample_weight=any_severe_spinal_weights.values\n    )\n\n    condition_losses.append(any_severe_spinal_loss)\n    condition_weights.append(any_severe_scalar)\n\n    return np.average(condition_losses, weights=condition_weights)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-13T20:03:18.848731Z","iopub.execute_input":"2024-10-13T20:03:18.849002Z","iopub.status.idle":"2024-10-13T20:03:18.867164Z","shell.execute_reply.started":"2024-10-13T20:03:18.848970Z","shell.execute_reply":"2024-10-13T20:03:18.866235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_solution(descriotion_csv):\n    \"\"\"Assuming that the data is extracted from solution_csv\"\"\"\n    unseen_processed = pd.read_csv(description_csv)\n    unseen_trained_csv = pd.read_csv(solution_csv)\n    train_main = unseen_trained_csv.merge(unseen_processed[['study_id']], on=['study_id'], how='inner')\n    train_main.drop_duplicates(inplace=True)\n    solution = train_main.melt(id_vars=[\"study_id\"], var_name=\"full_label\", value_name=\"severity\")\n    solution[\"row_id\"] = solution.apply(lambda row: str(row.study_id) + \"_\" + row.full_label, axis=1)\n    solution.severity = solution.severity.fillna(\"Normal/Mild\")\n    solution.loc[solution.severity == \"Normal/Mild\", \"normal_mild\"] = 1\n    solution.loc[solution.severity == \"Moderate\", \"moderate\"] = 1\n    solution.loc[solution.severity == \"Severe\", \"severe\"] = 1\n\n    solution.loc[solution.severity == \"Normal/Mild\", \"sample_weight\"] = 1\n    solution.loc[solution.severity == \"Moderate\", \"sample_weight\"] = 2\n    solution.loc[solution.severity == \"Severe\", \"sample_weight\"] = 4\n\n    solution = solution[[\"study_id\", \"row_id\", \"normal_mild\", \"moderate\", \"severe\", \"sample_weight\"]]\n    solution = solution.fillna(0)\n    solution = solution.sort_values(by=[\"row_id\"])\n    return solution\n    ","metadata":{"execution":{"iopub.status.busy":"2024-10-13T20:49:45.355818Z","iopub.execute_input":"2024-10-13T20:49:45.356710Z","iopub.status.idle":"2024-10-13T20:49:45.365972Z","shell.execute_reply.started":"2024-10-13T20:49:45.356667Z","shell.execute_reply":"2024-10-13T20:49:45.365118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"solution = generate_solution(description_csv)","metadata":{"execution":{"iopub.status.busy":"2024-10-13T20:50:13.821832Z","iopub.execute_input":"2024-10-13T20:50:13.822377Z","iopub.status.idle":"2024-10-13T20:50:13.981702Z","shell.execute_reply.started":"2024-10-13T20:50:13.822343Z","shell.execute_reply":"2024-10-13T20:50:13.980756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s = score(solution.copy(), submission.copy(), \"row_id\", 1)\nf\"Score {s}\"","metadata":{"execution":{"iopub.status.busy":"2024-10-13T20:03:19.009517Z","iopub.execute_input":"2024-10-13T20:03:19.009811Z","iopub.status.idle":"2024-10-13T20:03:19.047821Z","shell.execute_reply.started":"2024-10-13T20:03:19.009779Z","shell.execute_reply":"2024-10-13T20:03:19.047057Z"},"trusted":true},"execution_count":null,"outputs":[]}]}