{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":30747,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # This Python 3 environment comes with many helpful analytics libraries installed\n# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-07-13T07:34:05.838258Z","iopub.execute_input":"2024-07-13T07:34:05.838685Z","iopub.status.idle":"2024-07-13T07:34:05.843604Z","shell.execute_reply.started":"2024-07-13T07:34:05.838642Z","shell.execute_reply":"2024-07-13T07:34:05.842701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nbase_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification'\nos.listdir(base_path)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T07:34:05.845604Z","iopub.execute_input":"2024-07-13T07:34:05.845944Z","iopub.status.idle":"2024-07-13T07:34:05.861152Z","shell.execute_reply.started":"2024-07-13T07:34:05.845914Z","shell.execute_reply":"2024-07-13T07:34:05.860259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\ntrain_coor = pd.read_csv(os.path.join(base_path,'train_label_coordinates.csv'))\ntrain_desc = pd.read_csv(os.path.join(base_path,'train_series_descriptions.csv'))\ntrain_df = pd.read_csv(os.path.join(base_path,'train.csv'))\ntest_desc = pd.read_csv(os.path.join(base_path,'test_series_descriptions.csv'))\nsub         = pd.read_csv(os.path.join(base_path,'sample_submission.csv'))\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2024-07-13T07:34:05.862282Z","iopub.execute_input":"2024-07-13T07:34:05.862604Z","iopub.status.idle":"2024-07-13T07:34:05.981767Z","shell.execute_reply.started":"2024-07-13T07:34:05.862580Z","shell.execute_reply":"2024-07-13T07:34:05.980855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_desc","metadata":{"execution":{"iopub.status.busy":"2024-07-13T07:34:05.984337Z","iopub.execute_input":"2024-07-13T07:34:05.985008Z","iopub.status.idle":"2024-07-13T07:34:05.994749Z","shell.execute_reply.started":"2024-07-13T07:34:05.984979Z","shell.execute_reply":"2024-07-13T07:34:05.993773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_coor","metadata":{"execution":{"iopub.status.busy":"2024-07-13T07:34:05.996422Z","iopub.execute_input":"2024-07-13T07:34:05.996884Z","iopub.status.idle":"2024-07-13T07:34:06.011916Z","shell.execute_reply.started":"2024-07-13T07:34:05.996825Z","shell.execute_reply":"2024-07-13T07:34:06.011007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\ntrain_data = pd.merge(train_desc, train_coor, on=[\"study_id\", \"series_id\"], how=\"inner\")\ntrain_data","metadata":{"execution":{"iopub.status.busy":"2024-07-13T07:34:06.013325Z","iopub.execute_input":"2024-07-13T07:34:06.013619Z","iopub.status.idle":"2024-07-13T07:34:06.040609Z","shell.execute_reply.started":"2024-07-13T07:34:06.013594Z","shell.execute_reply":"2024-07-13T07:34:06.039642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"expand_df = []\n\ndef reshape(row):\n    for column, value in row.items():\n        if column != \"study_id\":  # 确保变量名正确\n            parts = column.split(\"_\")\n            level = parts[-2].capitalize() + '/' + parts[-1].capitalize()\n            condition = \" \".join(word.capitalize() for word in parts[:-2])\n            expand_df.append({\n                \"study_id\": row[\"study_id\"],\n                \"level\": level,  # 确保 level 是一个字符串\n                \"condition\": condition,\n                \"severity\": value\n            })\n\n# 假设 train_df 是你要处理的 DataFrame\nfor _, row in train_df.iterrows():\n    reshape(row)\n\nexpand_df = pd.DataFrame(expand_df)  # 创建 DataFrame 并重置索引\nexpand_df","metadata":{"execution":{"iopub.status.busy":"2024-07-13T07:34:06.041666Z","iopub.execute_input":"2024-07-13T07:34:06.041960Z","iopub.status.idle":"2024-07-13T07:34:06.649051Z","shell.execute_reply.started":"2024-07-13T07:34:06.041936Z","shell.execute_reply":"2024-07-13T07:34:06.648089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_merged_df = pd.merge(train_data, expand_df, on=[\"study_id\", \"condition\", \"level\"])\ntrain_merged_df","metadata":{"execution":{"iopub.status.busy":"2024-07-13T07:34:06.650311Z","iopub.execute_input":"2024-07-13T07:34:06.650686Z","iopub.status.idle":"2024-07-13T07:34:06.717099Z","shell.execute_reply.started":"2024-07-13T07:34:06.650649Z","shell.execute_reply":"2024-07-13T07:34:06.716108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_merged_df[\"row_id\"] = (\n    train_merged_df[\"study_id\"].astype(str) + \"_\" +\n    train_merged_df[\"condition\"].str.lower().str.replace(\" \", \"_\") + \"_\" + \n    train_merged_df[\"level\"].str.lower().str.replace(\"/\", \"_\")\n)\ntrain_merged_df['image_path'] = (\n    f'{base_path}/train_images/' + \n    train_merged_df['study_id'].astype(str) + '/' +\n    train_merged_df['series_id'].astype(str) + '/' +\n    train_merged_df['instance_number'].astype(str) + '.dcm'\n)\ntrain_merged_df","metadata":{"execution":{"iopub.status.busy":"2024-07-13T07:34:06.718248Z","iopub.execute_input":"2024-07-13T07:34:06.718555Z","iopub.status.idle":"2024-07-13T07:34:06.946023Z","shell.execute_reply.started":"2024-07-13T07:34:06.718531Z","shell.execute_reply":"2024-07-13T07:34:06.945132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_merged_df[\"severity\"] = train_merged_df[\"severity\"].map({'Normal/Mild': 'normal_mild', 'Moderate': 'moderate', 'Severe': 'severe'})\ntrain_merged_df","metadata":{"execution":{"iopub.status.busy":"2024-07-13T07:34:06.950474Z","iopub.execute_input":"2024-07-13T07:34:06.950824Z","iopub.status.idle":"2024-07-13T07:34:06.975446Z","shell.execute_reply.started":"2024-07-13T07:34:06.950802Z","shell.execute_reply":"2024-07-13T07:34:06.974506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install pydicom\nimport pydicom\ndef load_dcm(path):\n    dicom = pydicom.read_file(path)\n    img = dicom.pixel_array\n    img = img - np.min(img)\n    if np.max(img) != 0:\n        img = img / np.max(img)\n    img  = (img * 255).astype(np.uint8)\n    return img","metadata":{"execution":{"iopub.status.busy":"2024-07-13T07:34:06.976628Z","iopub.execute_input":"2024-07-13T07:34:06.976967Z","iopub.status.idle":"2024-07-13T07:34:06.984368Z","shell.execute_reply.started":"2024-07-13T07:34:06.976940Z","shell.execute_reply":"2024-07-13T07:34:06.983497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset, DataLoader\nimport tqdm as tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\n\nclass RSNA_Dataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df =  df\n        self.transform = transform\n        self.label_map = {'normal_mild': 0, 'moderate': 1, 'severe': 2}\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, index):\n        image_path = self.df[\"image_path\"][index]\n        image = load_dcm(image_path)\n        label = self.df[\"severity\"][index]\n        label = self.label_map[label]\n        # if math.isnan(label):\n        #     # 可以选择跳过这个样本或者赋一个默认值\n        #     label = None  # 或者其他默认值\n\n        if self.transform:\n            image = self.transform(image)\n\n        return image, label","metadata":{"execution":{"iopub.status.busy":"2024-07-13T07:34:06.985594Z","iopub.execute_input":"2024-07-13T07:34:06.985903Z","iopub.status.idle":"2024-07-13T07:34:06.995632Z","shell.execute_reply.started":"2024-07-13T07:34:06.985872Z","shell.execute_reply":"2024-07-13T07:34:06.994856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_map = {'normal_mild': 0, 'moderate': 1, 'severe': 2}","metadata":{"execution":{"iopub.status.busy":"2024-07-13T07:34:06.996698Z","iopub.execute_input":"2024-07-13T07:34:06.997086Z","iopub.status.idle":"2024-07-13T07:34:07.009099Z","shell.execute_reply.started":"2024-07-13T07:34:06.997052Z","shell.execute_reply":"2024-07-13T07:34:07.008239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision.transforms as T\ntrain_merged_df = train_merged_df.dropna()\ntransform = T.Compose([\n    T.Lambda(lambda x: (x * 255).astype(np.uint8)),  # Convert back to uint8 for PIL\n    T.ToPILImage(),\n    T.Resize((224, 224)),\n    T.Grayscale(num_output_channels=3),\n    T.ToTensor(),\n])\n\ndef create_dataset_dataloader(df, series_description, transform, batch_size=16):\n    filtered_df = df[df['series_description'] == series_description]\n    \n    train_df, val_df = train_test_split(filtered_df, test_size=0.2, random_state=42)\n    train_df = train_df.reset_index(drop=True)\n    val_df = val_df.reset_index(drop=True)\n\n    train_dataset = RSNA_Dataset(train_df, transform)\n    val_dataset = RSNA_Dataset(val_df, transform)\n\n    trainloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n    valloader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\n    \n    return trainloader, valloader, len(train_df), len(val_df)\n\ndataloaders = {}\nlengths = {}\n\ntrainloader_t1, valloader_t1, len_train_t1, len_val_t1 = create_dataset_dataloader(train_merged_df, 'Sagittal T1', transform)\ntrainloader_t2, valloader_t2, len_train_t2, len_val_t2 = create_dataset_dataloader(train_merged_df, 'Axial T2', transform)\ntrainloader_t2stir, valloader_t2stir, len_train_t2stir, len_val_t2stir = create_dataset_dataloader(train_merged_df, 'Sagittal T2/STIR', transform)\n\ndataloaders['Sagittal T1'] = (trainloader_t1, valloader_t1)\ndataloaders['Axial T2'] = (trainloader_t2, valloader_t2)\ndataloaders['Sagittal T2/STIR'] = (trainloader_t2stir, valloader_t2stir)\n\nlengths['Sagittal T1'] = (len_train_t1, len_val_t1)\nlengths['Axial T2'] = (len_train_t2, len_val_t2)\nlengths['Sagittal T2/STIR'] = (len_train_t2stir, len_val_t2stir)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T07:34:07.010110Z","iopub.execute_input":"2024-07-13T07:34:07.010388Z","iopub.status.idle":"2024-07-13T07:34:07.111012Z","shell.execute_reply.started":"2024-07-13T07:34:07.010366Z","shell.execute_reply":"2024-07-13T07:34:07.110267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision.models as models\nfrom torchvision import transforms\nfrom torch.utils.data import DataLoader\nfrom sklearn.model_selection import train_test_split\nimport pandas as pd\nfrom tqdm import tqdm\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2024-07-13T07:34:07.112412Z","iopub.execute_input":"2024-07-13T07:34:07.112734Z","iopub.status.idle":"2024-07-13T07:34:07.118137Z","shell.execute_reply.started":"2024-07-13T07:34:07.112708Z","shell.execute_reply":"2024-07-13T07:34:07.117225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision.models as models\n\nclass CustomEfficientNetV2(nn.Module):\n    def __init__(self, num_classes=3, pretrained_weights=None):\n        super(CustomEfficientNetV2, self).__init__()\n        self.model = models.efficientnet_v2_s(weights=None)\n        if pretrained_weights:\n            self.model.load_state_dict(torch.load(pretrained_weights))\n        num_ftrs = self.model.classifier[-1].in_features\n        self.model.classifier[-1] = nn.Linear(num_ftrs, num_classes)\n\n    def forward(self, x):\n        return self.model(x)\n\n    def unfreeze_model(self):\n        # Unfreeze the last 20 layers, keeping BatchNorm layers frozen\n        for layer in list(self.model.features.children())[-20:]:\n            if not isinstance(layer, nn.BatchNorm2d):\n                for param in layer.parameters():\n                    param.requires_grad = True\n        \n        # Unfreeze the classifier\n        for param in self.model.classifier.parameters():\n            param.requires_grad = True\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n# Path to the locally uploaded weights file\nweights_path = '/kaggle/input/efficientnetv2/pytorch/efficientnetv2s/1/efficientnet_v2_s-dd5fe13b.pth'\n\n# Initialize models\nsagittal_t1_model = CustomEfficientNetV2(num_classes=3).to(device)\naxial_t2_model = CustomEfficientNetV2(num_classes=3).to(device)\nsagittal_t2stir_model = CustomEfficientNetV2(num_classes=3).to(device)\n\n# Optionally freeze initial layers\nfor param in sagittal_t1_model.model.features.parameters():\n    param.requires_grad = False\nfor param in axial_t2_model.model.features.parameters():\n    param.requires_grad = False\nfor param in sagittal_t2stir_model.model.features.parameters():\n    param.requires_grad = False\n\n# Unfreeze the final fully connected layer\nfor param in sagittal_t1_model.model.classifier.parameters():\n    param.requires_grad = True\nfor param in axial_t2_model.model.classifier.parameters():\n    param.requires_grad = True\nfor param in sagittal_t2stir_model.model.classifier.parameters():\n    param.requires_grad = True\n\n# Training parameters\ncriterion = nn.CrossEntropyLoss()\n\n# Initialize separate optimizers for each model\noptimizer_sagittal_t1 = torch.optim.Adam(sagittal_t1_model.model.classifier.parameters(), lr=0.001)\noptimizer_axial_t2 = torch.optim.Adam(axial_t2_model.model.classifier.parameters(), lr=0.001)\noptimizer_sagittal_t2stir = torch.optim.Adam(sagittal_t2stir_model.model.classifier.parameters(), lr=0.001)\n\n# Store the models and optimizers in dictionaries for easy access\nmodels = {\n    'Sagittal T1': sagittal_t1_model,\n    'Axial T2': axial_t2_model,\n    'Sagittal T2/STIR': sagittal_t2stir_model,\n}\noptimizers = {\n    'Sagittal T1': optimizer_sagittal_t1,\n    'Axial T2': optimizer_axial_t2,\n    'Sagittal T2/STIR': optimizer_sagittal_t2stir,\n}","metadata":{"execution":{"iopub.status.busy":"2024-07-13T07:34:07.119516Z","iopub.execute_input":"2024-07-13T07:34:07.119785Z","iopub.status.idle":"2024-07-13T07:34:08.589180Z","shell.execute_reply.started":"2024-07-13T07:34:07.119762Z","shell.execute_reply":"2024-07-13T07:34:08.588329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainable_params = sum(p.numel() for p in sagittal_t1_model.parameters() if p.requires_grad)\nprint(f\"Number of parameters: {trainable_params}\")","metadata":{"execution":{"iopub.status.busy":"2024-07-13T07:34:08.590403Z","iopub.execute_input":"2024-07-13T07:34:08.591105Z","iopub.status.idle":"2024-07-13T07:34:08.599497Z","shell.execute_reply.started":"2024-07-13T07:34:08.591071Z","shell.execute_reply":"2024-07-13T07:34:08.598370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.optim.lr_scheduler as lr_scheduler\nfrom copy import deepcopy\n\ndef train_model(model, trainloader, valloader, len_train, len_val, optimizer, num_epochs=10, patience=3):\n    # Learning rate scheduler\n    scheduler = lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.1)\n    \n    best_val_acc = 0.0\n    best_model_wts = deepcopy(model.state_dict())\n    counter = 0\n    \n    for epoch in range(num_epochs):\n        model.train()\n        train_loss = 0\n        correct_train = 0\n        \n        with tqdm(trainloader, unit=\"batch\") as tepoch:\n            for images, labels in tepoch:\n                images = images.to(device)\n                labels = labels.clone().detach().to(device)  # 假设 labels 是一个张量\n                optimizer.zero_grad()\n                outputs = model(images)\n                loss = criterion(outputs, labels)\n                loss.backward()\n                optimizer.step()\n                train_loss += loss.item()\n                \n                probabilities = torch.softmax(outputs, dim=1)\n                _, predicted = torch.max(probabilities, 1)\n                correct_train += (predicted == labels).sum().item()\n                \n                tepoch.set_postfix(epoch=epoch+1)\n        \n        scheduler.step()\n        \n        train_loss /= len(trainloader)\n        train_acc = 100 * correct_train / len_train\n        \n        model.eval()\n        val_loss, correct_val = 0, 0\n        with torch.no_grad():\n            with tqdm(valloader, unit=\"batch\") as vepoch:\n                for images, labels in vepoch:\n                    # images, labels = images.to(device), torch.tensor([label_map[label] for label in labels]).to(device)\n                    images = images.to(device)\n                    labels = labels.clone().detach().to(device)  # 假设 labels 是一个张量\n                    outputs = model(images)\n                    loss = criterion(outputs, labels)\n                    val_loss += loss.item()\n                    \n                    probabilities = torch.softmax(outputs, dim=1).squeeze(0)\n                    _, predicted = torch.max(probabilities, 1)\n                    correct_val += (predicted == labels).sum().item()\n                    \n                    vepoch.set_postfix(epoch=epoch+1)\n        \n        val_loss /= len(valloader)\n        val_acc = 100 * correct_val / len_val\n        \n        print(f\"Epoch {epoch+1}, Train Loss: {train_loss:.4f}, Train Acc: {train_acc:.2f}%, Val Loss: {val_loss:.4f}, Val Acc: {val_acc:.2f}%\")\n\n        # Save the best model and check for early stopping\n        if val_acc > best_val_acc:\n            best_val_acc = val_acc\n            best_model_wts = deepcopy(model.state_dict())\n            counter = 0\n#             torch.save(best_model_wts, f'best_model_{epoch+1}.pth')\n        else:\n            counter += 1\n        \n        # Early stopping\n        if counter >= patience:\n            print(f\"Early stopping triggered after {epoch+1} epochs\")\n            break\n    \n    # Load best model weights\n    model.load_state_dict(best_model_wts)\n    return model, best_val_acc","metadata":{"execution":{"iopub.status.busy":"2024-07-13T07:34:08.601251Z","iopub.execute_input":"2024-07-13T07:34:08.601636Z","iopub.status.idle":"2024-07-13T07:34:08.617411Z","shell.execute_reply.started":"2024-07-13T07:34:08.601603Z","shell.execute_reply":"2024-07-13T07:34:08.616541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for desc, model in models.items():\n    if desc == 'Sagittal T1':\n        trainloader, valloader, len_train, len_val = trainloader_t1, valloader_t1, len_train_t1, len_val_t1\n    elif desc == 'Axial T2':\n        trainloader, valloader, len_train, len_val = trainloader_t2, valloader_t2, len_train_t2, len_val_t2\n    elif desc == 'Sagittal T2/STIR':\n        trainloader, valloader, len_train, len_val = trainloader_t2stir, valloader_t2stir, len_train_t2stir, len_val_t2stir\n    \n    print(f\"Training model for {desc}\")\n    train_model(model, trainloader, valloader, len_train, len_val, optimizers[desc])","metadata":{"execution":{"iopub.status.busy":"2024-07-13T07:34:08.618813Z","iopub.execute_input":"2024-07-13T07:34:08.619287Z","iopub.status.idle":"2024-07-13T08:48:21.479037Z","shell.execute_reply.started":"2024-07-13T07:34:08.619263Z","shell.execute_reply":"2024-07-13T08:48:21.478013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_path= '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_images'\n\n# Function to get image paths for a series\ndef get_image_paths(row):\n    series_path = os.path.join(test_path, str(row['study_id']), str(row['series_id']))\n    if os.path.exists(series_path):\n        return [os.path.join(series_path, f) for f in os.listdir(series_path) if os.path.isfile(os.path.join(series_path, f))]\n    return []\n\n# Mapping of series_description to conditions\ncondition_mapping = {\n    'Sagittal T1': {'left': 'left_neural_foraminal_narrowing', 'right': 'right_neural_foraminal_narrowing'},\n    'Axial T2': {'left': 'left_subarticular_stenosis', 'right': 'right_subarticular_stenosis'},\n    'Sagittal T2/STIR': 'spinal_canal_stenosis'\n}\n\n# Create a list to store the expanded rows\nexpanded_rows = []\n\n# Expand the dataframe by adding new rows for each file path\nfor index, row in test_desc.iterrows():\n    image_paths = get_image_paths(row)\n    conditions = condition_mapping.get(row['series_description'], {})\n    if isinstance(conditions, str):  # Single condition\n        conditions = {'left': conditions, 'right': conditions}\n    for side, condition in conditions.items():\n        for image_path in image_paths:\n            expanded_rows.append({\n                'study_id': row['study_id'],\n                'series_id': row['series_id'],\n                'series_description': row['series_description'],\n                'image_path': image_path,\n                'condition': condition,\n                'row_id': f\"{row['study_id']}_{condition}\"\n            })\n\n# Create a new dataframe from the expanded rows\nexpanded_test_desc = pd.DataFrame(expanded_rows)\n\n# Display the resulting dataframe\nexpanded_test_desc.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T09:12:45.278332Z","iopub.execute_input":"2024-07-13T09:12:45.278767Z","iopub.status.idle":"2024-07-13T09:12:45.354916Z","shell.execute_reply.started":"2024-07-13T09:12:45.278737Z","shell.execute_reply":"2024-07-13T09:12:45.353987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"levels = ['l1_l2', 'l2_l3', 'l3_l4', 'l4_l5', 'l5_s1']\n\n# Function to update row_id with levels\ndef update_row_id(row, levels):\n    level = levels[row.name % len(levels)]\n    return f\"{row['study_id']}_{row['condition']}_{level}\"\n\n# Update row_id in expanded_test_desc to include levels\nexpanded_test_desc['row_id'] = expanded_test_desc.apply(lambda row: update_row_id(row, levels), axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T09:12:46.922018Z","iopub.execute_input":"2024-07-13T09:12:46.922730Z","iopub.status.idle":"2024-07-13T09:12:46.932380Z","shell.execute_reply.started":"2024-07-13T09:12:46.922699Z","shell.execute_reply":"2024-07-13T09:12:46.931441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define a custom test dataset class\nclass TestDataset(Dataset):\n    def __init__(self, dataframe, transform=None):\n        self.dataframe = dataframe\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.dataframe)\n\n    def __getitem__(self, index):\n        image_path = self.dataframe['image_path'][index]\n        image = load_dcm(image_path)  # Define this function to load your DICOM images\n        if self.transform:\n            image = self.transform(image)\n        return image\n\n# Define the transforms\ntransform = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.Resize((224, 224)),\n    transforms.Grayscale(num_output_channels=3),\n    transforms.ToTensor(),\n])\n\n# Create a test dataset and dataloader\ntest_dataset = TestDataset(expanded_test_desc, transform)\ntestloader = DataLoader(test_dataset, batch_size=1, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T09:12:47.535690Z","iopub.execute_input":"2024-07-13T09:12:47.536169Z","iopub.status.idle":"2024-07-13T09:12:47.546010Z","shell.execute_reply.started":"2024-07-13T09:12:47.536132Z","shell.execute_reply":"2024-07-13T09:12:47.544746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(series_description):\n    return models.get(series_description, None)\n\n# Function to make predictions on the test data\ndef predict_test_data(testloader, expanded_test_desc):\n    predictions = []\n    normal_mild_probs = []\n    moderate_probs = []\n    severe_probs = []\n    \n    for model in models.values():\n        model.eval()\n        \n    with torch.no_grad():\n        for idx, images in enumerate(tqdm(testloader)):\n            images = images.to(device)\n            series_description = expanded_test_desc.iloc[idx]['series_description']\n            model = get_model(series_description)\n            if model:\n                model.eval()  # Set the model to eval mode\n                outputs = model(images)\n                probs = torch.softmax(outputs, dim=1).squeeze(0)\n                normal_mild_probs.append(probs[0].item())\n                moderate_probs.append(probs[1].item())\n                severe_probs.append(probs[2].item())\n                predictions.append(probs)\n            else:\n                normal_mild_probs.append(None)\n                moderate_probs.append(None)\n                severe_probs.append(None)\n                predictions.append(None)\n    return normal_mild_probs, moderate_probs, severe_probs, predictions","metadata":{"execution":{"iopub.status.busy":"2024-07-13T09:12:48.019068Z","iopub.execute_input":"2024-07-13T09:12:48.019862Z","iopub.status.idle":"2024-07-13T09:12:48.031939Z","shell.execute_reply.started":"2024-07-13T09:12:48.019809Z","shell.execute_reply":"2024-07-13T09:12:48.030819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make predictions on the test data\nnormal_mild_probs, moderate_probs, severe_probs, test_predictions = predict_test_data(testloader, expanded_test_desc)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T09:12:49.682822Z","iopub.execute_input":"2024-07-13T09:12:49.683214Z","iopub.status.idle":"2024-07-13T09:12:59.117802Z","shell.execute_reply.started":"2024-07-13T09:12:49.683184Z","shell.execute_reply":"2024-07-13T09:12:59.116896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"expanded_test_desc['normal_mild'] = normal_mild_probs\nexpanded_test_desc['moderate'] = moderate_probs\nexpanded_test_desc['severe'] = severe_probs","metadata":{"execution":{"iopub.status.busy":"2024-07-13T09:12:59.119913Z","iopub.execute_input":"2024-07-13T09:12:59.120364Z","iopub.status.idle":"2024-07-13T09:12:59.126519Z","shell.execute_reply.started":"2024-07-13T09:12:59.120329Z","shell.execute_reply":"2024-07-13T09:12:59.125626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = expanded_test_desc[[\"row_id\",\"normal_mild\",\"moderate\",\"severe\"]]\nsubmission","metadata":{"execution":{"iopub.status.busy":"2024-07-13T09:12:59.129459Z","iopub.execute_input":"2024-07-13T09:12:59.129739Z","iopub.status.idle":"2024-07-13T09:12:59.148090Z","shell.execute_reply.started":"2024-07-13T09:12:59.129715Z","shell.execute_reply":"2024-07-13T09:12:59.147215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_submission = submission.groupby('row_id').max().reset_index()\ngrouped_submission","metadata":{"execution":{"iopub.status.busy":"2024-07-13T09:12:59.150572Z","iopub.execute_input":"2024-07-13T09:12:59.151380Z","iopub.status.idle":"2024-07-13T09:12:59.172866Z","shell.execute_reply.started":"2024-07-13T09:12:59.151354Z","shell.execute_reply":"2024-07-13T09:12:59.171813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub[['normal_mild', 'moderate', 'severe']] = grouped_submission[['normal_mild', 'moderate', 'severe']]","metadata":{"execution":{"iopub.status.busy":"2024-07-13T09:12:59.174170Z","iopub.execute_input":"2024-07-13T09:12:59.174546Z","iopub.status.idle":"2024-07-13T09:12:59.180969Z","shell.execute_reply.started":"2024-07-13T09:12:59.174510Z","shell.execute_reply":"2024-07-13T09:12:59.180039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\n# Save the DataFrame to \"submission.csv\" in the desired directory\nsub.to_csv(\"/kaggle/working/submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T09:12:59.182139Z","iopub.execute_input":"2024-07-13T09:12:59.182478Z","iopub.status.idle":"2024-07-13T09:12:59.194526Z","shell.execute_reply.started":"2024-07-13T09:12:59.182452Z","shell.execute_reply":"2024-07-13T09:12:59.193657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}