{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":211952941,"sourceType":"kernelVersion"}],"dockerImageVersionId":30823,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport seaborn as sns\n\nimport matplotlib.pyplot as plt\nimport os\nimport time\nimport numpy as np\nimport glob\nimport json\nimport collections\nimport torch\nimport torch.nn as nn\n\nimport pydicom as dicom\nimport matplotlib.patches as patches\n\nfrom matplotlib import animation, rc\nimport pandas as pd\n\nimport pydicom as dicom # dicom\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:33:58.129226Z","iopub.execute_input":"2024-12-18T22:33:58.129634Z","iopub.status.idle":"2024-12-18T22:33:58.787463Z","shell.execute_reply.started":"2024-12-18T22:33:58.129592Z","shell.execute_reply":"2024-12-18T22:33:58.786815Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# read data\ntrain_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/'\n\ntrain  = pd.read_csv(train_path + 'train.csv')\nlabel = pd.read_csv(train_path + 'train_label_coordinates.csv')\ntrain_desc  = pd.read_csv(train_path + 'train_series_descriptions.csv')\ntest_desc   = pd.read_csv(train_path + 'test_series_descriptions.csv')\nsub         = pd.read_csv(train_path + 'sample_submission.csv')\nlen(test_desc) #number of test_description.csv rows ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:33:58.788651Z","iopub.execute_input":"2024-12-18T22:33:58.789088Z","iopub.status.idle":"2024-12-18T22:33:58.938733Z","shell.execute_reply.started":"2024-12-18T22:33:58.789051Z","shell.execute_reply":"2024-12-18T22:33:58.937654Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_desc.head(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:33:58.940571Z","iopub.execute_input":"2024-12-18T22:33:58.940832Z","iopub.status.idle":"2024-12-18T22:33:58.951498Z","shell.execute_reply.started":"2024-12-18T22:33:58.940811Z","shell.execute_reply":"2024-12-18T22:33:58.950810Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to generate image paths based on directory structure\ndef generate_image_paths(df, data_dir):\n    image_paths = []\n    for study_id, series_id in zip(df['study_id'], df['series_id']):\n        study_dir = os.path.join(data_dir, str(study_id))\n        series_dir = os.path.join(study_dir, str(series_id))\n        images = os.listdir(series_dir)\n        image_paths.extend([os.path.join(series_dir, img) for img in images])\n    return image_paths\n\n# Generate image paths for train and test data\ntrain_image_paths = generate_image_paths(train_desc, f'{train_path}/train_images')\ntest_image_paths = generate_image_paths(test_desc, f'{train_path}/test_images')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:33:58.952573Z","iopub.execute_input":"2024-12-18T22:33:58.953248Z","iopub.status.idle":"2024-12-18T22:34:42.355434Z","shell.execute_reply.started":"2024-12-18T22:33:58.953215Z","shell.execute_reply":"2024-12-18T22:34:42.354501Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the base path for test images\nbase_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_images/'\n\n# Function to get image paths for a series\ndef get_image_paths(row):\n    series_path = os.path.join(base_path, str(row['study_id']), str(row['series_id']))\n    if os.path.exists(series_path):\n        return [os.path.join(series_path, f) for f in os.listdir(series_path) if os.path.isfile(os.path.join(series_path, f))]\n    return []\n\n# Mapping of series_description to conditions\ncondition_mapping = {\n    'Sagittal T1': {'left': 'left_neural_foraminal_narrowing', 'right': 'right_neural_foraminal_narrowing'},\n    'Axial T2': {'left': 'left_subarticular_stenosis', 'right': 'right_subarticular_stenosis'},\n    'Sagittal T2/STIR': 'spinal_canal_stenosis'\n}\n\n# Create a list to store the expanded rows\nexpanded_rows = []\n\n# Expand the dataframe by adding new rows for each file path\nfor index, row in test_desc.iterrows():\n    image_paths = get_image_paths(row)\n    conditions = condition_mapping.get(row['series_description'], {})\n    if isinstance(conditions, str):  # Single condition\n        conditions = {'left': conditions, 'right': conditions}\n    for side, condition in conditions.items():\n        for image_path in image_paths:\n            expanded_rows.append({\n                'study_id': row['study_id'],\n                'series_id': row['series_id'],\n                'series_description': row['series_description'],\n                'image_path': image_path,\n                'condition': condition,\n                'row_id': f\"{row['study_id']}_{condition}\"\n            })\n\n# Create a new dataframe from the expanded rows\nexpanded_test_desc = pd.DataFrame(expanded_rows)\n\n# Display the resulting dataframe\nexpanded_test_desc.head(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:34:42.356299Z","iopub.execute_input":"2024-12-18T22:34:42.356611Z","iopub.status.idle":"2024-12-18T22:34:42.433936Z","shell.execute_reply.started":"2024-12-18T22:34:42.356581Z","shell.execute_reply":"2024-12-18T22:34:42.433167Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_data = expanded_test_desc","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:34:53.789021Z","iopub.execute_input":"2024-12-18T22:34:53.789329Z","iopub.status.idle":"2024-12-18T22:34:53.793019Z","shell.execute_reply.started":"2024-12-18T22:34:53.789308Z","shell.execute_reply":"2024-12-18T22:34:53.792132Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\n# Define a function to check if a path exists\ndef check_exists(path):\n    return os.path.exists(path)\n\n# Define a function to check if a study ID directory exists\ndef check_study_id(row):\n    study_id = row['study_id']\n    path = f'{train_path}/train_images/{study_id}'\n    return check_exists(path)\n\n# Define a function to check if a series ID directory exists\ndef check_series_id(row):\n    study_id = row['study_id']\n    series_id = row['series_id']\n    path = f'{train_path}/train_images/{study_id}/{series_id}'\n    return check_exists(path)\n\n# Define a function to check if an image file exists\ndef check_image_exists(row):\n    image_path = row['image_path']\n    return check_exists(image_path)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:36:17.707541Z","iopub.execute_input":"2024-12-18T22:36:17.707904Z","iopub.status.idle":"2024-12-18T22:36:17.712900Z","shell.execute_reply.started":"2024-12-18T22:36:17.707874Z","shell.execute_reply":"2024-12-18T22:36:17.711973Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_dicom(path):\n    dicom = pydicom.dcmread(path)\n    data = dicom.pixel_array\n    data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:36:19.488042Z","iopub.execute_input":"2024-12-18T22:36:19.488332Z","iopub.status.idle":"2024-12-18T22:36:19.493034Z","shell.execute_reply.started":"2024-12-18T22:36:19.488310Z","shell.execute_reply":"2024-12-18T22:36:19.492032Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision.models as models\nfrom torchvision import transforms\nfrom torch.utils.data import DataLoader\nfrom sklearn.model_selection import train_test_split\nimport pandas as pd\nfrom tqdm import tqdm\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:38:34.436191Z","iopub.execute_input":"2024-12-18T22:38:34.436525Z","iopub.status.idle":"2024-12-18T22:38:34.582766Z","shell.execute_reply.started":"2024-12-18T22:38:34.436496Z","shell.execute_reply":"2024-12-18T22:38:34.581869Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"expanded_test_desc","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:40:25.027097Z","iopub.execute_input":"2024-12-18T22:40:25.027387Z","iopub.status.idle":"2024-12-18T22:40:25.040272Z","shell.execute_reply.started":"2024-12-18T22:40:25.027365Z","shell.execute_reply":"2024-12-18T22:40:25.039272Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"levels = ['l1_l2', 'l2_l3', 'l3_l4', 'l4_l5', 'l5_s1']\n\n# Function to update row_id with levels\ndef update_row_id(row, levels):\n    level = levels[row.name % len(levels)]\n    return f\"{row['study_id']}_{row['condition']}_{level}\"\n\n# Update row_id in expanded_test_desc to include levels\nexpanded_test_desc['row_id'] = expanded_test_desc.apply(lambda row: update_row_id(row, levels), axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:42:52.734780Z","iopub.execute_input":"2024-12-18T22:42:52.735125Z","iopub.status.idle":"2024-12-18T22:42:52.743348Z","shell.execute_reply.started":"2024-12-18T22:42:52.735101Z","shell.execute_reply":"2024-12-18T22:42:52.742524Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"expanded_test_desc.head(2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:43:02.239248Z","iopub.execute_input":"2024-12-18T22:43:02.239657Z","iopub.status.idle":"2024-12-18T22:43:02.252338Z","shell.execute_reply.started":"2024-12-18T22:43:02.239623Z","shell.execute_reply":"2024-12-18T22:43:02.251481Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as transforms\nimport torch\nimport torch.optim.lr_scheduler as lr_scheduler\nfrom tqdm import tqdm\n# Define a custom test dataset class\nclass TestDataset(Dataset):\n    def __init__(self, dataframe, transform=None):\n        self.dataframe = dataframe\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.dataframe)\n\n    def __getitem__(self, index):\n        image_path = self.dataframe['image_path'][index]\n        image = load_dicom(image_path)  # Define this function to load your DICOM images\n        if self.transform:\n            image = self.transform(image)\n        return image\n\n# Define the transforms\ntransform = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.Resize((224, 224)),\n    transforms.Grayscale(num_output_channels=3),\n    transforms.ToTensor(),\n])\n\n# Create a test dataset and dataloader\ntest_dataset = TestDataset(expanded_test_desc, transform)\ntestloader = DataLoader(test_dataset, batch_size=1, shuffle=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:44:36.558297Z","iopub.execute_input":"2024-12-18T22:44:36.558611Z","iopub.status.idle":"2024-12-18T22:44:36.565046Z","shell.execute_reply.started":"2024-12-18T22:44:36.558587Z","shell.execute_reply":"2024-12-18T22:44:36.564156Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for image in testloader:\n    print(image.shape)\n    break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:44:51.098377Z","iopub.execute_input":"2024-12-18T22:44:51.098754Z","iopub.status.idle":"2024-12-18T22:44:51.154187Z","shell.execute_reply.started":"2024-12-18T22:44:51.098699Z","shell.execute_reply":"2024-12-18T22:44:51.153265Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CustomResNet50(nn.Module):\n    def __init__(self, num_classes=3, pretrained_weights=None):\n        super(CustomResNet50, self).__init__()\n        # pretrained=False ile modelin rastgele ağırlıklarla başlatılmasını sağla\n        self.model = models.resnet50(pretrained=False)\n        \n        # Eğer manuel ağırlık yolu verilmişse, bu ağırlıkları yükle\n        if pretrained_weights:\n            self.model.load_state_dict(torch.load(pretrained_weights))\n        \n        num_ftrs = self.model.fc.in_features  # Son katmanın özellik sayısını al\n        self.model.fc = nn.Linear(num_ftrs, num_classes)  # Son katmanı değiştir\n\n    def forward(self, x):\n        return self.model(x)\n\n    def unfreeze_model(self):\n        \"\"\"Tüm katmanları çöz.\"\"\"\n        for param in self.model.parameters():\n            param.requires_grad = True\n\n    def unfreeze_specific_layers(self, layer_names=None):\n        \"\"\"\n        Belirli katmanları çözmek için kullanılabilir.\n        Eğer layer_names None ise, tüm katmanlar çözülür.\n        \"\"\"\n        for name, param in self.model.named_parameters():\n            if layer_names is None or any(layer in name for layer in layer_names):\n                param.requires_grad = True\n            else:\n                param.requires_grad = False\nimport torch\n\n# Function to get the model based on series_description and load pretrained weights\ndef get_model(series_description, weights_paths):\n    model_name = series_description\n    if model_name in weights_paths:\n        # Create a model instance (CustomResNet50 in this case)\n        model = CustomResNet50(num_classes=3).to(device)\n        \n        # Load pretrained weights\n        model.load_state_dict(torch.load(weights_paths[model_name], map_location=device))\n        model.eval()  # Set the model to eval mode\n        return model\n    return None\n\n# Function to make predictions on the test data\ndef predict_test_data(testloader, expanded_test_desc, weights_paths):\n    predictions = []\n    normal_mild_probs = []\n    moderate_probs = []\n    severe_probs = []\n    \n    for idx, images in enumerate(tqdm(testloader)):\n        images = images.to(device)\n        series_description = expanded_test_desc.iloc[idx]['series_description']\n        model = get_model(series_description, weights_paths)\n        \n        if model:\n            with torch.no_grad():\n                outputs = model(images)\n                probs = torch.softmax(outputs, dim=1).squeeze(0)\n                normal_mild_probs.append(probs[0].item())\n                moderate_probs.append(probs[1].item())\n                severe_probs.append(probs[2].item())\n                predictions.append(probs)\n        else:\n            normal_mild_probs.append(None)\n            moderate_probs.append(None)\n            severe_probs.append(None)\n            predictions.append(None)\n    \n    return normal_mild_probs, moderate_probs, severe_probs, predictions\nweights_paths = {\n    'Sagittal T1': '/kaggle/input/unpretrained-plz-dont-give-any-error-messages/best_model_1.pth',\n    'Axial T2': '/kaggle/input/unpretrained-plz-dont-give-any-error-messages/best_model_2.pth',\n    'Sagittal T2/STIR': '/kaggle/input/unpretrained-plz-dont-give-any-error-messages/best_model_3.pth'\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T23:03:08.560256Z","iopub.execute_input":"2024-12-18T23:03:08.560613Z","iopub.status.idle":"2024-12-18T23:03:08.570634Z","shell.execute_reply.started":"2024-12-18T23:03:08.560582Z","shell.execute_reply":"2024-12-18T23:03:08.569807Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Make predictions on the test data\nnormal_mild_probs, moderate_probs, severe_probs, test_predictions = predict_test_data(testloader, expanded_test_desc, weights_paths)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T23:03:12.072304Z","iopub.execute_input":"2024-12-18T23:03:12.072618Z","iopub.status.idle":"2024-12-18T23:03:16.596495Z","shell.execute_reply.started":"2024-12-18T23:03:12.072592Z","shell.execute_reply":"2024-12-18T23:03:16.595218Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_predictions[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:54:04.207030Z","iopub.execute_input":"2024-12-18T22:54:04.207400Z","iopub.status.idle":"2024-12-18T22:54:04.404649Z","shell.execute_reply.started":"2024-12-18T22:54:04.207370Z","shell.execute_reply":"2024-12-18T22:54:04.403887Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Add predictions and probabilities to the test DataFrame\nexpanded_test_desc['normal_mild'] = normal_mild_probs\nexpanded_test_desc['moderate'] = moderate_probs\nexpanded_test_desc['severe'] = severe_probs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:54:13.369069Z","iopub.execute_input":"2024-12-18T22:54:13.369353Z","iopub.status.idle":"2024-12-18T22:54:13.374040Z","shell.execute_reply.started":"2024-12-18T22:54:13.369330Z","shell.execute_reply":"2024-12-18T22:54:13.373026Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = expanded_test_desc[[\"row_id\",\"normal_mild\",\"moderate\",\"severe\"]]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:54:20.484398Z","iopub.execute_input":"2024-12-18T22:54:20.484699Z","iopub.status.idle":"2024-12-18T22:54:20.490026Z","shell.execute_reply.started":"2024-12-18T22:54:20.484676Z","shell.execute_reply":"2024-12-18T22:54:20.489183Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:54:29.724629Z","iopub.execute_input":"2024-12-18T22:54:29.724949Z","iopub.status.idle":"2024-12-18T22:54:29.737644Z","shell.execute_reply.started":"2024-12-18T22:54:29.724925Z","shell.execute_reply":"2024-12-18T22:54:29.736643Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group by 'row_id' and sum the values\ngrouped_submission = submission.groupby('row_id').max().reset_index()\n\n# Normalize the columns\ngrouped_submission[['normal_mild', 'moderate', 'severe']] = grouped_submission[['normal_mild', 'moderate', 'severe']].div(grouped_submission[['normal_mild', 'moderate', 'severe']].sum(axis=1), axis=0)\n\n# Check the first 3 rows\ngrouped_submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:54:48.962995Z","iopub.execute_input":"2024-12-18T22:54:48.963358Z","iopub.status.idle":"2024-12-18T22:54:48.990403Z","shell.execute_reply.started":"2024-12-18T22:54:48.963325Z","shell.execute_reply":"2024-12-18T22:54:48.989563Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(grouped_submission)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:54:57.619321Z","iopub.execute_input":"2024-12-18T22:54:57.619673Z","iopub.status.idle":"2024-12-18T22:54:57.624805Z","shell.execute_reply.started":"2024-12-18T22:54:57.619643Z","shell.execute_reply":"2024-12-18T22:54:57.623969Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub[['normal_mild', 'moderate', 'severe']] = grouped_submission[['normal_mild', 'moderate', 'severe']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:55:13.312574Z","iopub.execute_input":"2024-12-18T22:55:13.312926Z","iopub.status.idle":"2024-12-18T22:55:13.318536Z","shell.execute_reply.started":"2024-12-18T22:55:13.312896Z","shell.execute_reply":"2024-12-18T22:55:13.317647Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\n# Save the DataFrame to \"submission.csv\" in the desired directory\nsub.to_csv(\"/kaggle/working/submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:55:22.353612Z","iopub.execute_input":"2024-12-18T22:55:22.353960Z","iopub.status.idle":"2024-12-18T22:55:22.359850Z","shell.execute_reply.started":"2024-12-18T22:55:22.353935Z","shell.execute_reply":"2024-12-18T22:55:22.358788Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub.head(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:55:27.894680Z","iopub.execute_input":"2024-12-18T22:55:27.895062Z","iopub.status.idle":"2024-12-18T22:55:27.904493Z","shell.execute_reply.started":"2024-12-18T22:55:27.895034Z","shell.execute_reply":"2024-12-18T22:55:27.903414Z"}},"outputs":[],"execution_count":null}]}