{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":9071670,"sourceType":"datasetVersion","datasetId":5471909}],"dockerImageVersionId":30746,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#new submission\nimport os\nimport pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torchvision import models, transforms\nfrom torch.utils.data import Dataset, DataLoader\nimport pydicom\nfrom PIL import Image, ImageOps\nimport random\nfrom sklearn.preprocessing import LabelEncoder, label_binarize\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_curve, auc\nimport matplotlib.pyplot as plt\nfrom itertools import cycle\nfrom functools import lru_cache\nfrom torch.cuda.amp import GradScaler, autocast\n\n#DEBUG = True\n#if DEBUG == True:\n#    BASE_DIR = '/kaggle/input/rsna-lsdc-2024-submission-debug-dataset'\n#else:\n#    BASE_DIR = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/'\n\nBASE_DIR = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/'\nTRAIN_LABELS = os.path.join(BASE_DIR, 'train_label_coordinates.csv')\nSERIES_DESC = os.path.join(BASE_DIR, 'train_series_descriptions.csv')\nIMAGE_DIR = os.path.join(BASE_DIR, 'train_images')\n\n# Custom transformation functions\ndef resize(image, size):\n    return image.resize(size, Image.Resampling.LANCZOS)\n\ndef random_horizontal_flip(image, p=0.5):\n    if random.random() < p:\n        return ImageOps.mirror(image)\n    return image\n\ndef random_rotation(image, degrees):\n    return image.rotate(random.uniform(-degrees, degrees))\n\ndef to_tensor(image):\n    image = np.array(image, dtype=np.float32).transpose((2, 0, 1))\n    return torch.tensor(image / 255.0)\n\ndef normalize(image, mean, std):\n    for t, m, s in zip(image, mean, std):\n        t.sub_(m).div_(s)\n    return image\n\n# Custom Dataset to load DICOM images\nclass DicomDataset(Dataset):\n    def __init__(self, df, image_dir, transform=None, is_test=False):\n        self.df = df\n        self.image_dir = image_dir\n        self.transform = transform\n        self.is_test = is_test\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        dicom_path = os.path.join(self.image_dir, str(row['study_id']), str(row['series_id']), f\"{row['instance_number']}.dcm\")\n        \n        dicom = pydicom.dcmread(dicom_path)\n        image = dicom.pixel_array\n        image = Image.fromarray(image).convert('RGB')\n        \n        if self.transform:\n            image = self.transform(image)\n        \n        if self.is_test:\n            label = torch.tensor(0, dtype=torch.long)  # Placeholder label for test data\n        else:\n            label = torch.tensor(row['severity_encoded'], dtype=torch.long)\n        \n        return image, label, row['row_id']\n\n@lru_cache(maxsize=None)\ndef load_and_process_data(is_train=True):\n    if is_train:\n        labels_df = pd.read_csv(TRAIN_LABELS)\n        series_df = pd.read_csv(SERIES_DESC)\n        train_df = pd.read_csv(os.path.join(BASE_DIR, 'train.csv'))\n        image_dir = IMAGE_DIR\n        \n        # Merge all training data\n        df = pd.merge(labels_df, series_df, on=['study_id', 'series_id'])\n        df = pd.merge(df, train_df, on='study_id')\n        \n        # Get all condition columns from train.csv\n        condition_columns = [col for col in train_df.columns if col != 'study_id']\n        \n        # Melt the DataFrame to create a row for each condition\n        df_melted = pd.melt(df, id_vars=['study_id', 'series_id', 'instance_number'], \n                            value_vars=condition_columns, \n                            var_name='condition', value_name='severity')\n        \n        # Split condition into condition and level\n        df_melted[['condition', 'level']] = df_melted['condition'].str.rsplit('_', n=1, expand=True)\n        \n        # Create row_id\n        df_melted['row_id'] = df_melted.apply(lambda row: f\"{row['study_id']}_{row['condition']}_{row['level']}\", axis=1)\n        \n        conditions = df_melted['condition'].unique().tolist()\n        \n    else:\n        series_df = pd.read_csv(os.path.join(BASE_DIR, 'test_series_descriptions.csv'))\n        image_dir = os.path.join(BASE_DIR, 'test_images')\n        \n        # Load conditions from training data\n        train_labels = pd.read_csv(TRAIN_LABELS)\n        conditions = train_labels['condition'].unique().tolist()\n        levels = train_labels['level'].unique().tolist()\n        \n        # Create a list to store all image file information\n        image_files = []\n        for root, dirs, files in os.walk(image_dir):\n            for file in files:\n                if file.endswith('.dcm'):\n                    parts = os.path.relpath(root, image_dir).split(os.sep)\n                    if len(parts) >= 2:\n                        study_id, series_id = parts[:2]\n                        instance_number = os.path.splitext(file)[0]\n                        image_files.append({\n                            'study_id': study_id,\n                            'series_id': series_id,\n                            'instance_number': instance_number\n                        })\n        \n        # Create a DataFrame from the image files\n        image_df = pd.DataFrame(image_files)\n        \n        # Merge the image information with the series description\n        series_df[\"study_id\"] = series_df[\"study_id\"].astype(object)\n        image_df[\"study_id\"] = image_df[\"study_id\"].astype(object)\n        series_df[\"series_id\"] = series_df[\"series_id\"].astype(object)\n        image_df[\"series_id\"] = image_df[\"series_id\"].astype(object)\n        df = pd.merge(series_df, image_df, on=['study_id', 'series_id'], how='right')\n\n        combinations = []\n        for _, row in df.iterrows():\n            for condition in conditions:\n                for level in levels:\n                    combinations.append({\n                        'study_id': row['study_id'],\n                        'series_id': row['series_id'],\n                        'instance_number': row['instance_number'],\n                        'condition': condition,\n                        'level': level\n                    })\n        df_melted = pd.DataFrame(combinations)\n        # Create a row for each condition for each image\n        #df_melted = df.loc[df.index.repeat(len(conditions))].reset_index(drop=True)\n        #df_melted['condition'] = conditions * len(df)\n        #df_melted['level'] = levels * len(df) * len(conditions)\n        #df_melted['level'] = levels * len(df)\n        \n        # Add placeholder severity (will be predicted by the model)\n        df_melted['severity'] = 'unknown'\n        #TODO: dynamically assign a value for 'level' for your test data\n        \n        # Create row_id\n        df_melted['row_id'] = df_melted.apply(lambda row: f\"{row['study_id']}_{row['condition']}_{row['level']}\", axis=1)\n    \n    # LabelEncoder for severity\n    le = LabelEncoder()\n    if is_train:\n        df_melted['severity_encoded'] = le.fit_transform(df_melted['severity']).astype('int64')\n    else:\n        le.classes_ = np.array(['Normal/Mild', 'Moderate', 'Severe'])  # Set classes based on training data\n    \n    return df_melted, image_dir, le, conditions\n\n# Custom transformation functions\ndef transform(image):\n    image = resize(image, (224, 224))\n    image = random_horizontal_flip(image)\n    image = random_rotation(image, 10)\n    image = to_tensor(image)\n    image = normalize(image, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n    return image\n\n# Define a CNN model\nclass CNNModel(nn.Module):\n    def __init__(self, num_conditions=5):\n        super(CNNModel, self).__init__()\n        self.feature_extractor = nn.Sequential(\n            nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False),\n            nn.BatchNorm2d(64),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(kernel_size=3, stride=2, padding=1),\n            nn.Conv2d(64, 128, kernel_size=3, stride=2, padding=1, bias=False),\n            nn.BatchNorm2d(128),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(128, 256, kernel_size=3, stride=2, padding=1, bias=False),\n            nn.BatchNorm2d(256),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(256, 512, kernel_size=3, stride=2, padding=1, bias=False),\n            nn.BatchNorm2d(512),\n            nn.ReLU(inplace=True),\n        )\n        self.fc1 = nn.Linear(512 * 7 * 7, 256)  # Adjusted based on the output feature map size\n        self.fc2 = nn.Linear(256, num_conditions * 3)\n\n    def forward(self, x):\n        x = self.feature_extractor(x)\n        x = x.view(x.size(0), -1)\n        x = nn.ReLU()(self.fc1(x))\n        x = self.fc2(x)\n        return x\n\nscaler = GradScaler()\n\ndef train_model(model, train_loader, criterion, optimizer, num_epochs=10, device='cpu'):\n    model.train()\n    for epoch in range(num_epochs):\n        running_loss = 0.0\n        for images, labels, _ in train_loader:\n            images, labels = images.to(device), labels.to(device).long()\n            optimizer.zero_grad()\n            \n            with autocast():\n                outputs = model(images)\n                loss = criterion(outputs, labels)\n            \n            scaler.scale(loss).backward()\n            scaler.step(optimizer)\n            scaler.update()\n\n            running_loss += loss.item()\n        \n        print(f'Epoch [{epoch+1}/{num_epochs}], Loss: {running_loss/len(train_loader):.4f}')\n    print('Training complete')\n\ndef evaluate_model(model, data_loader, device, return_labels=False):\n    model.eval()\n    all_preds = []\n    all_labels = []\n    all_row_ids = []\n    with torch.no_grad():\n        for images, labels, row_ids in data_loader:\n            images = images.to(device)\n            outputs = model(images)\n            all_preds.append(outputs.cpu())\n            if return_labels:\n                all_labels.append(labels)\n            all_row_ids.extend(row_ids)\n    \n    all_preds = torch.cat(all_preds).numpy()\n    if return_labels:\n        all_labels = torch.cat(all_labels).numpy()\n        return all_labels, all_preds, all_row_ids\n    else:\n        return all_preds, all_row_ids\n\ndef plot_roc_curve(y_true, y_pred, num_classes):\n    # Apply softmax to raw predictions\n    y_pred = torch.nn.functional.softmax(torch.tensor(y_pred), dim=1).numpy()\n    \n    # Binarize the labels\n    y_true = label_binarize(y_true, classes=range(num_classes))\n    \n    fpr = dict()\n    tpr = dict()\n    roc_auc = dict()\n    for i in range(num_classes):\n        fpr[i], tpr[i], _ = roc_curve(y_true[:, i], y_pred[:, i])\n        roc_auc[i] = auc(fpr[i], tpr[i])\n\n    plt.figure()\n    colors = cycle(['blue', 'red', 'green'])\n    for i, color in zip(range(num_classes), colors):\n        plt.plot(fpr[i], tpr[i], color=color, lw=2,\n                 label=f'ROC curve of class {i} (area = {roc_auc[i]:0.2f})')\n    \n    plt.plot([0, 1], [0, 1], 'k--', lw=2)\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.05])\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver Operating Characteristic (ROC) Curve')\n    plt.legend(loc=\"lower right\")\n    plt.show()\n\ndef process_row_id(row_id):\n    parts = row_id.split('_')\n    study_id = parts[0]\n    condition = '_'.join(parts[1:-1]).lower()\n    level = parts[-1].lower().replace('/', '_')\n    return f\"{study_id}_{condition}_{level}\"\n\ndef create_submission(y_pred, row_ids, le):\n    y_pred_proba = torch.nn.functional.softmax(torch.tensor(y_pred), dim=1).numpy()\n    \n    submission = pd.DataFrame({\n        'row_id': row_ids,\n        'normal_mild': y_pred_proba[:, 0],\n        'moderate': y_pred_proba[:, 1],\n        'severe': y_pred_proba[:, 2]\n    })\n    \n    submission['row_id'] = submission['row_id'].str.split('_').apply(lambda x: f\"{x[0]}_{('_'.join(x[1:-1])).lower()}_{x[-1].lower().replace('/', '_')}\")\n    \n    submission = submission.drop_duplicates(subset='row_id', keep='first')\n    \n    prob_columns = ['normal_mild', 'moderate', 'severe']\n    submission[prob_columns] = submission[prob_columns].div(submission[prob_columns].sum(axis=1), axis=0)\n    \n    submission[prob_columns] = submission[prob_columns].round(7)\n    \n    return submission\n\n\n# Main script\nimage_dir = IMAGE_DIR  # Update this to your image directory path\ntrain_df, train_image_dir, le, conditions = load_and_process_data(is_train=True)  # Your function to load and process the DataFrame\n\nsubset_size = 1500  # Adjust this value as needed\nsubset_indices = random.sample(range(len(train_df)), subset_size)\nsubset_df = train_df.iloc[subset_indices].reset_index(drop=True)\n\ntrain_df, val_df = train_test_split(subset_df, test_size=0.2, random_state=42)\ntrain_df.reset_index(drop=True, inplace=True)\nval_df.reset_index(drop=True, inplace=True)\n\ntrain_dataset = DicomDataset(train_df, train_image_dir, transform)\nval_dataset = DicomDataset(val_df, train_image_dir, transform)\n\ntrain_loader = DataLoader(train_dataset, batch_size=32, shuffle=True, num_workers=os.cpu_count())\nval_loader = DataLoader(val_dataset, batch_size=32, shuffle=False, num_workers=os.cpu_count())\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nnum_classes = len(le.classes_)\nmodel = CNNModel(num_classes).to(device)\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\ntrain_model(model, train_loader, criterion, optimizer, num_epochs=10)\n\nval_true, val_pred, _ = evaluate_model(model, val_loader, device, return_labels=True)\n\nplot_roc_curve(val_true, val_pred, num_classes)\n\ntest_df, test_image_dir, _, conditions = load_and_process_data(is_train=False)\n\ntest_dataset = DicomDataset(test_df, test_image_dir, transform, is_test=True)\ntest_loader = DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=os.cpu_count())\n\ntest_pred, row_ids = evaluate_model(model, test_loader, device, return_labels=False)\n\nsubmission = create_submission(test_pred, row_ids, le)\nsubmission = submission[[\"row_id\", \"normal_mild\", \"moderate\", \"severe\"]]\ndefault_na_value = float(1.0/3.0)\nsubmission[\"normal_mild\"] = pd.to_numeric(submission[\"normal_mild\"], errors='coerce')\nsubmission[\"moderate\"] = pd.to_numeric(submission[\"moderate\"], errors='coerce')\nsubmission[\"severe\"] = pd.to_numeric(submission[\"severe\"], errors='coerce')\nmask = submission[['normal_mild', 'moderate', 'severe']].isna().any(axis=1)\nsubmission.loc[mask, ['normal_mild', 'moderate', 'severe']] = default_na_value\nsubmission = submission.sort_values(by=\"row_id\")\nsubmission.to_csv('submission.csv', index=False)\npd.read_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-08-11T19:23:53.656995Z","iopub.execute_input":"2024-08-11T19:23:53.657492Z","iopub.status.idle":"2024-08-11T19:33:48.540780Z","shell.execute_reply.started":"2024-08-11T19:23:53.657452Z","shell.execute_reply":"2024-08-11T19:33:48.539501Z"},"trusted":true},"execution_count":null,"outputs":[]}]}