{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.11"},"colab":{"gpuType":"T4","provenance":[]},"accelerator":"GPU","kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":43032.523587,"end_time":"2025-05-02T09:33:25.640951","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-05-01T21:36:13.117364","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install -q kaggle\n# !mkdir -p ~/.kaggle\n# !cp kaggle.json ~/.kaggle/\n# !chmod 600 ~/.kaggle/kaggle.json\n# !kaggle competitions download -c rsna-2024-lumbar-spine-degenerative-classification\n# !unzip -qq /content/rsna-2024-lumbar-spine-degenerative-classification.zip\n\n!pip install pydicom\n!pip install matplotlib seaborn scikit-learn\n\n","metadata":{"execution":{"iopub.execute_input":"2025-05-01T21:36:17.226062Z","iopub.status.busy":"2025-05-01T21:36:17.225833Z","iopub.status.idle":"2025-05-01T21:36:23.990405Z","shell.execute_reply":"2025-05-01T21:36:23.989449Z"},"id":"Mu5gOtacLtn0","outputId":"7106a8d2-7bab-4307-b13f-47e02c843c8e","papermill":{"duration":6.770244,"end_time":"2025-05-01T21:36:23.992004","exception":false,"start_time":"2025-05-01T21:36:17.221760","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import cv2\nimport os\nimport time\nimport json\nimport glob\nimport random\nimport collections\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pydicom as dicom\nfrom tqdm import tqdm\nfrom copy import deepcopy\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix\nimport joblib\nimport pydicom as dicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models, applications, optimizers, callbacks\nfrom tensorflow.keras import backend as K\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n","metadata":{"execution":{"iopub.execute_input":"2025-05-01T21:36:23.999134Z","iopub.status.busy":"2025-05-01T21:36:23.998495Z","iopub.status.idle":"2025-05-01T21:36:40.192184Z","shell.execute_reply":"2025-05-01T21:36:40.191200Z"},"papermill":{"duration":16.198546,"end_time":"2025-05-01T21:36:40.193602","exception":false,"start_time":"2025-05-01T21:36:23.995056","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Define the path to the training data\ntrain_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/'\n\n# Load CSV files containing metadata and labels\ntrain = pd.read_csv(os.path.join(train_path, 'train.csv'))\nlabel = pd.read_csv(os.path.join(train_path, 'train_label_coordinates.csv'))\ntrain_desc = pd.read_csv(os.path.join(train_path, 'train_series_descriptions.csv'))\ntest_desc = pd.read_csv(os.path.join(train_path, 'test_series_descriptions.csv'))\nsub = pd.read_csv(os.path.join(train_path, 'sample_submission.csv'))\n","metadata":{"execution":{"iopub.execute_input":"2025-05-01T21:36:40.200179Z","iopub.status.busy":"2025-05-01T21:36:40.199498Z","iopub.status.idle":"2025-05-01T21:36:40.360056Z","shell.execute_reply":"2025-05-01T21:36:40.359117Z"},"papermill":{"duration":0.165321,"end_time":"2025-05-01T21:36:40.361713","exception":false,"start_time":"2025-05-01T21:36:40.196392","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# # Display the first few rows of each dataframe\n# print(\"Test Descriptions:\")\n# print(test_desc.head(5))\n\n# print(\"\\nTrain Data:\")\n# print(train.head(5))\n\n# print(\"\\nTrain Series Descriptions:\")\n# print(train_desc.head(5))\n\nimport random\n\n# Generate Image Paths\n# ---------------------\ndef generate_image_paths(df, data_dir):\n    image_paths = []\n    for study_id, series_id in zip(df['study_id'], df['series_id']):\n        study_dir = os.path.join(data_dir, str(study_id))\n        series_dir = os.path.join(study_dir, str(series_id))\n        # Sort images by filename to maintain DICOM sequence\n        images = sorted(os.listdir(series_dir))\n        image_paths.extend([os.path.join(series_dir, img) for img in images])\n    return image_paths\n\n# Generate image paths for training and testing datasets\ntrain_image_paths = generate_image_paths(train_desc, os.path.join(train_path, 'train_images'))\ntest_image_paths = generate_image_paths(test_desc, os.path.join(train_path, 'test_images'))\n\n# Example usage\nprint(\"\\nSample Train Image Path:\", train_image_paths[2])\nprint(\"Number of Train Descriptions:\", len(train_desc))\nprint(\"Number of Train Image Paths:\", len(train_image_paths))\n\n# Display DICOM Images\n# ---------------------\nimport pydicom\ndef display_dicom_images(image_paths, num_images=3):\n    \"\"\"\n    Display a specified number of DICOM images.\n\n    Parameters:\n        image_paths (List[str]): List of DICOM image file paths.\n        num_images (int): Number of images to display.\n    \"\"\"\n    plt.figure(figsize=(15, 5))\n    for i, path in enumerate(image_paths[:num_images]):\n        ds = pydicom.dcmread(path)\n        plt.subplot(1, num_images, i+1)\n        plt.imshow(ds.pixel_array, cmap=plt.cm.bone)\n        plt.title(f\"Image {i+1}\")\n        plt.axis('off')\n    plt.show()\n\n# Display the first three DICOM images from training data\ndisplay_dicom_images(train_image_paths)\n\n# Display DICOM Images with Coordinates\n# -------------------------------------\ndef display_dicom_with_coordinates(image_paths, label_df):\n    \"\"\"\n    Display DICOM images with annotated coordinates.\n\n    Parameters:\n        image_paths (List[str]): List of DICOM image file paths.\n        label_df (DataFrame): DataFrame containing label coordinates.\n    \"\"\"\n    fig, axs = plt.subplots(1, len(image_paths), figsize=(18, 6))\n\n    for idx, path in enumerate(image_paths):\n        study_id = int(path.split('/')[-3])\n        series_id = int(path.split('/')[-2])\n\n        # Filter labels for the current study and series\n        filtered_labels = label_df[\n            (label_df['study_id'] == study_id) &\n            (label_df['series_id'] == series_id)\n        ]\n\n        # Read DICOM image\n        ds = pydicom.dcmread(path)\n\n        # Plot DICOM image\n        axs[idx].imshow(ds.pixel_array, cmap='gray')\n        axs[idx].set_title(f\"Study ID: {study_id}, Series ID: {series_id}\")\n        axs[idx].axis('off')\n\n        # Plot coordinates\n        for _, row in filtered_labels.iterrows():\n            axs[idx].plot(row['x'], row['y'], 'ro', markersize=5)\n\n    plt.tight_layout()\n    plt.show()\n\ndef load_dicom_files(path_to_folder):\n    \"\"\"\n    Load and sort DICOM files from a specified folder.\n\n    Parameters:\n        path_to_folder (str): Directory containing DICOM files.\n\n    Returns:\n        List[str]: Sorted list of DICOM file paths.\n    \"\"\"\n    files = [os.path.join(path_to_folder, f) for f in os.listdir(path_to_folder) if f.endswith('.dcm')]\n    files.sort(key=lambda x: int(os.path.splitext(os.path.basename(x))[0].split('-')[-1]))\n    return files\n\n# Example: Display DICOM images with coordinates for a specific study\nstudy_id = \"100206310\"\nstudy_folder = os.path.join(train_path, 'train_images', study_id)\n\nimage_paths = []\nfor series_folder in os.listdir(study_folder):\n    series_folder_path = os.path.join(study_folder, series_folder)\n    dicom_files = load_dicom_files(series_folder_path)\n    if dicom_files:\n        image_paths.append(dicom_files[0])  # Add the first image from each series\n\ndisplay_dicom_with_coordinates(image_paths, label)\n\n# Data Reshaping and Merging\n# --------------------------\ndef reshape_row(row):\n    \"\"\"\n    Reshape a single row of the DataFrame to separate conditions, levels, and severities.\n\n    Parameters:\n        row (Series): A row from the DataFrame.\n\n    Returns:\n        DataFrame: Reshaped DataFrame.\n    \"\"\"\n    data = {'study_id': [], 'condition': [], 'level': [], 'severity': []}\n\n    for column, value in row.items():\n        if column not in ['study_id', 'series_id', 'instance_number', 'x', 'y', 'series_description']:\n            parts = column.split('_')\n            condition = ' '.join([word.capitalize() for word in parts[:-2]])\n            level = parts[-2].capitalize() + '/' + parts[-1].capitalize()\n            data['study_id'].append(row['study_id'])\n            data['condition'].append(condition)\n            data['level'].append(level)\n            data['severity'].append(value)\n\n    return pd.DataFrame(data)\n\n# Reshape the training DataFrame\nnew_train_df = pd.concat([reshape_row(row) for _, row in train.iterrows()], ignore_index=True)\n\n# Display the first few rows of the reshaped DataFrame\nprint(\"\\nReshaped Train Data:\")\nprint(new_train_df.head(5))\n\n# Print columns for verification\nprint(\"\\nColumns in new_train_df:\")\nprint(\", \".join(new_train_df.columns))\n\nprint(\"\\nColumns in label:\")\nprint(\", \".join(label.columns))\n\nprint(\"\\nColumns in test_desc:\")\nprint(\", \".join(test_desc.columns))\n\nprint(\"\\nColumns in sub:\")\nprint(\", \".join(sub.columns))\n\n# Merge DataFrames\n# -----------------\n# Merge reshaped training data with label coordinates\nmerged_df = pd.merge(new_train_df, label, on=['study_id', 'condition', 'level'], how='inner')\n\n# Further merge with training series descriptions\nfinal_merged_df = pd.merge(merged_df, train_desc, on=['series_id', 'study_id'], how='inner')\n\n# Display the first few rows of the final merged DataFrame\nprint(\"\\nFinal Merged Data:\")\nprint(final_merged_df.head(5))\n\n# Example Queries\nprint(\"\\nEntries for Study ID 100206310:\")\nprint(final_merged_df[final_merged_df['study_id'] == 100206310].sort_values(['x','y'], ascending=True))\n\nprint(\"\\nEntries for Series ID 1012284084:\")\nprint(final_merged_df[final_merged_df['series_id'] == 1012284084].sort_values(\"instance_number\"))\n\n# Create the 'row_id' column by combining study_id, condition, and level\nfinal_merged_df['row_id'] = (\n    final_merged_df['study_id'].astype(str) + '_' +\n    final_merged_df['condition'].str.lower().str.replace(' ', '_') + '_' +\n    final_merged_df['level'].str.lower().str.replace('/', '_')\n)\n\n# Create the 'image_path' column based on directory structure\nfinal_merged_df['image_path'] = (\n    os.path.join(train_path, 'train_images') + '/' +\n    final_merged_df['study_id'].astype(str) + '/' +\n    final_merged_df['series_id'].astype(str) + '/' +\n    final_merged_df['instance_number'].astype(str) + '.dcm'\n)\n\n# Display the updated DataFrame\nprint(\"\\nUpdated Final Merged DataFrame:\")\nprint(final_merged_df.head(5))\n\nnormal_mild_count = final_merged_df[final_merged_df[\"severity\"] == \"Normal/Mild\"].shape[0]\nmoderate_count = final_merged_df[final_merged_df[\"severity\"] == \"Moderate\"].shape[0]\nsevere_count = final_merged_df[final_merged_df[\"severity\"] == \"Severe\"].shape[0]\n\nprint(f\"\\nNormal/Mild Count: {normal_mild_count}\")\nprint(f\"Moderate Count: {moderate_count}\")\nprint(f\"Severe Count: {severe_count}\")\n\nbase_path = '/content/test_images/'\n\ndef get_image_paths(row):\n    series_path = os.path.join(base_path, str(row['study_id']), str(row['series_id']))\n    if os.path.exists(series_path):\n        return [\n            os.path.join(series_path, f)\n            for f in os.listdir(series_path)\n            if os.path.isfile(os.path.join(series_path, f))\n        ]\n    return []\n\n# Mapping of series_description to conditions\ncondition_mapping = {\n    'Sagittal T1': {\n        'left': 'left_neural_foraminal_narrowing',\n        'right': 'right_neural_foraminal_narrowing'\n    },\n    'Axial T2': {\n        'left': 'left_subarticular_stenosis',\n        'right': 'right_subarticular_stenosis'\n    },\n    'Sagittal T2/STIR': 'spinal_canal_stenosis'\n}\n\n# Expand the test descriptions by adding new rows for each image path and condition\nexpanded_rows = []\n\nfor index, row in test_desc.iterrows():\n    image_paths = get_image_paths(row)\n    conditions = condition_mapping.get(row['series_description'], {})\n\n    # Handle single or multiple conditions\n    if isinstance(conditions, str):\n        conditions = {'left': conditions, 'right': conditions}\n\n    for side, condition in conditions.items():\n        for image_path in image_paths:\n            expanded_rows.append({\n                'study_id': row['study_id'],\n                'series_id': row['series_id'],\n                'series_description': row['series_description'],\n                'image_path': image_path,\n                'condition': condition,\n                'row_id': f\"{row['study_id']}_{condition}\"\n            })\n\n# Create a new DataFrame from the expanded rows\nexpanded_test_desc = pd.DataFrame(expanded_rows)\n\n# Display the first few rows of the expanded test descriptions\nprint(\"\\nExpanded Test Descriptions:\")\nprint(expanded_test_desc.head(5))\n\n# Update Severity Labels\n# -----------------------\n# Map severity labels to simplified categories\nfinal_merged_df['severity'] = final_merged_df['severity'].map({\n    'Normal/Mild': 'normal_mild',\n    'Moderate': 'moderate',\n    'Severe': 'severe'\n})\n\n# Assign train and test data\ntrain_data = final_merged_df\ntest_data = expanded_test_desc\n\n# Display sample data\nprint(\"\\nSample Train Data:\")\nprint(train_data.head(10))\n\nprint(\"\\nSample Test Data:\")\nprint(test_data.head(10))\n\n# Display the shape of the training data\nprint(\"\\nTrain Data Shape:\", train_data.shape)\n\n# Verify File Paths\n# -----------------\ndef check_exists(path):\n    return os.path.exists(path)\n\ndef check_study_id(row):\n    study_id = row['study_id']\n    path = os.path.join(train_path, 'train_images', str(study_id))\n    return check_exists(path)\n\ndef check_series_id(row):\n    study_id = row['study_id']\n    series_id = row['series_id']\n    path = os.path.join(train_path, 'train_images', str(study_id), str(series_id))\n    return check_exists(path)\n\ndef check_image_exists(row):\n    image_path = row['image_path']\n    return check_exists(image_path)\n\n# Apply existence checks to the training data\ntrain_data['study_id_exists'] = train_data.apply(check_study_id, axis=1)\ntrain_data['series_id_exists'] = train_data.apply(check_series_id, axis=1)\ntrain_data['image_exists'] = train_data.apply(check_image_exists, axis=1)\n\n# Filter training data to include only existing paths\ntrain_data = train_data[\n    train_data['study_id_exists'] &\n    train_data['series_id_exists'] &\n    train_data['image_exists']\n]\nprint(\"\\nTrain Data Shape after Filtering:\", train_data.shape)\n\n# Load and Display Sample Images\n# ------------------------------\ndef load_dicom(path):\n    dicom = pydicom.dcmread(path)\n    data = dicom.pixel_array\n    data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data / np.max(data)  # Normalize the image\n    return (data * 255).astype(np.uint8)\n\nimages = []\nrow_ids = []\nselected_indices = random.sample(range(len(train_data)), 2)\nfor i in selected_indices:\n    image = load_dicom(train_data.iloc[i]['image_path'])\n    images.append(image)\n    row_ids.append(train_data.iloc[i]['row_id'])\n\n# Plot the selected images\nfig, ax = plt.subplots(1, 2, figsize=(8, 4))\nfor i in range(2):\n    ax[i].imshow(images[i], cmap='gray')\n    ax[i].set_title(f'Row ID: {row_ids[i]}', fontsize=8)\n    ax[i].axis('off')\nplt.tight_layout()\nplt.show()\n\n# Remove any rows with missing values\ntrain_data = train_data.dropna()\n\n# Define Visualization Functions\n# -------------------------------\ndef plot_confusion_matrix(cm, classes, title):\n    plt.figure(figsize=(8, 6))\n    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues',\n                xticklabels=classes, yticklabels=classes, cbar=False)\n    plt.title(title)\n    plt.xlabel('Predicted Label')\n    plt.ylabel('True Label')\n    plt.xticks(rotation=45)\n    plt.yticks(rotation=45)\n    plt.show()\n\nimport cv2\n\n# Data Loading and Preprocessing Functions\n# ----------------------------------------\ndef load_dicom_tf(path):\n    dicom = pydicom.dcmread(path.numpy().decode('utf-8'))\n    data = dicom.pixel_array\n    data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data / np.max(data)\n    return data.astype(np.float32)","metadata":{"execution":{"iopub.execute_input":"2025-05-01T21:36:40.379262Z","iopub.status.busy":"2025-05-01T21:36:40.378821Z","iopub.status.idle":"2025-05-01T21:38:06.905201Z","shell.execute_reply":"2025-05-01T21:38:06.904395Z"},"papermill":{"duration":86.53176,"end_time":"2025-05-01T21:38:06.906521","exception":false,"start_time":"2025-05-01T21:36:40.374761","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#2\nimport tensorflow as tf\nimport cv2\nimport pydicom\nimport numpy as np\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom tensorflow.keras import applications, layers, optimizers, callbacks\n\nfrom tensorflow.keras.applications.efficientnet import preprocess_input\nimport tensorflow as tf\nfrom tensorflow.keras.applications import EfficientNetB2\nfrom tensorflow.keras import layers, models\n# from tensorflow.keras import applications, layers, optimizers, callbacks\n# from sklearn.utils.class_weight import compute_class_weight","metadata":{"execution":{"iopub.execute_input":"2025-05-01T21:38:06.957270Z","iopub.status.busy":"2025-05-01T21:38:06.957015Z","iopub.status.idle":"2025-05-01T21:38:06.965388Z","shell.execute_reply":"2025-05-01T21:38:06.964676Z"},"papermill":{"duration":0.034444,"end_time":"2025-05-01T21:38:06.966475","exception":false,"start_time":"2025-05-01T21:38:06.932031","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#3\n\nclass_counts = [37626, 7950, 3081]\ntotal_samples = sum(class_counts)\nclass_weights = compute_class_weight('balanced', classes=np.array([0,1,2]), y=np.repeat([0,1,2], class_counts))\nclass_weights = dict(enumerate(class_weights))\n# class_weights1 = list(class_weights)\n\ndef focal_loss(y_true, y_pred, alpha=list(class_weights.values()), gamma=2.0):\n    y_true = tf.cast(y_true, tf.int32)\n    ce = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y_true, logits=y_pred)\n    probs = tf.nn.softmax(y_pred, axis=-1)\n    probs = tf.gather(probs, y_true, batch_dims=1)\n    alpha = tf.gather(alpha, y_true)\n    modulating_factor = tf.pow(1.0 - probs, gamma)\n    return tf.reduce_mean(alpha * modulating_factor * ce)\n\ndef apply_clahe(image, clipLimit=2, tileGridSize=(16, 16)):\n    if len(image.shape) == 3 and image.shape[2] == 3:\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n    if image.dtype != np.uint8:\n        image = (image * 255).astype(np.uint8) if np.issubdtype(image.dtype, np.floating) else image.astype(np.uint8)\n    clahe = cv2.createCLAHE(clipLimit=clipLimit, tileGridSize=tileGridSize)\n    return clahe.apply(image)\n\ndef apply_bilateral_filter(image, diameter=5, sigma_color=10, sigma_space=10):\n    if image.dtype == np.float32 or image.dtype == np.float64:\n        if image.max() <= 1.0:\n            sigma_color = sigma_color / 255.0\n        else:\n            image = (image / 255.0).astype(np.float32)\n    elif image.dtype != np.uint8:\n        image = image.astype(np.uint8)\n    return cv2.bilateralFilter(image, diameter, sigma_color, sigma_space)\n\nimport tensorflow as tf\n\ndef augment_image_normalized(image):\n    # image = tf.image.random_flip_left_right(image)\n    # image = tf.image.random_flip_up_down(image)\n    image = tf.image.random_brightness(image, max_delta=5/255.0)\n    image = tf.image.random_contrast(image, lower=0.9, upper=1.1)\n    image_uint8 = tf.image.convert_image_dtype(image, tf.uint8)\n    image_uint8 = tf.image.random_jpeg_quality(image_uint8, 80, 100)\n    image = tf.image.convert_image_dtype(image_uint8, tf.float32)\n\n    return image\n\ndef remove_background(image):\n    if len(image.shape) == 3 and image.shape[2] == 3:\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n    if image.dtype != np.uint8:\n        image = (image * 255).astype(np.uint8) if image.dtype == np.float32 else image.astype(np.uint8)\n    _, mask = cv2.threshold(image, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)\n    return cv2.bitwise_and(image, image, mask=mask)\n\ndef preprocess_image(image, label=None, is_training=False):\n    def opencv_process(img_tensor):\n        img_np = img_tensor.numpy().squeeze(axis=-1)  # Convert to 2D (H, W)\n        img_np = apply_bilateral_filter(img_np)\n        img_np = apply_clahe(img_np)\n        img_np = remove_background(img_np)\n        if is_training:\n           img_np = augment_image_normalized(tf.expand_dims(img_np, axis=-1)).numpy().squeeze(axis=-1)\n        img_np = np.expand_dims(img_np, axis=-1)  # Add channel back (H, W, 1)\n        return img_np.astype(np.float32)\n\n    image = tf.expand_dims(image, axis=-1)\n    image = tf.py_function(opencv_process, [image], tf.float32)\n    image.set_shape([None, None, 1])  # Set dynamic shape\n    image = tf.image.resize(image, [224, 224])\n    image = tf.image.grayscale_to_rgb(image)\n    image = tf.keras.applications.resnet50.preprocess_input(image)\n    return (image, label) if label is not None else image\n\ndef create_dataset(df, batch_size=64, is_test=False, is_training=False):\n    if is_test:\n        def load_wrapper(path):\n            image = tf.py_function(load_dicom_tf, [path], tf.float32)\n            image.set_shape((None, None))\n            return preprocess_image(image)\n\n        dataset = tf.data.Dataset.from_tensor_slices(df['image_path'])\n        dataset = dataset.map(load_wrapper, num_parallel_calls=tf.data.AUTOTUNE)\n    else:\n        def load_wrapper(path, label):\n            # Load DICOM and ensure shape\n            image = tf.py_function(load_dicom_tf, [path], tf.float32)\n            image.set_shape([None, None])  # Shape [H, W]\n            return preprocess_image(image, label, is_training=True)\n\n        labels = df['severity'].map({'normal_mild': 0, 'moderate': 1, 'severe': 2}).astype(np.int32)\n        dataset = tf.data.Dataset.from_tensor_slices((df['image_path'], labels))\n        dataset = dataset.map(load_wrapper, num_parallel_calls=tf.data.AUTOTUNE)\n\n        if is_training:\n            dataset = dataset.apply(tf.data.experimental.rejection_resample(\n                class_func=lambda image, label: label,\n                target_dist=list(class_weights.values()),\n                initial_dist=list(class_weights.values())\n            )).map(lambda resampled_label, original_sample: original_sample)\n\n    dataset = dataset.batch(batch_size).prefetch(tf.data.AUTOTUNE)\n    return dataset","metadata":{"execution":{"iopub.execute_input":"2025-05-01T21:38:07.012000Z","iopub.status.busy":"2025-05-01T21:38:07.011801Z","iopub.status.idle":"2025-05-01T21:38:07.037217Z","shell.execute_reply":"2025-05-01T21:38:07.036729Z"},"papermill":{"duration":0.049518,"end_time":"2025-05-01T21:38:07.038266","exception":false,"start_time":"2025-05-01T21:38:06.988748","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#copilot\n\ntf.keras.mixed_precision.set_global_policy('mixed_bfloat16')\n\n\n# Custom weighted loss function\ndef weighted_log_loss(y_true, y_pred):\n    weights = tf.gather([1.0, 2.0, 4.0], tf.cast(y_true, tf.int32))\n    loss = tf.keras.losses.sparse_categorical_crossentropy(y_true, y_pred)\n    return tf.reduce_mean(loss * weights)\n\n\ndef build_efficientnetb2(num_classes=3):\n    base_model = applications.EfficientNetB2(\n        include_top=False,\n        weights='imagenet',\n        input_shape=(224, 224, 3)\n    )\n\n    # فقط بلاک‌های آخر را آموزش‌پذیر بگذار\n    for layer in base_model.layers:\n        if 'block6' in layer.name or 'block7' in layer.name:\n            layer.trainable = True\n        else:\n            layer.trainable = False\n\n    inputs = tf.keras.Input(shape=(224, 224, 3))\n    x = base_model(inputs, training=False)\n    x = layers.GlobalAveragePooling2D()(x)\n    x = layers.Dense(128, activation='relu')(x)  # لایه‌ی واسط اختیاری\n    outputs = layers.Dense(num_classes, activation='linear')(x)\n\n    model = tf.keras.Model(inputs, outputs)\n    return model\n# \n\n# Focal_loss=weighted_log_loss\n\n# Train model\ndef train_model(model, train_dataset, val_dataset, series_name):\n    model.compile(\n        optimizer=optimizers.Adam(0.0001),\n        loss=focal_loss,\n        metrics=['accuracy']\n    )\n\n    callbacks_list = [\n        callbacks.EarlyStopping(patience=10, restore_best_weights=True),\n        callbacks.ModelCheckpoint(f'best_{series_name}.keras', save_best_only=True)\n    ]\n\n    history = model.fit(\n        train_dataset,\n        validation_data=val_dataset,\n        epochs=50,\n        callbacks=callbacks_list\n    )\n\n    return history\n\n# Evaluate model\ndef evaluate_model(model, dataset):\n    y_true = []\n    y_pred = []\n\n    for batch in dataset:\n        images, labels = batch[0], batch[1]\n        y_true.extend(labels.numpy())\n        preds = model.predict(images)\n        y_pred.extend(np.argmax(preds, axis=1))\n\n    y_true = np.array(y_true)\n    y_pred = np.array(y_pred)\n    valid_indices = ~np.isnan(y_true)\n    y_true = y_true[valid_indices]\n    y_pred = y_pred[valid_indices]\n\n    return {\n        'precision': precision_score(y_true, y_pred, average='weighted', zero_division=0),\n        'recall': recall_score(y_true, y_pred, average='weighted', zero_division=0),\n        'f1': f1_score(y_true, y_pred, average='weighted', zero_division=0),\n        'cm': confusion_matrix(y_true, y_pred)\n    }\n\n\nfrom keras.callbacks import EarlyStopping\nfrom sklearn.model_selection import train_test_split\n\n\n# Initialize models for each series\nseries_models = {\n    'Sagittal T1': build_efficientnetb2(),\n    'Axial T2': build_efficientnetb2(),\n    'Sagittal T2/STIR': build_efficientnetb2()\n}\n\nclass_names = ['normal_mild', 'moderate', 'severe']\nresults = {}\n\n# # Ensure final_merged_df is defined correctly\n# if 'final_merged_df' not in globals():\n#     raise NameError(\"final_merged_df is not defined. Please make sure your dataset is loaded correctly.\")\n\nfor series_name, model in series_models.items():\n    series_df = final_merged_df[\n        (final_merged_df['series_description'] == series_name) &\n        (final_merged_df['severity'].isin(class_names))\n    ].copy()\n\n    if series_df.empty:\n        print(f\"Skipping {series_name} - no valid data.\")\n        continue\n\n    # Split dataset\n    train_df, temp_df = train_test_split(\n        series_df,\n        test_size=0.3,\n        stratify=series_df['severity'],\n        random_state=42\n    )\n\n    val_df, test_df = train_test_split(\n        temp_df,\n        test_size=0.6667,\n        stratify=temp_df['severity'],\n        random_state=42\n    )\n\n    # Display class distribution\n    print(f\"\\nClass distribution for {series_name}:\")\n    print(\"Train:\", train_df['severity'].value_counts())\n    print(\"Validation:\", val_df['severity'].value_counts())\n    print(\"Test:\", test_df['severity'].value_counts())\n\n    train_ds = create_dataset(train_df, batch_size=64,is_test=False, is_training=True)\n    val_ds = create_dataset(val_df, batch_size=64 ,is_test=False,)\n    test_ds = create_dataset(test_df, batch_size=64,is_test=False,)\n\n    # Train model\n    history = train_model(model, train_ds, val_ds, series_name)\n\n    # Evaluate model on TEST set\n    result = evaluate_model(model, test_ds)\n    results[series_name] = result\n\n    plot_confusion_matrix(result['cm'], class_names, f'{series_name} Confusion Matrix')\n\n    print(f\"\\nMetrics for {series_name}:\")\n    print(f\"Precision: {result['precision']:.4f}\")\n    print(f\"Recall: {result['recall']:.4f}\")\n    print(f\"F1-Score: {result['f1']:.4f}\")\n\n    plt.figure(figsize=(12, 5))\n\n    plt.subplot(1, 2, 1)\n    plt.plot(history.history['loss'], label='Train Loss')\n    plt.plot(history.history['val_loss'], label='Val Loss')\n    plt.title(f'{series_name} Loss')\n    plt.legend()\n\n    plt.subplot(1, 2, 2)\n    plt.plot(history.history['accuracy'], label='Train Acc')\n    plt.plot(history.history['val_accuracy'], label='Val Acc')\n    plt.title(f'{series_name} Accuracy')\n    plt.legend()\n\n    plt.show()\n","metadata":{"execution":{"iopub.execute_input":"2025-05-01T21:38:07.084034Z","iopub.status.busy":"2025-05-01T21:38:07.083825Z","iopub.status.idle":"2025-05-02T09:33:17.549320Z","shell.execute_reply":"2025-05-02T09:33:17.548560Z"},"papermill":{"duration":42911.329117,"end_time":"2025-05-02T09:33:18.389835","exception":false,"start_time":"2025-05-01T21:38:07.060718","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def apply_clahe(image, clipLimit=2.0, tileGridSize=(16, 16)):\n#     if len(image.shape) == 3 and image.shape[2] == 3:\n#         image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n#     if image.dtype != np.uint8:\n#         image = (image * 255).astype(np.uint8) if np.issubdtype(image.dtype, np.floating) else image.astype(np.uint8)\n#     clahe = cv2.createCLAHE(clipLimit=clipLimit, tileGridSize=tileGridSize)\n#     return clahe.apply(image)\n\n# def apply_bilateral_filter(image, diameter=5, sigma_color=10, sigma_space=10):\n#     if image.dtype == np.float32 or image.dtype == np.float64:\n#         if image.max() <= 1.0:\n#             sigma_color = sigma_color / 255.0\n#         else:\n#             image = (image / 255.0).astype(np.float32)\n#     elif image.dtype != np.uint8:\n#         image = image.astype(np.uint8)\n#     return cv2.bilateralFilter(image, diameter, sigma_color, sigma_space)\n\n# def remove_background(image):\n#     if len(image.shape) == 3 and image.shape[2] == 3:\n#         image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n#     if image.dtype != np.uint8:\n#         image = (image * 255).astype(np.uint8) if image.dtype == np.float32 else image.astype(np.uint8)\n#     _, mask = cv2.threshold(image, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)\n#     return cv2.bitwise_and(image, image, mask=mask)\n\n# def preprocess_image(image, label=None):\n#     def opencv_process(img_tensor):\n#         img_np = img_tensor.numpy().squeeze(axis=-1)  # Convert to 2D (H, W)\n#         img_np = apply_bilateral_filter(img_np, )\n#         img_np = apply_clahe(img_np)\n#         # img_np = remove_background(img_np)\n#         img_np = np.expand_dims(img_np, axis=-1)  # Add channel back (H, W, 1)\n#         return img_np.astype(np.float32)\n\n#     image = tf.expand_dims(image, axis=-1)  # Add channel for processing\n#     image = tf.py_function(opencv_process, [image], tf.float32)\n#     image.set_shape([None, None, 1])  # Set dynamic shape\n#     image = tf.image.resize(image, [224, 224])\n#     image = tf.image.grayscale_to_rgb(image)\n#     image = tf.keras.applications.efficientnetb1.preprocess_input(image)\n#     return (image, label) if label is not None else image\n\n# def create_dataset(df, batch_size=64, is_test=False):\n#     if is_test:\n#         def load_wrapper(path):\n#             image = tf.py_function(load_dicom_tf, [path], tf.float32)\n#             image.set_shape((None, None))\n#             return preprocess_image(image)\n\n#         dataset = tf.data.Dataset.from_tensor_slices(df['image_path'])\n#         dataset = dataset.map(load_wrapper, num_parallel_calls=tf.data.AUTOTUNE)\n#     else:\n#         def load_wrapper(path, label):\n\n#             # Load DICOM and ensure shape\n#             image = tf.py_function(load_dicom_tf, [path], tf.float32)\n#             image.set_shape([None, None])  # Shape [H, W]\n#             return preprocess_image(image, label)\n\n#         labels = df['severity'].map({'normal_mild': 0, 'moderate': 1, 'severe': 2}).astype(np.int32)\n#         dataset = tf.data.Dataset.from_tensor_slices((df['image_path'], labels))\n#         dataset = dataset.map(load_wrapper, num_parallel_calls=tf.data.AUTOTUNE)\n\n#     dataset = dataset.batch(batch_size).prefetch(tf.data.AUTOTUNE)\n#     return dataset","metadata":{"execution":{"iopub.execute_input":"2025-05-02T09:33:21.117355Z","iopub.status.busy":"2025-05-02T09:33:21.116489Z","iopub.status.idle":"2025-05-02T09:33:21.121605Z","shell.execute_reply":"2025-05-02T09:33:21.121066Z"},"id":"rAXmigDylqC-","papermill":{"duration":1.255923,"end_time":"2025-05-02T09:33:21.122759","exception":false,"start_time":"2025-05-02T09:33:19.866836","status":"completed"},"tags":[]},"outputs":[],"execution_count":null}]}