{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":30733,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<h3 id=\"2\" style=\"\n    color: white;\n    display: fill;\n    border-radius: 5px;\n    background-color: #339CFF;\n    font-size: 250%;\n    font-family: Verdana;\n    text-align: center;\n    letter-spacing: 0.5px;\n\">Data reading understanidng</h3>","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\nimport pydicom\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\n\n# Load the datasets\ntrain_df = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv')\ntrain_label_coords_df = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_label_coordinates.csv')\ntrain_series_desc_df = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_series_descriptions.csv')\ntest_series_desc_df = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_series_descriptions.csv')\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-07-10T07:58:16.847155Z","iopub.execute_input":"2024-07-10T07:58:16.847520Z","iopub.status.idle":"2024-07-10T07:58:16.946295Z","shell.execute_reply.started":"2024-07-10T07:58:16.847491Z","shell.execute_reply":"2024-07-10T07:58:16.945507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Basic information about the datasets\nprint(\"\\nInfo for train.csv\")\nprint(train_df.info())\nprint(\"\\nInfo for train_label_coordinates.csv\")\nprint(train_label_coords_df.info())","metadata":{"execution":{"iopub.status.busy":"2024-07-10T07:58:17.521675Z","iopub.execute_input":"2024-07-10T07:58:17.522370Z","iopub.status.idle":"2024-07-10T07:58:17.553767Z","shell.execute_reply.started":"2024-07-10T07:58:17.522341Z","shell.execute_reply":"2024-07-10T07:58:17.552945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the distribution of conditions and severity levels\nplt.figure(figsize=(12, 6))\nsns.countplot(data=train_df.melt(id_vars=['study_id'], var_name='condition', value_name='severity'), \n              y='condition', hue='severity')\nplt.title('Distribution of Conditions and Severity Levels')\nplt.xlabel('Count')\nplt.ylabel('Condition')\nplt.legend(title='Severity')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-10T07:58:18.250305Z","iopub.execute_input":"2024-07-10T07:58:18.251113Z","iopub.status.idle":"2024-07-10T07:58:19.036152Z","shell.execute_reply.started":"2024-07-10T07:58:18.251082Z","shell.execute_reply":"2024-07-10T07:58:19.035360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h3 id=\"2\" style=\"\n    color: white;\n    display: fill;\n    border-radius: 5px;\n    background-color: #339CFF;\n    font-size: 250%;\n    font-family: Verdana;\n    text-align: center;\n    letter-spacing: 0.5px;\n\">Visualization</h3>","metadata":{}},{"cell_type":"markdown","source":"#### Next Steps in EDA\nNow that we have a basic understanding of the data, we'll dive deeper into the analysis with the following steps:\n\n- Detailed Missing Value Analysis: Investigate the patterns and implications of missing values.\n- Condition Severity Analysis: Analyze the severity levels across different conditions and vertebral levels.\n- Coordinates Analysis: Examine the x/y coordinates for the label locations.\n- Correlation Analysis: Identify any correlations between different conditions and severity levels.\n- Visualization Enhancements: Create more detailed visualizations to better understand the relationships in the data.","metadata":{}},{"cell_type":"code","source":"# Detailed missing value analysis\nmissing_values = train_df.isnull().sum()\nmissing_percentage = (missing_values / len(train_df)) * 100\nmissing_data = pd.DataFrame({'Missing Values': missing_values, 'Percentage': missing_percentage})\n\nprint(\"\\nDetailed missing values analysis in train.csv:\")\nprint(missing_data)\n\n# Plot missing values\nplt.figure(figsize=(12, 6))\nsns.barplot(x=missing_data.index, y=missing_data['Percentage'])\nplt.title('Missing Values Percentage in train.csv')\nplt.xticks(rotation=90)\nplt.ylabel('Percentage')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T07:58:19.276036Z","iopub.execute_input":"2024-07-10T07:58:19.276384Z","iopub.status.idle":"2024-07-10T07:58:19.813539Z","shell.execute_reply.started":"2024-07-10T07:58:19.276351Z","shell.execute_reply":"2024-07-10T07:58:19.812672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Distribution of conditions in train.csv\nconditions = train_df.columns[1:]\ncondition_counts = train_df[conditions].apply(pd.Series.value_counts).fillna(0).T\n\n# Distribution of severity levels for each condition\nseverity_counts = train_df[conditions].apply(lambda x: x.value_counts()).fillna(0)\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T07:58:20.420768Z","iopub.execute_input":"2024-07-10T07:58:20.421544Z","iopub.status.idle":"2024-07-10T07:58:20.471547Z","shell.execute_reply.started":"2024-07-10T07:58:20.421512Z","shell.execute_reply":"2024-07-10T07:58:20.470583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Condition severity analysis\nseverity_counts = train_df[conditions].apply(lambda x: x.value_counts()).fillna(0)\n\n# Plot severity distribution for each condition\nseverity_counts.plot(kind='bar', stacked=True, figsize=(12, 6))\nplt.title('Severity Distribution for Each Condition')\nplt.xlabel('Condition')\nplt.ylabel('Count')\nplt.legend(title='Severity Level')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T07:58:20.690253Z","iopub.execute_input":"2024-07-10T07:58:20.690552Z","iopub.status.idle":"2024-07-10T07:58:21.327336Z","shell.execute_reply.started":"2024-07-10T07:58:20.690528Z","shell.execute_reply":"2024-07-10T07:58:21.326455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the distribution of x and y coordinates\nplt.figure(figsize=(12, 6))\nplt.subplot(1, 2, 1)\nsns.histplot(train_label_coords_df['x'], kde=True)\nplt.title('Distribution of x Coordinates')\nplt.xlabel('x Coordinate')\n\nplt.subplot(1, 2, 2)\nsns.histplot(train_label_coords_df['y'], kde=True)\nplt.title('Distribution of y Coordinates')\nplt.xlabel('y Coordinate')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T07:58:22.419368Z","iopub.execute_input":"2024-07-10T07:58:22.419987Z","iopub.status.idle":"2024-07-10T07:58:23.989102Z","shell.execute_reply.started":"2024-07-10T07:58:22.419958Z","shell.execute_reply":"2024-07-10T07:58:23.988074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert severity levels to numeric for correlation analysis\nseverity_mapping = {'Normal/Mild': 1, 'Moderate': 2, 'Severe': 3}\nseverity_numeric_df = train_df.copy()\nfor condition in conditions:\n    severity_numeric_df[condition] = severity_numeric_df[condition].map(severity_mapping)\n\n# Calculate the correlation matrix\ncorrelation_matrix = severity_numeric_df[conditions].corr()\n\n# Plot the correlation matrix\nplt.figure(figsize=(12, 6))\nsns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', linewidths=0.5)\nplt.title('Correlation Matrix of Conditions')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T07:58:23.990668Z","iopub.execute_input":"2024-07-10T07:58:23.990956Z","iopub.status.idle":"2024-07-10T07:58:26.041963Z","shell.execute_reply.started":"2024-07-10T07:58:23.990930Z","shell.execute_reply":"2024-07-10T07:58:26.041075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Enhanced visualization: Distribution of severity levels for each vertebral level\nvertebral_levels = [col.split('_')[-2] + '_' + col.split('_')[-1] for col in conditions]\nseverity_per_level = train_df.melt(id_vars=['study_id'], var_name='condition', value_name='severity')\nseverity_per_level['vertebral_level'] = severity_per_level['condition'].apply(lambda x: '_'.join(x.split('_')[-2:]))\n\nplt.figure(figsize=(12, 6))\nsns.countplot(data=severity_per_level, y='vertebral_level', hue='severity')\nplt.title('Severity Levels Distribution Across Vertebral Levels')\nplt.xlabel('Count')\nplt.ylabel('Vertebral Level')\nplt.legend(title='Severity')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T07:58:26.043310Z","iopub.execute_input":"2024-07-10T07:58:26.043662Z","iopub.status.idle":"2024-07-10T07:58:26.502492Z","shell.execute_reply.started":"2024-07-10T07:58:26.043632Z","shell.execute_reply":"2024-07-10T07:58:26.501545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h3 id=\"2\" style=\"\n    color: white;\n    display: fill;\n    border-radius: 5px;\n    background-color: #339CFF;\n    font-size: 250%;\n    font-family: Verdana;\n    text-align: center;\n    letter-spacing: 0.5px;\n\">Data Preprocessing and Feature Engineering</h3>\n\n","metadata":{}},{"cell_type":"markdown","source":"### Step 1: Handling Missing Values","metadata":{}},{"cell_type":"code","source":"# For simplicity, we will fill missing severity levels with 'Normal/Mild'\ntrain_df.fillna('Normal/Mild', inplace=True)\n\n# Verify that there are no missing values\nprint(\"\\nMissing values in train.csv after filling:\")\nprint(train_df.isnull().sum())\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T07:58:26.504482Z","iopub.execute_input":"2024-07-10T07:58:26.505140Z","iopub.status.idle":"2024-07-10T07:58:26.524455Z","shell.execute_reply.started":"2024-07-10T07:58:26.505103Z","shell.execute_reply":"2024-07-10T07:58:26.523588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Step 2: Encoding Categorical Variables","metadata":{}},{"cell_type":"code","source":"# Encode severity levels to numeric\nseverity_mapping = {'Normal/Mild': 1, 'Moderate': 2, 'Severe': 3}\nfor condition in conditions:\n    train_df[condition] = train_df[condition].map(severity_mapping)\n\n# Verify the encoding\nprint(\"\\nEncoded severity levels in train.csv:\")\nprint(train_df.head())\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T07:58:28.490030Z","iopub.execute_input":"2024-07-10T07:58:28.490871Z","iopub.status.idle":"2024-07-10T07:58:28.529059Z","shell.execute_reply.started":"2024-07-10T07:58:28.490839Z","shell.execute_reply":"2024-07-10T07:58:28.528121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Step 3: Feature Engineering","metadata":{}},{"cell_type":"code","source":"# Example: Calculate the average severity level for each study\ntrain_df['average_severity'] = train_df[conditions].mean(axis=1)\n\n# Verify the new feature\nprint(\"\\nNew feature 'average_severity' in train.csv:\")\nprint(train_df[['study_id', 'average_severity']].head())\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T07:58:30.093128Z","iopub.execute_input":"2024-07-10T07:58:30.093507Z","iopub.status.idle":"2024-07-10T07:58:30.107006Z","shell.execute_reply.started":"2024-07-10T07:58:30.093478Z","shell.execute_reply":"2024-07-10T07:58:30.106129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Step 4: Merge Coordinates Data","metadata":{}},{"cell_type":"code","source":"# Merge the coordinates data with the main training data\nmerged_df = pd.merge(train_df, train_label_coords_df, on='study_id')\n\n# Verify the merged DataFrame\nprint(\"\\nMerged DataFrame:\")\nprint(merged_df.head())\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T07:48:26.779349Z","iopub.execute_input":"2024-07-10T07:48:26.779999Z","iopub.status.idle":"2024-07-10T07:48:26.828273Z","shell.execute_reply.started":"2024-07-10T07:48:26.779971Z","shell.execute_reply":"2024-07-10T07:48:26.827368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Step 5: Data Normalization","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\n# Normalize the x and y coordinates\nscaler = StandardScaler()\nmerged_df[['x', 'y']] = scaler.fit_transform(merged_df[['x', 'y']])\n\n# Verify the normalization\nprint(\"\\nNormalized x and y coordinates:\")\nprint(merged_df[['x', 'y']].head())\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T07:48:55.472959Z","iopub.execute_input":"2024-07-10T07:48:55.473952Z","iopub.status.idle":"2024-07-10T07:48:55.552158Z","shell.execute_reply.started":"2024-07-10T07:48:55.473915Z","shell.execute_reply":"2024-07-10T07:48:55.551222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h3 id=\"2\" style=\"\n    color: white;\n    display: fill;\n    border-radius: 5px;\n    background-color: #339CFF;\n    font-size: 250%;\n    font-family: Verdana;\n    text-align: center;\n    letter-spacing: 0.5px;\n\">Image visuals</h3>","metadata":{}},{"cell_type":"code","source":"import os\nimport pydicom\nimport matplotlib.pyplot as plt\n\n# Function to load and display DICOM images with debug prints\ndef load_and_display_dicom_images(study_id, num_images=5):\n    # Define the directory containing the images\n    study_dir = os.path.join('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images', study_id)\n    image_files = sorted([f for f in os.listdir(study_dir) if f.endswith('.dcm')])[:num_images]\n    \n    \n    plt.figure(figsize=(50, 40))\n    for i, image_file in enumerate(image_files):\n        image_path = os.path.join(study_dir, image_file)\n        #print(f\"Reading DICOM file: {image_path}\")\n        \n        try:\n            dicom_image = pydicom.dcmread(image_path)\n            plt.subplot(1, num_images, i + 1)\n            plt.imshow(dicom_image.pixel_array, cmap='gray')\n            plt.title(f'{study_id} - {image_file}')\n            plt.axis('off')\n        except Exception as e:\n            print(f\"Error reading DICOM file {image_file}: {e}\")\n    \n    plt.tight_layout()\n    plt.show()\n\n# Example usage with a sample study_id\nsample_study_id = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/1012284084'  # replace with an actual study_id from the dataset\n\nload_and_display_dicom_images(sample_study_id)\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T07:59:32.063026Z","iopub.execute_input":"2024-07-10T07:59:32.063651Z","iopub.status.idle":"2024-07-10T07:59:33.356078Z","shell.execute_reply.started":"2024-07-10T07:59:32.063617Z","shell.execute_reply":"2024-07-10T07:59:33.355150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Additional Visualizations: Histogram of Pixel Values","metadata":{}},{"cell_type":"code","source":"def plot_histograms_of_pixel_values(study_id, num_images=5):\n    study_dir = os.path.join('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images', study_id)\n    image_files = sorted([f for f in os.listdir(study_dir) if f.endswith('.dcm')])[:num_images]\n    \n    plt.figure(figsize=(15, 10))\n    for i, image_file in enumerate(image_files):\n        image_path = os.path.join(study_dir, image_file)\n        dicom_image = pydicom.dcmread(image_path)\n        pixel_array = dicom_image.pixel_array\n        \n        plt.subplot(1, num_images, i + 1)\n        plt.hist(pixel_array.ravel(), bins=50, color='gray')\n        plt.title(f'{study_id} - {image_file}')\n        plt.xlabel('Pixel Intensity')\n        plt.ylabel('Frequency')\n    \n    plt.tight_layout()\n    plt.show()\n\n# Example usage with a sample study_id\nsample_study_id = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/1012284084'  # replace with an actual study_id from the dataset'  # replace with an actual study_id from the dataset'  # replace with an actual study_id from the dataset\n\nplot_histograms_of_pixel_values(sample_study_id)\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T08:04:17.860091Z","iopub.execute_input":"2024-07-10T08:04:17.860552Z","iopub.status.idle":"2024-07-10T08:04:19.618400Z","shell.execute_reply.started":"2024-07-10T08:04:17.860521Z","shell.execute_reply":"2024-07-10T08:04:19.617369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\n# Load the datasets\ntrain_df = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv')\ntrain_label_coords_df = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_label_coordinates.csv')\n\n# Convert severity labels to numeric values\nseverity_mapping = {'Normal/Mild': 1, 'Moderate': 2, 'Severe': 3}\nconditions = train_df.columns[1:]\nfor condition in conditions:\n    train_df[condition] = train_df[condition].map(severity_mapping)\n\n# Merge the datasets on 'study_id'\nmerged_df = pd.merge(train_label_coords_df, train_df, on='study_id')\n\n# Display the first few rows of the merged DataFrame\nmerged_df.head()\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T08:31:38.376492Z","iopub.execute_input":"2024-07-10T08:31:38.377142Z","iopub.status.idle":"2024-07-10T08:31:38.523665Z","shell.execute_reply.started":"2024-07-10T08:31:38.377099Z","shell.execute_reply":"2024-07-10T08:31:38.522793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Abnormal Images with the conditions","metadata":{}},{"cell_type":"code","source":"# Function to load and display DICOM images with ROI and conditions for abnormal images\ndef display_abnormal_images(merged_df, num_images=5):\n    # Filter study IDs with any abnormal condition (severity > 1)\n    abnormal_studies = merged_df[merged_df[conditions].apply(lambda row: any(v > 1 for v in row), axis=1)]['study_id'].unique()\n    \n    plt.figure(figsize=(60, 40))\n    count = 0\n    for study_id in abnormal_studies:\n        if count >= num_images:\n            break\n        study_id_str = str(study_id)\n        study_dir = os.path.join('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images', study_id_str)\n        series_dirs = [os.path.join(study_dir, d) for d in os.listdir(study_dir) if os.path.isdir(os.path.join(study_dir, d))]\n        for series_dir in series_dirs:\n            image_files = sorted([f for f in os.listdir(series_dir) if f.endswith('.dcm')])\n            if not image_files:\n                continue\n\n            image_path = os.path.join(series_dir, image_files[0])\n            try:\n                dicom_image = pydicom.dcmread(image_path)\n                plt.subplot(1, num_images, count + 1)\n                plt.imshow(dicom_image.pixel_array, cmap='gray')\n                plt.title(f'{study_id_str} - {os.path.basename(series_dir)} - {image_files[0]}')\n                plt.axis('off')\n\n                # Highlight regions of interest\n                coords_df = merged_df[merged_df['study_id'] == study_id]\n                instance_number = int(image_files[0].split('.')[0])  # assuming the image file name is the instance number\n                roi = coords_df[coords_df['instance_number'] == instance_number]\n                for _, row in roi.iterrows():\n                    rect = patches.Rectangle((row['x']-20, row['y']-20), 40, 40, linewidth=2, edgecolor='r', facecolor='none')\n                    plt.gca().add_patch(rect)\n                \n                # Display condition\n                condition_text = \"\\n\".join([f\"{cond}: {row[cond]}\" for cond in conditions])\n                plt.text(30, 20, condition_text, color='yellow', fontsize=12, backgroundcolor='black')\n\n                count += 1\n                if count >= num_images:\n                    break\n            except Exception as e:\n                continue\n    \n    plt.tight_layout()\n    plt.show()\n\n# Display abnormal images\ndisplay_abnormal_images(merged_df, num_images=5)\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T08:40:22.761875Z","iopub.execute_input":"2024-07-10T08:40:22.762306Z","iopub.status.idle":"2024-07-10T08:41:04.791852Z","shell.execute_reply.started":"2024-07-10T08:40:22.762268Z","shell.execute_reply":"2024-07-10T08:41:04.790499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to load and display DICOM images with ROI and conditions for abnormal images\ndef display_abnormal_images(merged_df, num_images=5):\n    # Filter study IDs with any abnormal condition (severity > 1)\n    abnormal_studies = merged_df[merged_df[conditions].apply(lambda row: any(v > 1 for v in row), axis=1)]['study_id'].unique()\n    \n    plt.figure(figsize=(25, 15))\n    count = 0\n    for study_id in abnormal_studies:\n        if count >= num_images:\n            break\n        study_id_str = str(study_id)\n        study_dir = os.path.join('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images', study_id_str)\n        series_dirs = [os.path.join(study_dir, d) for d in os.listdir(study_dir) if os.path.isdir(os.path.join(study_dir, d))]\n        for series_dir in series_dirs:\n            image_files = sorted([f for f in os.listdir(series_dir) if f.endswith('.dcm')])\n            if not image_files:\n                continue\n\n            image_path = os.path.join(series_dir, image_files[0])\n            try:\n                dicom_image = pydicom.dcmread(image_path)\n                plt.subplot(1, num_images, count + 1)\n                plt.imshow(dicom_image.pixel_array, cmap='gray')\n                plt.title(f'{study_id_str} - {os.path.basename(series_dir)} - {image_files[0]}', fontsize=16)\n                plt.axis('off')\n\n                # Highlight regions of interest\n                coords_df = merged_df[merged_df['study_id'] == study_id]\n                instance_number = int(image_files[0].split('.')[0])  # assuming the image file name is the instance number\n                roi = coords_df[coords_df['instance_number'] == instance_number]\n                for _, row in roi.iterrows():\n                    rect = patches.Rectangle((row['x']-20, row['y']-20), 40, 40, linewidth=2, edgecolor='r', facecolor='none')\n                    plt.gca().add_patch(rect)\n                \n                # Display condition\n                condition_text = \"\\n\".join([f\"{cond}: {row[cond]}\" for cond in conditions])\n                plt.text(10, 10, condition_text, color='yellow', fontsize=14, backgroundcolor='black')\n\n                count += 1\n                if count >= num_images:\n                    break\n            except Exception as e:\n                continue\n    \n    plt.tight_layout()\n    plt.show()\n\n# Display abnormal images\ndisplay_abnormal_images(merged_df, num_images=5)\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T08:42:43.814734Z","iopub.execute_input":"2024-07-10T08:42:43.815099Z","iopub.status.idle":"2024-07-10T08:43:01.621097Z","shell.execute_reply.started":"2024-07-10T08:42:43.815068Z","shell.execute_reply":"2024-07-10T08:43:01.620178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}