{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":11848,"databundleVersionId":862157,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"##### Histopathologic Cancer Detection - Exploratory Data Analysis\n##### Author: Aaron Storey\n##### Date: December 09, 2024\n##### Version: 1.0\n\nThis notebook performs comprehensive exploratory data analysis on the histopathologic cancer detection dataset.\nThe analysis includes data distribution, image properties, and quality checks to inform model development.\n","metadata":{}},{"cell_type":"markdown","source":"##### 1. Import Required Libraries","metadata":{}},{"cell_type":"code","source":"import os\nfrom typing import Tuple, List, Dict, Optional\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom PIL import Image\nfrom tqdm.notebook import tqdm\nimport cv2\nfrom pathlib import Path\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom sklearn.model_selection import GroupShuffleSplit\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Set random seed for reproducibility\nRANDOM_SEED = 42\nnp.random.seed(RANDOM_SEED)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T16:10:40.395306Z","iopub.execute_input":"2024-12-09T16:10:40.395772Z","iopub.status.idle":"2024-12-09T16:10:42.822044Z","shell.execute_reply.started":"2024-12-09T16:10:40.395735Z","shell.execute_reply":"2024-12-09T16:10:42.820688Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##### 2. Configuration and Setup","metadata":{}},{"cell_type":"code","source":"class Config:\n    \"\"\"Configuration class to store all parameters and paths.\"\"\"\n    \n    def __init__(self):\n        # Data paths\n        self.DATA_DIR = Path('/kaggle/input/histopathologic-cancer-detection')\n        self.TRAIN_DIR = self.DATA_DIR / 'train'\n        self.TEST_DIR = self.DATA_DIR / 'test'\n        self.LABELS_FILE = self.DATA_DIR / 'train_labels.csv'\n        \n        # Analysis parameters\n        self.SAMPLE_SIZE = 1000  # Number of images to analyze for detailed statistics\n        self.IMAGE_SIZE = 96  # Expected image size\n        self.WSI_ID_LENGTH = 3  # Length of Whole Slide Image ID prefix\n        \n        # Visualization parameters\n        self.PLOT_SIZE = (12, 8)\n        self.SAMPLE_GRID_SIZE = (4, 4)\n        \nconfig = Config()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T16:11:21.242560Z","iopub.execute_input":"2024-12-09T16:11:21.243522Z","iopub.status.idle":"2024-12-09T16:11:21.250018Z","shell.execute_reply.started":"2024-12-09T16:11:21.243478Z","shell.execute_reply":"2024-12-09T16:11:21.248716Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##### 3. Data Loading and Initial Inspection","metadata":{}},{"cell_type":"code","source":"def load_data(config: Config) -> Tuple[pd.DataFrame, pd.DataFrame]:\n    \"\"\"\n    Load training labels and generate file paths for images.\n    \n    Args:\n        config: Configuration object containing paths\n        \n    Returns:\n        Tuple containing:\n            - Complete DataFrame with paths and labels\n            - Test DataFrame with paths\n    \"\"\"\n    # Load training labels\n    train_df = pd.read_csv(config.LABELS_FILE)\n    print(f\"Training labels shape: {train_df.shape}\")\n    \n    # Add file paths\n    train_df['path'] = train_df['id'].apply(lambda x: str(config.TRAIN_DIR / f\"{x}.tif\"))\n    \n    # Create WSI (Whole Slide Image) ID\n    train_df['wsi_id'] = train_df['id'].str[:config.WSI_ID_LENGTH]\n    \n    # Get test files\n    test_files = list(config.TEST_DIR.glob('*.tif'))\n    test_df = pd.DataFrame({\n        'id': [f.stem for f in test_files],\n        'path': [str(f) for f in test_files]\n    })\n    \n    return train_df, test_df\n\ntrain_df, test_df = load_data(config)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T16:11:23.600189Z","iopub.execute_input":"2024-12-09T16:11:23.601218Z","iopub.status.idle":"2024-12-09T16:11:26.731822Z","shell.execute_reply.started":"2024-12-09T16:11:23.601170Z","shell.execute_reply":"2024-12-09T16:11:26.730470Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##### 4. Class Distribution Analysis","metadata":{}},{"cell_type":"code","source":"def analyze_class_distribution(df: pd.DataFrame) -> Dict[str, float]:\n    \"\"\"\n    Analyze and visualize class distribution in the dataset.\n    \n    Args:\n        df: DataFrame containing labels\n        \n    Returns:\n        Dictionary containing distribution statistics\n    \"\"\"\n    # Calculate class distribution\n    class_dist = df['label'].value_counts(normalize=True)\n    \n    # Create visualization\n    plt.figure(figsize=config.PLOT_SIZE)\n    sns.countplot(data=df, x='label')\n    plt.title('Distribution of Classes')\n    plt.xlabel('Class (0: Benign, 1: Malignant)')\n    plt.ylabel('Count')\n    plt.show()\n    \n    # Calculate statistics\n    stats = {\n        'total_samples': len(df),\n        'class_0_ratio': class_dist[0],\n        'class_1_ratio': class_dist[1],\n        'imbalance_ratio': class_dist[0] / class_dist[1]\n    }\n    \n    return stats\n\ndistribution_stats = analyze_class_distribution(train_df)\nprint(\"\\nClass Distribution Statistics:\")\nfor k, v in distribution_stats.items():\n    print(f\"{k}: {v:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T16:00:07.797303Z","iopub.execute_input":"2024-12-09T16:00:07.797785Z","iopub.status.idle":"2024-12-09T16:00:08.136168Z","shell.execute_reply.started":"2024-12-09T16:00:07.797721Z","shell.execute_reply":"2024-12-09T16:00:08.135072Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"##### 5. Image Analysis","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T16:00:08.137664Z","iopub.execute_input":"2024-12-09T16:00:08.138008Z","iopub.status.idle":"2024-12-09T16:00:08.142718Z","shell.execute_reply.started":"2024-12-09T16:00:08.137973Z","shell.execute_reply":"2024-12-09T16:00:08.141517Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def analyze_image_properties(df: pd.DataFrame, sample_size: int) -> Dict[str, np.ndarray]:\n    \"\"\"\n    Analyze image properties on a sample of images.\n    \n    Args:\n        df: DataFrame containing image paths\n        sample_size: Number of images to analyze\n        \n    Returns:\n        Dictionary containing image statistics\n    \"\"\"\n    # Sample images\n    sample_df = df.sample(sample_size, random_state=RANDOM_SEED)\n    \n    # Initialize storage\n    pixel_means = []\n    pixel_stds = []\n    aspect_ratios = []\n    \n    for path in tqdm(sample_df['path'], desc=\"Analyzing images\"):\n        img = np.array(Image.open(path))\n        pixel_means.append(img.mean(axis=(0,1)))\n        pixel_stds.append(img.std(axis=(0,1)))\n        aspect_ratios.append(img.shape[1] / img.shape[0])\n    \n    return {\n        'means': np.array(pixel_means),\n        'stds': np.array(pixel_stds),\n        'aspect_ratios': np.array(aspect_ratios)\n    }\n\nimage_stats = analyze_image_properties(train_df, config.SAMPLE_SIZE)\n\n# Visualize statistics\nplt.figure(figsize=config.PLOT_SIZE)\nfor i, color in enumerate(['red', 'green', 'blue']):\n    plt.hist(image_stats['means'][:, i], alpha=0.5, label=color, bins=50)\nplt.title('Distribution of Color Channel Means')\nplt.xlabel('Pixel Value')\nplt.ylabel('Count')\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T16:00:08.144179Z","iopub.execute_input":"2024-12-09T16:00:08.144598Z","iopub.status.idle":"2024-12-09T16:00:24.861762Z","shell.execute_reply.started":"2024-12-09T16:00:08.144563Z","shell.execute_reply":"2024-12-09T16:00:24.860540Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##### 6. WSI Group Analysis","metadata":{}},{"cell_type":"code","source":"def analyze_wsi_groups(df: pd.DataFrame) -> pd.DataFrame:\n    \"\"\"\n    Analyze distribution of samples across Whole Slide Images.\n    \n    Args:\n        df: DataFrame containing WSI IDs and labels\n        \n    Returns:\n        DataFrame with WSI statistics\n    \"\"\"\n    # Group by WSI ID\n    wsi_stats = df.groupby('wsi_id').agg({\n        'label': ['count', 'mean']\n    }).reset_index()\n    \n    wsi_stats.columns = ['wsi_id', 'sample_count', 'malignant_ratio']\n    \n    # Visualize distribution\n    plt.figure(figsize=config.PLOT_SIZE)\n    plt.scatter(wsi_stats['sample_count'], wsi_stats['malignant_ratio'], alpha=0.5)\n    plt.title('WSI Sample Count vs Malignant Ratio')\n    plt.xlabel('Number of Samples')\n    plt.ylabel('Ratio of Malignant Samples')\n    plt.show()\n    \n    return wsi_stats\n\nwsi_stats = analyze_wsi_groups(train_df)\nprint(\"\\nWSI Group Statistics:\")\nprint(wsi_stats.describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T16:00:24.863247Z","iopub.execute_input":"2024-12-09T16:00:24.863746Z","iopub.status.idle":"2024-12-09T16:00:25.235378Z","shell.execute_reply.started":"2024-12-09T16:00:24.863701Z","shell.execute_reply":"2024-12-09T16:00:25.234269Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##### 7. Sample Image Visualization","metadata":{}},{"cell_type":"code","source":"def visualize_samples(df: pd.DataFrame, grid_size: Tuple[int, int],\n                     random_state: Optional[int] = None) -> None:\n    \"\"\"\n    Visualize a grid of sample images from each class.\n    \n    Args:\n        df: DataFrame containing image paths and labels\n        grid_size: Tuple of (rows, cols) for the visualization grid\n        random_state: Random seed for reproducibility\n    \"\"\"\n    rows, cols = grid_size\n    total_samples = rows * cols\n    fig, axes = plt.subplots(rows, cols, figsize=(15, 15))\n    axes = axes.ravel()  # Flatten the axes array for easier indexing\n    \n    # Get equal samples from each class\n    samples_per_class = total_samples // 2\n    \n    # Get samples for benign (class 0)\n    benign_samples = df[df['label'] == 0].sample(samples_per_class, random_state=random_state)\n    # Get samples for malignant (class 1)\n    malignant_samples = df[df['label'] == 1].sample(samples_per_class, random_state=random_state)\n    \n    # Combine samples\n    samples = pd.concat([benign_samples, malignant_samples])\n    \n    # Plot each image\n    for idx, (_, row) in enumerate(samples.iterrows()):\n        img = np.array(Image.open(row['path']))\n        axes[idx].imshow(img)\n        axes[idx].axis('off')\n        axes[idx].set_title(f\"Class {row['label']}\")\n    \n    plt.tight_layout()\n    plt.show()\n\nvisualize_samples(train_df, config.SAMPLE_GRID_SIZE, RANDOM_SEED)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T16:00:25.236713Z","iopub.execute_input":"2024-12-09T16:00:25.237046Z","iopub.status.idle":"2024-12-09T16:00:26.913608Z","shell.execute_reply.started":"2024-12-09T16:00:25.237013Z","shell.execute_reply":"2024-12-09T16:00:26.912335Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##### 8. Train-Validation Split Analysis\n","metadata":{}},{"cell_type":"code","source":"def analyze_train_val_split(df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame]:\n    \"\"\"\n    Analyze and perform train-validation split using WSI groups.\n    \n    Args:\n        df: DataFrame to split\n        \n    Returns:\n        Tuple of (train_df, val_df)\n    \"\"\"\n    # Perform split\n    splitter = GroupShuffleSplit(n_splits=1, test_size=0.2, random_state=RANDOM_SEED)\n    train_idx, val_idx = next(splitter.split(df, groups=df['wsi_id']))\n    \n    train_data = df.iloc[train_idx]\n    val_data = df.iloc[val_idx]\n    \n    # Analyze split statistics\n    print(\"\\nSplit Statistics:\")\n    print(f\"Training samples: {len(train_data)}\")\n    print(f\"Validation samples: {len(val_data)}\")\n    print(\"\\nClass Distribution:\")\n    print(\"Training:\")\n    print(train_data['label'].value_counts(normalize=True))\n    print(\"\\nValidation:\")\n    print(val_data['label'].value_counts(normalize=True))\n    \n    return train_data, val_data\n\ntrain_data, val_data = analyze_train_val_split(train_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T16:11:33.883774Z","iopub.execute_input":"2024-12-09T16:11:33.884160Z","iopub.status.idle":"2024-12-09T16:11:34.274911Z","shell.execute_reply.started":"2024-12-09T16:11:33.884124Z","shell.execute_reply":"2024-12-09T16:11:34.273754Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##### 9. Save Processed Data","metadata":{}},{"cell_type":"code","source":"def save_processed_data(train_data: pd.DataFrame, val_data: pd.DataFrame,\n                       test_df: pd.DataFrame) -> None:\n    \"\"\"\n    Save processed DataFrames for use in training.\n    \n    Args:\n        train_data: Training DataFrame\n        val_data: Validation DataFrame\n        test_df: Test DataFrame\n    \"\"\"\n    # Save DataFrames\n    train_data.to_csv('train_processed.csv', index=False)\n    val_data.to_csv('val_processed.csv', index=False)\n    test_df.to_csv('test_processed.csv', index=False)\n    \n    print(\"Processed data saved successfully.\")\n\nsave_processed_data(train_data, val_data, test_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T16:05:38.136362Z","iopub.execute_input":"2024-12-09T16:05:38.136758Z","iopub.status.idle":"2024-12-09T16:05:39.461000Z","shell.execute_reply.started":"2024-12-09T16:05:38.136718Z","shell.execute_reply":"2024-12-09T16:05:39.459598Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##### 10. Summary Statistics and Recommendations","metadata":{}},{"cell_type":"code","source":"print(\"\\nEDA Summary and Recommendations:\")\nprint(\"\\n1. Dataset Composition:\")\nprint(f\"- Total training samples: {len(train_df)}\")\nprint(f\"- Class imbalance ratio: {distribution_stats['imbalance_ratio']:.2f}\")\nprint(f\"- Number of unique WSIs: {len(train_df['wsi_id'].unique())}\")\n\nprint(\"\\n2. Image Properties:\")\nprint(f\"- Mean pixel values (RGB): {image_stats['means'].mean(axis=0)}\")\nprint(f\"- Std pixel values (RGB): {image_stats['stds'].mean(axis=0)}\")\n\nprint(\"\\n3. Recommendations for Training:\")\nprint(\"- Implement class weighting or balanced sampling\")\nprint(\"- Use WSI-based stratification for validation\")\nprint(\"- Apply standardization using calculated means and stds\")\nprint(\"- Monitor for potential data leakage across WSIs\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-09T16:05:42.866092Z","iopub.execute_input":"2024-12-09T16:05:42.867136Z","iopub.status.idle":"2024-12-09T16:05:42.891019Z","shell.execute_reply.started":"2024-12-09T16:05:42.867091Z","shell.execute_reply":"2024-12-09T16:05:42.889639Z"}},"outputs":[],"execution_count":null}]}