{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":11848,"databundleVersionId":862157,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n    #for filename in filenames:\n        #print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-17T16:02:49.306116Z","iopub.execute_input":"2025-06-17T16:02:49.307013Z","iopub.status.idle":"2025-06-17T16:02:49.574501Z","shell.execute_reply.started":"2025-06-17T16:02:49.306982Z","shell.execute_reply":"2025-06-17T16:02:49.573804Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#imports\nimport pathlib as pl\nimport PIL\nfrom tqdm import tqdm\ntqdm.pandas()\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom pathlib import Path\nfrom PIL import Image\nimport math\nimport pandas as pd\nfrom typing import List, Optional, Tuple, Dict\n\nimport time","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T16:02:49.575997Z","iopub.execute_input":"2025-06-17T16:02:49.576264Z","iopub.status.idle":"2025-06-17T16:02:49.584658Z","shell.execute_reply.started":"2025-06-17T16:02:49.576247Z","shell.execute_reply":"2025-06-17T16:02:49.584066Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Histopathological Cancer Detection with CNN","metadata":{}},{"cell_type":"markdown","source":"## 1. EDA","metadata":{}},{"cell_type":"markdown","source":"### Initial Data Assessment","metadata":{}},{"cell_type":"code","source":"data = pl.Path('/kaggle/input/histopathologic-cancer-detection')\ntest = data/'test'\ntrain = data/'train'\ntrain_labels = data/'train_labels.csv'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T16:02:49.585256Z","iopub.execute_input":"2025-06-17T16:02:49.585447Z","iopub.status.idle":"2025-06-17T16:02:49.596728Z","shell.execute_reply.started":"2025-06-17T16:02:49.585422Z","shell.execute_reply":"2025-06-17T16:02:49.596006Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labels_df = pd.read_csv(train_labels)\ntest_list = list(test.glob('*.tif'))\ntest_df = pd.DataFrame({'paths': test_list})\nassert len(labels_df) == 220025\nassert len(test_df) == 57458","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T16:02:49.597383Z","iopub.execute_input":"2025-06-17T16:02:49.597664Z","iopub.status.idle":"2025-06-17T16:02:52.070109Z","shell.execute_reply.started":"2025-06-17T16:02:49.597613Z","shell.execute_reply":"2025-06-17T16:02:52.069543Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labels_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T16:02:52.072205Z","iopub.execute_input":"2025-06-17T16:02:52.072415Z","iopub.status.idle":"2025-06-17T16:02:52.092140Z","shell.execute_reply.started":"2025-06-17T16:02:52.072398Z","shell.execute_reply":"2025-06-17T16:02:52.091378Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def gen_train_paths(row):\n    file = row.id + '.tif'\n    path = train/file\n    #check that all of the training images exist (slow)\n    #assert path.exists()\n    return path\nlabels_df['paths'] = labels_df.progress_apply(gen_train_paths, axis = 1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T16:02:52.092903Z","iopub.execute_input":"2025-06-17T16:02:52.093114Z","iopub.status.idle":"2025-06-17T16:02:54.872622Z","shell.execute_reply.started":"2025-06-17T16:02:52.093091Z","shell.execute_reply":"2025-06-17T16:02:54.871744Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n = labels_df.label.sum()\nN = len(labels_df)\ns = f'There are {n} positive training images and {N-n} negative training images.\\nThis is a slight imbalance in favor of the negative class but unlikely to severely impact training.'\nprint(s)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T16:02:54.873347Z","iopub.execute_input":"2025-06-17T16:02:54.873634Z","iopub.status.idle":"2025-06-17T16:02:54.879912Z","shell.execute_reply.started":"2025-06-17T16:02:54.873603Z","shell.execute_reply":"2025-06-17T16:02:54.878695Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Structure Verification","metadata":{}},{"cell_type":"code","source":"def validate_image_dimensions(data_df, expected_shape=(96, 96, 3), sample_size=None):\n    \"\"\"\n    Validate that all images in the dataset have the expected dimensions.\n    \n    Parameters:\n    - data_df: DataFrame with 'paths' column containing image file paths\n    - expected_shape: Tuple of expected (height, width, channels)\n    - sample_size: Optional integer to check only a sample of images\n    \n    Returns:\n    - Dictionary containing validation results and statistics\n    \"\"\"\n    \n    \n    # Determine which images to check\n    if sample_size and sample_size < len(data_df):\n        check_df = data_df.sample(n=sample_size, random_state=42)\n        print(f\"Checking sample of {sample_size} images from {len(data_df)} total\")\n    else:\n        check_df = data_df\n        print(f\"Checking all {len(check_df)} images\")\n    \n    # Initialize tracking variables\n    correct_shape_count = 0\n    incorrect_shapes = []\n    corrupted_files = []\n    shape_distribution = {}\n    \n    start_time = time.time()\n    \n    # Check each image\n    for idx, row in tqdm(check_df.iterrows(), total=len(check_df), desc=\"Validating images\"):\n        try:\n            img = Image.open(row['paths'])\n            img_array = np.array(img)\n            shape = img_array.shape\n            \n            # Track shape distribution\n            shape_str = str(shape)\n            shape_distribution[shape_str] = shape_distribution.get(shape_str, 0) + 1\n            \n            # Check if shape matches expected\n            if shape == expected_shape:\n                correct_shape_count += 1\n            else:\n                incorrect_shapes.append({\n                    'id': row['id'],\n                    'path': row['paths'],\n                    'actual_shape': shape,\n                    'expected_shape': expected_shape\n                })\n                \n        except Exception as e:\n            corrupted_files.append({\n                'id': row['id'],\n                'path': row['paths'],\n                'error': str(e)\n            })\n    \n    elapsed_time = time.time() - start_time\n    \n    # Compile results\n    results = {\n        'total_checked': len(check_df),\n        'correct_shape_count': correct_shape_count,\n        'incorrect_shape_count': len(incorrect_shapes),\n        'corrupted_count': len(corrupted_files),\n        'shape_distribution': shape_distribution,\n        'incorrect_shapes': incorrect_shapes,\n        'corrupted_files': corrupted_files,\n        'validation_time': f\"{elapsed_time:.2f} seconds\",\n        'all_correct': len(incorrect_shapes) == 0 and len(corrupted_files) == 0\n    }\n    \n    # Print summary\n    print(f\"\\nValidation Summary:\")\n    print(f\"Total images checked: {results['total_checked']}\")\n    print(f\"Correct shape ({expected_shape}): {results['correct_shape_count']} \"\n          f\"({100 * results['correct_shape_count'] / results['total_checked']:.1f}%)\")\n    print(f\"Incorrect shape: {results['incorrect_shape_count']}\")\n    print(f\"Corrupted/unreadable: {results['corrupted_count']}\")\n    print(f\"Time taken: {results['validation_time']}\")\n    \n    if results['incorrect_shape_count'] > 0:\n        print(f\"\\nShape distribution found:\")\n        for shape, count in shape_distribution.items():\n            print(f\"  {shape}: {count} images\")\n    \n    return results","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-06-17T16:02:54.880408Z","iopub.execute_input":"2025-06-17T16:02:54.880616Z","iopub.status.idle":"2025-06-17T16:02:55.028069Z","shell.execute_reply.started":"2025-06-17T16:02:54.880598Z","shell.execute_reply":"2025-06-17T16:02:55.027270Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"validate_image_dimensions(labels_df, expected_shape=(96, 96, 3), sample_size=3000)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T16:02:55.028869Z","iopub.execute_input":"2025-06-17T16:02:55.029059Z","iopub.status.idle":"2025-06-17T16:03:37.169683Z","shell.execute_reply.started":"2025-06-17T16:02:55.029040Z","shell.execute_reply":"2025-06-17T16:03:37.168951Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Visual Inspection","metadata":{}},{"cell_type":"code","source":"def visualize_histopathology_samples(\n    data_df: pd.DataFrame,\n    n: int,\n    figsize_per_image: Tuple[float, float] = (3, 3.5),\n    max_cols: int = 5,\n    title: str = \"Histopathologic Image Samples\",\n    save_path: Optional[Path] = None,\n    dpi: int = 100,\n    random_state: Optional[int] = None\n) -> plt.Figure:\n    \"\"\"\n    Visualize a random sample of histopathology images in a grid layout with class labels.\n    \n    This function creates a professional visualization suitable for exploratory data\n    analysis of the Kaggle histopathologic cancer detection dataset. Each image is\n    displayed with its classification label and filename for easy reference.\n    \n    Parameters\n    ----------\n    data_df : pd.DataFrame\n        DataFrame containing columns 'id', 'label', and 'paths'.\n        - 'id': Image identifier (string)\n        - 'label': Binary label (0 or 1)\n        - 'paths': Path to image file (string or Path object)\n    n : int\n        Number of images to randomly sample and display.\n        If n exceeds the DataFrame size, all images are displayed.\n    figsize_per_image : Tuple[float, float], default=(3, 3.5)\n        Size (width, height) in inches for each image subplot.\n        Includes space for title and labels.\n    max_cols : int, default=5\n        Maximum number of columns in the grid layout.\n        Rows are calculated automatically based on the number of images.\n    title : str, default=\"Histopathologic Image Samples\"\n        Main title for the entire figure.\n    save_path : Path, optional\n        If provided, saves the figure to this path.\n    dpi : int, default=100\n        Dots per inch for saved figure (if save_path is provided).\n    random_state : int, optional\n        Random seed for reproducible sampling.\n    \n    Returns\n    -------\n    matplotlib.figure.Figure\n        The created figure object for further customization if needed.\n    \n    Raises\n    ------\n    ValueError\n        If required columns are missing from the DataFrame.\n        If an image cannot be loaded or is not in the expected format.\n    \n    Examples\n    --------\n    >>> # Load and prepare data\n    >>> labels_df = pd.read_csv('train_labels.csv')\n    >>> train_dir = Path('train')\n    >>> labels_df['paths'] = labels_df['id'].apply(lambda x: train_dir / f'{x}.tif')\n    >>> \n    >>> # Visualize 20 random samples\n    >>> fig = visualize_histopathology_samples(\n    ...     labels_df, \n    ...     n=20,\n    ...     max_cols=4,\n    ...     save_path=Path('eda_sample_images.png')\n    ... )\n    \"\"\"\n    \n    # Validate required columns\n    required_columns = {'id', 'label', 'paths'}\n    if not required_columns.issubset(data_df.columns):\n        missing = required_columns - set(data_df.columns)\n        raise ValueError(f\"Missing required columns: {missing}\")\n    \n    # Sample n rows from the dataframe\n    if len(data_df) <= n:\n        sample_df = data_df.copy()\n    else:\n        sample_df = data_df.sample(n=n, random_state=random_state)\n    \n    # Convert paths column to Path objects if needed\n    sample_df['paths'] = sample_df['paths'].apply(lambda x: Path(x) if not isinstance(x, Path) else x)\n    \n    # Calculate grid dimensions\n    n_images = len(sample_df)\n    n_cols = min(n_images, max_cols)\n    n_rows = math.ceil(n_images / n_cols)\n    \n    # Create figure with calculated size\n    fig_width = n_cols * figsize_per_image[0]\n    fig_height = n_rows * figsize_per_image[1]\n    fig = plt.figure(figsize=(fig_width, fig_height))\n    fig.suptitle(title, fontsize=16, fontweight='bold', y=0.98)\n    \n    # Create grid specification for better spacing control\n    gs = fig.add_gridspec(n_rows, n_cols, hspace=0.3, wspace=0.2, \n                         top=0.94, bottom=0.02, left=0.02, right=0.98)\n    \n    # Process and display each image\n    for idx, (_, row) in enumerate(sample_df.iterrows()):\n        grid_row = idx // n_cols\n        grid_col = idx % n_cols\n        ax = fig.add_subplot(gs[grid_row, grid_col])\n        \n        try:\n            # Load and display image\n            img = Image.open(row['paths'])\n            img_array = np.array(img)\n            \n            # Verify expected image format\n            if img_array.shape != (96, 96, 3):\n                print(f\"Warning: Image {row['id']} has unexpected shape {img_array.shape}\")\n            \n            ax.imshow(img_array)\n            \n            # Get label information from DataFrame row\n            image_id = row['id']\n            label = row['label']\n            \n            # Format label text with appropriate color\n            if label == 1:\n                label_text = \"Positive (Cancer)\"\n                label_color = 'darkred'\n            elif label == 0:\n                label_text = \"Negative (Normal)\"\n                label_color = 'darkgreen'\n            else:\n                label_text = \"Unknown Label\"\n                label_color = 'gray'\n            \n            # Set title with class label\n            ax.set_title(f\"{label_text}\\n{image_id}\", \n                        fontsize=10, pad=5, color=label_color)\n            \n            # Remove axis ticks for cleaner appearance\n            ax.set_xticks([])\n            ax.set_yticks([])\n            \n            # Add subtle border\n            for spine in ax.spines.values():\n                spine.set_edgecolor('gray')\n                spine.set_linewidth(0.5)\n                \n        except Exception as e:\n            # Handle loading errors gracefully\n            ax.text(0.5, 0.5, f\"Error loading\\n{row['id']}\\n{str(e)}\", \n                   ha='center', va='center', transform=ax.transAxes,\n                   fontsize=9, color='red', wrap=True)\n            ax.set_xticks([])\n            ax.set_yticks([])\n    \n    # Remove any empty subplots\n    for idx in range(n_images, n_rows * n_cols):\n        grid_row = idx // n_cols\n        grid_col = idx % n_cols\n        ax = fig.add_subplot(gs[grid_row, grid_col])\n        ax.set_visible(False)\n    \n    # Adjust layout to prevent overlap\n    plt.tight_layout()\n    \n    # Save figure if path provided\n    if save_path:\n        fig.savefig(save_path, dpi=dpi, bbox_inches='tight', \n                   facecolor='white', edgecolor='none')\n        print(f\"Figure saved to: {save_path}\")\n    \n    # Close the figure to prevent duplicate display\n    plt.close(fig)\n    \n    return fig\n\n\ndef create_class_comparison_visualization(\n    data_df: pd.DataFrame,\n    n_per_class: int = 10,\n    figsize_per_image: Tuple[float, float] = (3, 3),\n    save_path: Optional[Path] = None,\n    random_state: Optional[int] = None\n) -> plt.Figure:\n    \"\"\"\n    Create a side-by-side comparison visualization of positive and negative samples.\n    \n    This specialized function arranges images to facilitate visual comparison between\n    cancer-positive and cancer-negative tissue samples, which is particularly useful\n    for understanding class differences during exploratory data analysis.\n    \n    Parameters\n    ----------\n    data_df : pd.DataFrame\n        DataFrame containing columns 'id', 'label', and 'paths'.\n    n_per_class : int, default=10\n        Number of samples to display for each class.\n    figsize_per_image : Tuple[float, float], default=(2.5, 3)\n        Size for each image subplot.\n    save_path : Path, optional\n        Path to save the comparison figure.\n    random_state : int, optional\n        Random seed for reproducible sampling.\n    \n    Returns\n    -------\n    matplotlib.figure.Figure\n        The created comparison figure.\n    \n    Examples\n    --------\n    >>> # Create comparison visualization\n    >>> fig = create_class_comparison_visualization(\n    ...     data_df,\n    ...     n_per_class=8,\n    ...     save_path=Path('class_comparison.png')\n    ... )\n    \"\"\"\n    \n    # Separate positive and negative samples\n    positive_df = data_df[data_df['label'] == 1]\n    negative_df = data_df[data_df['label'] == 0]\n    \n    # Sample from each class\n    if len(positive_df) > n_per_class:\n        positive_sample = positive_df.sample(n=n_per_class, random_state=random_state)\n    else:\n        positive_sample = positive_df\n    \n    if len(negative_df) > n_per_class:\n        negative_sample = negative_df.sample(n=n_per_class, random_state=random_state)\n    else:\n        negative_sample = negative_df\n    \n    # Combine samples alternating between classes for easy comparison\n    combined_df = pd.concat([positive_sample, negative_sample])\n    \n    # Reset index to ensure proper ordering\n    positive_sample = positive_sample.reset_index(drop=True)\n    negative_sample = negative_sample.reset_index(drop=True)\n    \n    # Create interleaved dataframe for side-by-side comparison\n    interleaved_rows = []\n    for i in range(max(len(positive_sample), len(negative_sample))):\n        if i < len(positive_sample):\n            interleaved_rows.append(positive_sample.iloc[i])\n        if i < len(negative_sample):\n            interleaved_rows.append(negative_sample.iloc[i])\n    \n    comparison_df = pd.DataFrame(interleaved_rows)\n    \n    # Create visualization with clear class separation\n    n_total = len(comparison_df)\n    fig = visualize_histopathology_samples(\n        comparison_df,\n        n=n_total,  # Display all samples since we already sampled\n        figsize_per_image=figsize_per_image,\n        max_cols=2,  # Two columns for side-by-side comparison\n        title=f\"Positive vs Negative Sample Comparison (n={min(len(positive_sample), len(negative_sample))} per class)\",\n        save_path=save_path\n    )\n    \n    return fig\n#This code block was generated by Claude Opus 4","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-06-17T16:03:37.170397Z","iopub.execute_input":"2025-06-17T16:03:37.170608Z","iopub.status.idle":"2025-06-17T16:03:37.186886Z","shell.execute_reply.started":"2025-06-17T16:03:37.170585Z","shell.execute_reply":"2025-06-17T16:03:37.185982Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"visualize_histopathology_samples(\n    data_df = labels_df,\n    n = 9,\n    figsize_per_image = (4,4),\n    max_cols = 3,\n    title = \"Histopathologic Image Samples\",\n    save_path = None,\n    dpi = 100,\n    random_state = 42\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T16:03:37.187714Z","iopub.execute_input":"2025-06-17T16:03:37.187959Z","iopub.status.idle":"2025-06-17T16:03:37.681915Z","shell.execute_reply.started":"2025-06-17T16:03:37.187937Z","shell.execute_reply":"2025-06-17T16:03:37.681170Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def detect_monochromatic_images(data_df, variance_threshold=5.0, n_samples=None):\n    \"\"\"\n    Detect images that are essentially monochromatic (single color).\n    \n    Parameters:\n    - data_df: DataFrame containing 'id', 'label', and 'paths' columns\n    - variance_threshold: Maximum standard deviation to consider an image monochromatic\n    - n_samples: Optional number of images to check (None for all)\n    \n    Returns:\n    - DataFrame containing information about monochromatic images\n    \"\"\"\n    import numpy as np\n    from PIL import Image\n    from tqdm import tqdm\n    \n    # Determine sample size\n    if n_samples:\n        check_df = data_df.sample(n=min(n_samples, len(data_df)), random_state=42)\n    else:\n        check_df = data_df\n    \n    monochromatic_images = []\n    \n    for _, row in tqdm(check_df.iterrows(), total=len(check_df), \n                      desc=\"Scanning for monochromatic images\"):\n        try:\n            img = Image.open(row['paths'])\n            img_array = np.array(img)\n            \n            # Calculate statistics for each channel\n            channel_stats = {}\n            for c, channel_name in enumerate(['R', 'G', 'B']):\n                channel_data = img_array[:, :, c]\n                channel_stats[channel_name] = {\n                    'mean': np.mean(channel_data),\n                    'std': np.std(channel_data),\n                    'min': np.min(channel_data),\n                    'max': np.max(channel_data)\n                }\n            \n            # Calculate overall image variance\n            overall_std = np.std(img_array)\n            \n            # Determine if image is monochromatic\n            is_monochromatic = overall_std < variance_threshold\n            \n            # Classify the type of monochromatic image\n            if is_monochromatic:\n                mean_intensity = np.mean(img_array)\n                if mean_intensity < 50:\n                    mono_type = 'dark'\n                elif mean_intensity > 200:\n                    mono_type = 'white'\n                else:\n                    mono_type = 'gray'\n                \n                monochromatic_images.append({\n                    'id': row['id'],\n                    'label': row['label'],\n                    'path': row['paths'],\n                    'overall_std': overall_std,\n                    'mean_intensity': mean_intensity,\n                    'type': mono_type,\n                    'r_std': channel_stats['R']['std'],\n                    'g_std': channel_stats['G']['std'],\n                    'b_std': channel_stats['B']['std']\n                })\n                \n        except Exception as e:\n            print(f\"Error processing {row['id']}: {e}\")\n    \n    # Create results DataFrame\n    results_df = pd.DataFrame(monochromatic_images)\n    \n    # Print summary\n    print(f\"\\nMonochromatic Image Detection Results:\")\n    print(f\"Total images checked: {len(check_df)}\")\n    print(f\"Monochromatic images found: {len(results_df)}\")\n    \n    if len(results_df) > 0:\n        print(f\"\\nBreakdown by type:\")\n        type_counts = results_df['type'].value_counts()\n        for mono_type, count in type_counts.items():\n            print(f\"  {mono_type}: {count} images\")\n        \n        print(f\"\\nLabel distribution of monochromatic images:\")\n        label_counts = results_df['label'].value_counts()\n        for label, count in label_counts.items():\n            print(f\"  Class {label}: {count} images ({100*count/len(results_df):.1f}%)\")\n    \n    return results_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T16:03:37.682753Z","iopub.execute_input":"2025-06-17T16:03:37.682964Z","iopub.status.idle":"2025-06-17T16:03:37.692716Z","shell.execute_reply.started":"2025-06-17T16:03:37.682947Z","shell.execute_reply":"2025-06-17T16:03:37.691965Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mono_df = detect_monochromatic_images(labels_df, variance_threshold=5.0, n_samples=10000)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T16:03:37.693564Z","iopub.execute_input":"2025-06-17T16:03:37.693799Z","iopub.status.idle":"2025-06-17T16:05:17.541052Z","shell.execute_reply.started":"2025-06-17T16:03:37.693775Z","shell.execute_reply":"2025-06-17T16:05:17.540332Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def visualize_monochromatic_analysis(mono_df, data_df, n_examples=6, save_path=None):\n    \"\"\"\n    Visualize monochromatic images and their distribution in the dataset.\n    \"\"\"\n    import matplotlib.pyplot as plt\n    import numpy as np\n    from PIL import Image\n    \n    fig = plt.figure(figsize=(15, 10))\n    \n    # Plot 1: Examples of monochromatic images\n    n_to_show = min(n_examples, len(mono_df))\n    for i in range(n_to_show):\n        ax = plt.subplot(3, n_examples, i + 1)\n        img_info = mono_df.iloc[i]\n        \n        img = Image.open(img_info['path'])\n        ax.imshow(np.array(img))\n        ax.set_title(f\"ID: {img_info['id']}\\nLabel: {img_info['label']}\\n\"\n                    f\"Type: {img_info['type']}\\nStd: {img_info['overall_std']:.2f}\",\n                    fontsize=8)\n        ax.axis('off')\n    \n    # Plot 2: Distribution of monochromatic images by label\n    ax2 = plt.subplot(3, 2, 3)\n    if len(mono_df) > 0:\n        label_counts = mono_df['label'].value_counts()\n        colors = ['darkgreen', 'darkred']\n        bars = ax2.bar(label_counts.index, label_counts.values, \n                       color=[colors[i] for i in label_counts.index])\n        ax2.set_xlabel('Class Label')\n        ax2.set_ylabel('Count')\n        ax2.set_title('Monochromatic Images by Class')\n        ax2.set_xticks([0, 1])\n        ax2.set_xticklabels(['Negative', 'Positive'])\n        \n        # Add percentage labels\n        total_by_class = data_df['label'].value_counts()\n        for i, (label, count) in enumerate(label_counts.items()):\n            percentage = 100 * count / total_by_class[label]\n            ax2.text(i, count + 0.5, f'{percentage:.2f}%', ha='center')\n    \n    # Plot 3: Intensity distribution of monochromatic images\n    ax3 = plt.subplot(3, 2, 4)\n    if len(mono_df) > 0:\n        ax3.hist(mono_df['mean_intensity'], bins=30, edgecolor='black', alpha=0.7)\n        ax3.set_xlabel('Mean Pixel Intensity')\n        ax3.set_ylabel('Count')\n        ax3.set_title('Intensity Distribution of Monochromatic Images')\n        ax3.axvline(x=50, color='red', linestyle='--', label='Dark threshold')\n        ax3.axvline(x=200, color='blue', linestyle='--', label='White threshold')\n        ax3.legend()\n    \n    # Plot 4: Proportion comparison\n    ax4 = plt.subplot(3, 1, 3)\n    total_counts = data_df['label'].value_counts().sort_index()\n    if len(mono_df) > 0:\n        mono_counts = mono_df['label'].value_counts().sort_index()\n        \n        x = np.arange(2)\n        width = 0.35\n        \n        bars1 = ax4.bar(x - width/2, total_counts, width, label='Total Images', alpha=0.7)\n        bars2 = ax4.bar(x + width/2, mono_counts, width, label='Monochromatic', alpha=0.7)\n        \n        ax4.set_xlabel('Class Label')\n        ax4.set_ylabel('Count')\n        ax4.set_title('Comparison of Total vs Monochromatic Images by Class')\n        ax4.set_xticks(x)\n        ax4.set_xticklabels(['Negative', 'Positive'])\n        ax4.legend()\n        ax4.set_yscale('log')  # Log scale to show both clearly\n    \n    plt.tight_layout()\n    \n    if save_path:\n        plt.savefig(save_path, dpi=300, bbox_inches='tight')\n    \n    plt.close()\n    return fig","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-06-17T16:05:17.543744Z","iopub.execute_input":"2025-06-17T16:05:17.543952Z","iopub.status.idle":"2025-06-17T16:05:17.554420Z","shell.execute_reply.started":"2025-06-17T16:05:17.543937Z","shell.execute_reply":"2025-06-17T16:05:17.553819Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"visualize_monochromatic_analysis(mono_df, labels_df, n_examples=8, save_path=None)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T16:05:17.555195Z","iopub.execute_input":"2025-06-17T16:05:17.555684Z","iopub.status.idle":"2025-06-17T16:05:18.766536Z","shell.execute_reply.started":"2025-06-17T16:05:17.555660Z","shell.execute_reply":"2025-06-17T16:05:18.765361Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"create_class_comparison_visualization(labels_df, n_per_class = 4, figsize_per_image = (3.5,3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T16:05:18.767307Z","iopub.execute_input":"2025-06-17T16:05:18.767519Z","iopub.status.idle":"2025-06-17T16:05:19.411517Z","shell.execute_reply.started":"2025-06-17T16:05:18.767502Z","shell.execute_reply":"2025-06-17T16:05:19.410498Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Pixel Intensities","metadata":{}},{"cell_type":"code","source":"def analyze_pixel_distributions(data_df, n_samples=5000, random_state=42):\n    \"\"\"\n    Analyze pixel intensity distributions across classes and color channels.\n    \n    Parameters:\n    - data_df: DataFrame with 'paths' and 'label' columns\n    - n_samples: Number of images to analyze (default 5000 for efficiency)\n    - random_state: Random seed for reproducible sampling\n    \n    Returns:\n    - Dictionary containing computed statistics and data for visualization\n    \"\"\"\n    import numpy as np\n    from PIL import Image\n    from tqdm import tqdm\n    import warnings\n    warnings.filterwarnings('ignore')\n    \n    # Sample data for efficiency\n    sample_df = data_df.sample(n=min(n_samples, len(data_df)), random_state=random_state)\n    \n    # Initialize storage for pixel values by class and channel\n    pixel_data = {\n        0: {'R': [], 'G': [], 'B': [], 'gray': []},\n        1: {'R': [], 'G': [], 'B': [], 'gray': []}\n    }\n    \n    # Collect pixel intensities\n    for _, row in tqdm(sample_df.iterrows(), total=len(sample_df), \n                      desc=\"Analyzing pixel intensities\"):\n        try:\n            img = Image.open(row['paths'])\n            img_array = np.array(img)\n            \n            # Extract each channel\n            pixel_data[row['label']]['R'].extend(img_array[:,:,0].flatten())\n            pixel_data[row['label']]['G'].extend(img_array[:,:,1].flatten())\n            pixel_data[row['label']]['B'].extend(img_array[:,:,2].flatten())\n            \n            # Calculate grayscale for overall intensity\n            gray = np.mean(img_array, axis=2)\n            pixel_data[row['label']]['gray'].extend(gray.flatten())\n            \n        except Exception as e:\n            print(f\"Error processing {row['id']}: {e}\")\n    \n    # Compute statistics\n    statistics = {}\n    for label in [0, 1]:\n        statistics[label] = {}\n        for channel in ['R', 'G', 'B', 'gray']:\n            channel_data = np.array(pixel_data[label][channel])\n            statistics[label][channel] = {\n                'mean': np.mean(channel_data),\n                'std': np.std(channel_data),\n                'median': np.median(channel_data),\n                'q1': np.percentile(channel_data, 25),\n                'q3': np.percentile(channel_data, 75)\n            }\n    \n    return pixel_data, statistics","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T16:05:19.412358Z","iopub.execute_input":"2025-06-17T16:05:19.412607Z","iopub.status.idle":"2025-06-17T16:05:19.421175Z","shell.execute_reply.started":"2025-06-17T16:05:19.412587Z","shell.execute_reply":"2025-06-17T16:05:19.420435Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data, stats = analyze_pixel_distributions(labels_df, n_samples=5000, random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T16:05:19.421911Z","iopub.execute_input":"2025-06-17T16:05:19.422136Z","iopub.status.idle":"2025-06-17T16:05:58.203822Z","shell.execute_reply.started":"2025-06-17T16:05:19.422118Z","shell.execute_reply":"2025-06-17T16:05:58.203150Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def visualize_pixel_distributions(pixel_data, statistics, save_path=None):\n    \"\"\"\n    Create comprehensive visualization of pixel intensity distributions.\n    \"\"\"\n    import matplotlib.pyplot as plt\n    import seaborn as sns\n    \n    # Set up the figure with subplots\n    fig = plt.figure(figsize=(20, 10))\n    \n    # Define colors for consistency\n    colors = {'R': '#E74C3C', 'G': '#27AE60', 'B': '#3498DB', 'gray': '#7F8C8D'}\n    class_names = {0: 'Negative (Normal)', 1: 'Positive (Cancer)'}\n    \n    # Create 2x3 grid: top row for histograms, bottom for box plots\n    for idx, (label, label_name) in enumerate(class_names.items()):\n        # Histogram for RGB channels\n        ax1 = plt.subplot(2, 3, idx*3 + 1)\n        for channel in ['R', 'G', 'B']:\n            # Sample for visualization efficiency\n            sample_pixels = np.random.choice(pixel_data[label][channel], \n                                           size=min(100000, len(pixel_data[label][channel])), \n                                           replace=False)\n            ax1.hist(sample_pixels, bins=50, alpha=0.6, density=True, \n                    label=f\"{channel} (μ={statistics[label][channel]['mean']:.1f})\",\n                    color=colors[channel])\n        \n        ax1.set_title(f'{label_name} - RGB Channel Distributions', fontsize=12, fontweight='bold')\n        ax1.set_xlabel('Pixel Intensity')\n        ax1.set_ylabel('Density')\n        ax1.legend()\n        ax1.grid(True, alpha=0.3)\n        \n        # Box plot comparison\n        ax2 = plt.subplot(2, 3, idx*3 + 2)\n        box_data = []\n        box_labels = []\n        for channel in ['R', 'G', 'B', 'gray']:\n            # Sample for box plot\n            sample_pixels = np.random.choice(pixel_data[label][channel], \n                                           size=min(5000, len(pixel_data[label][channel])), \n                                           replace=False)\n            box_data.append(sample_pixels)\n            box_labels.append(channel)\n        \n        bp = ax2.boxplot(box_data, labels=box_labels, patch_artist=True, showfliers=False)\n        for patch, channel in zip(bp['boxes'], ['R', 'G', 'B', 'gray']):\n            patch.set_facecolor(colors[channel])\n            patch.set_alpha(0.7)\n        \n        ax2.set_title(f'{label_name} - Intensity Distributions by Channel', fontsize=12, fontweight='bold')\n        ax2.set_ylabel('Pixel Intensity')\n        ax2.grid(True, alpha=0.3, axis='y')\n    \n    # Create comparison plot between classes\n    ax3 = plt.subplot(2, 3, 3)\n    for channel in ['R', 'G', 'B']:\n        for label in [0, 1]:\n            sample_pixels = np.random.choice(pixel_data[label][channel], \n                                           size=min(50000, len(pixel_data[label][channel])), \n                                           replace=False)\n            ax3.hist(sample_pixels, bins=50, alpha=0.4, density=True,\n                    label=f\"{class_names[label]} - {channel}\",\n                    color=colors[channel], linestyle='-' if label == 0 else '--')\n    \n    ax3.set_title('Class Comparison - All Channels', fontsize=12, fontweight='bold')\n    ax3.set_xlabel('Pixel Intensity')\n    ax3.set_ylabel('Density')\n    ax3.legend(bbox_to_anchor=(1.05, 1), loc='upper left')\n    ax3.grid(True, alpha=0.3)\n    \n    # Statistical comparison heatmap\n    ax4 = plt.subplot(2, 3, 6)\n    \n    # Prepare data for heatmap\n    diff_matrix = []\n    labels = []\n    for channel in ['R', 'G', 'B', 'gray']:\n        diff_mean = statistics[1][channel]['mean'] - statistics[0][channel]['mean']\n        diff_std = statistics[1][channel]['std'] - statistics[0][channel]['std']\n        diff_matrix.append([diff_mean, diff_std])\n        labels.append(channel)\n    \n    im = ax4.imshow(diff_matrix, cmap='RdBu_r', aspect='auto', vmin=-10, vmax=10)\n    ax4.set_xticks([0, 1])\n    ax4.set_xticklabels(['Mean Diff', 'Std Diff'])\n    ax4.set_yticks(range(len(labels)))\n    ax4.set_yticklabels(labels)\n    ax4.set_title('Statistical Differences (Positive - Negative)', fontsize=12, fontweight='bold')\n    \n    # Add colorbar\n    cbar = plt.colorbar(im, ax=ax4)\n    cbar.set_label('Intensity Difference')\n    \n    # Add values to heatmap\n    for i in range(len(labels)):\n        for j in range(2):\n            text = ax4.text(j, i, f'{diff_matrix[i][j]:.1f}',\n                           ha=\"center\", va=\"center\", color=\"black\", fontsize=10)\n    \n    plt.tight_layout()\n    \n    if save_path:\n        plt.savefig(save_path, dpi=300, bbox_inches='tight')\n        print(f\"Visualization saved to {save_path}\")\n    \n    plt.close()\n    return fig","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-06-17T16:05:58.204735Z","iopub.execute_input":"2025-06-17T16:05:58.204927Z","iopub.status.idle":"2025-06-17T16:06:01.033453Z","shell.execute_reply.started":"2025-06-17T16:05:58.204912Z","shell.execute_reply":"2025-06-17T16:06:01.032700Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"visualize_pixel_distributions(data, stats, save_path=None)","metadata":{"execution":{"iopub.status.busy":"2025-06-17T16:06:01.034277Z","iopub.execute_input":"2025-06-17T16:06:01.034534Z","iopub.status.idle":"2025-06-17T16:06:49.579249Z","shell.execute_reply.started":"2025-06-17T16:06:01.034508Z","shell.execute_reply":"2025-06-17T16:06:49.578613Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Model Specification","metadata":{}},{"cell_type":"code","source":"#!pip install keras-tuner\n#!pip install ipdb\nimport tensorflow as tf\nfrom tensorflow import keras \nimport keras_tuner\nimport ipdb\nfrom keras import Sequential, Input, layers, optimizers, losses\ndef build_model(hp):\n    m = keras.Sequential()\n    m.add(Input(shape = (96,96,3)))\n    \n    crop = hp.Int(\"crop\", min_value = 32, max_value = 96, sampling = 'linear', default = 32)\n    m.add(layers.CenterCrop(crop,crop))\n    #m.add(layers.Lambda(lambda x: tf.cast(x, tf.float32)))\n    #m.add(layers.Rescaling(1/255))\n\n    n_blocks = hp.Int('n_blocks', min_value = 1, max_value = 3, default = 1)\n    init_filters = hp.Int('init_filters', min_value = 16, max_value = 64, sampling = 'log', default = 32)\n    filter_scale = hp.Float('filter_scale', min_value = 1.25, max_value = 2, default = 1.5)\n    n_filters = init_filters\n    padding = hp.Choice('padding', ['valid', 'same'], default = 'same')\n    conv_per = hp.Int('conv_per_block', min_value = 1, max_value = 3, default = 2)\n    batch = hp.Boolean('batch')\n    \n    #ipdb.set_trace()    \n    for i in range(n_blocks):\n        for j in range(conv_per):\n            m.add(layers.Conv2D(filters = n_filters, kernel_size = (3,3), padding = padding))\n            n_filters = int(np.floor(n_filters*filter_scale))\n            \n            if batch:\n                m.add(layers.BatchNormalization())\n\n            m.add(layers.ReLU())\n            \n        m.add(layers.MaxPooling2D(pool_size = (2,2), strides = None, padding = 'valid'))\n\n\n    dropout = hp.Float('dropout', min_value = .4, max_value = .8, default = .6)\n    m.add(layers.Flatten())\n    m.add(layers.Dropout(dropout))\n\n    units = hp.Int('dense_units', min_value = 128, max_value = 256, sampling = 'log', default = 200)\n    m.add(layers.Dense(units = units, activation = 'relu'))\n    m.add(layers.Dropout(dropout))\n\n    m.add(layers.Dense(1, activation = 'sigmoid'))\n\n    smoothing = hp.Float('smoothing', min_value = 1e-8, max_value = 1e-1, sampling = 'log',default = 1e-4)\n    loss = keras.losses.BinaryCrossentropy(\n        from_logits=False,\n        label_smoothing=smoothing,\n        axis=-1,\n        reduction=\"sum_over_batch_size\",\n        name=\"binary_crossentropy\",\n        dtype=None,\n    )\n    \n    lr = hp.Float('learning_rate', min_value = 1e-5, max_value = 1e-3, sampling = 'log', default = 1e-5)\n    opt = hp.Choice('optimizer', ['adam', 'rmsprop'], default = 'adam')\n    \n    rho = 1 - hp.Float('rho', min_value = 1e-2, max_value = 2e-1, sampling = 'log', default = .1)\n    momentum = hp.Float('momentum', min_value = 1e-2, max_value = 5e-1, sampling = 'log', default = 1e-2)\n\n    beta_1 = 1 - hp.Float('beta_1', min_value = 1e-2, max_value = 2e-1, sampling = 'log', default = 1e-1)\n    beta_2 = 1 - hp.Float('beta_2', min_value = 1e-4, max_value = 1e-2, sampling = 'log', default = 1e-3)\n    if opt == 'adam':\n        \n        optimizer = keras.optimizers.Adam(\n            learning_rate=lr,\n            beta_1=beta_1,\n            beta_2=beta_2,\n            epsilon=1e-07,\n            amsgrad=False,\n            weight_decay=None,\n            clipnorm=None,\n            clipvalue=None,\n            global_clipnorm=None,\n            use_ema=False,\n            ema_momentum=0.99,\n            ema_overwrite_frequency=None,\n            loss_scale_factor=None,\n            gradient_accumulation_steps=None,\n            name=\"adam\"\n        )\n    else:\n        \n        optimizer = keras.optimizers.RMSprop(\n            learning_rate=lr,\n            rho=rho,\n            momentum=momentum,\n            epsilon=1e-07,\n            centered=False,\n            weight_decay=None,\n            clipnorm=None,\n            clipvalue=None,\n            global_clipnorm=None,\n            use_ema=False,\n            ema_momentum=0.99,\n            ema_overwrite_frequency=None,\n            loss_scale_factor=None,\n            gradient_accumulation_steps=None,\n            name=\"rmsprop\"\n        )\n    m.compile(\n        optimizer=optimizer,\n        loss=loss,\n        loss_weights=None,\n        metrics=['accuracy', 'AUC'],\n        weighted_metrics=None,\n        run_eagerly=False,\n        steps_per_execution=1,\n        jit_compile=\"auto\",\n        auto_scale_loss=True,\n    )\n        \n    return m\ndefault_model = build_model(keras_tuner.HyperParameters())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T16:19:11.198912Z","iopub.execute_input":"2025-06-17T16:19:11.199231Z","iopub.status.idle":"2025-06-17T16:19:11.290968Z","shell.execute_reply.started":"2025-06-17T16:19:11.199209Z","shell.execute_reply":"2025-06-17T16:19:11.290212Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"early = tf.keras.callbacks.EarlyStopping(monitor = 'val_AUC', \n                                         patience = 2,\n                                        restore_best_weights=True,\n                                        start_from_epoch=4)\nbayes = keras_tuner.BayesianOptimization(\n    hypermodel=build_model,\n    objective='AUC',\n    max_trials=16,\n    num_initial_points=8,\n    alpha=0.0001,\n    beta=2.6,\n    seed=42,\n    hyperparameters=None,\n    tune_new_entries=True,\n    allow_new_entries=True,\n    max_retries_per_trial=1,\n    overwrite =True,\n    max_consecutive_failed_trials=5\n)\nbayes.search_space_summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T16:07:14.524790Z","iopub.execute_input":"2025-06-17T16:07:14.525252Z","iopub.status.idle":"2025-06-17T16:07:14.613577Z","shell.execute_reply.started":"2025-06-17T16:07:14.525220Z","shell.execute_reply":"2025-06-17T16:07:14.613044Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Train Model","metadata":{}},{"cell_type":"markdown","source":"#### Generate Datasets","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.model_selection import train_test_split\nimport pandas as pd\nfrom pathlib import Path\n\ndef create_generators(data_df, validation_size=0.2, batch_size=32, random_state=42):\n    \"\"\"\n    Create training and validation generators from a single DataFrame.\n    \n    Parameters\n    ----------\n    data_df : pd.DataFrame\n        DataFrame containing 'paths' and 'label' columns with all training data\n        The 'paths' column can contain either strings or pathlib Path objects\n        The 'label' column can contain either integers (0, 1) or strings\n    validation_size : float, default=0.2\n        Proportion of data to use for validation (0.0 to 1.0)\n    batch_size : int, default=32\n        Batch size for training and validation\n    random_state : int, default=42\n        Random seed for reproducible train/validation split\n    \n    Returns\n    -------\n    train_generator : DirectoryIterator\n        Generator for training data with augmentation\n    val_generator : DirectoryIterator\n        Generator for validation data without augmentation\n    train_df : pd.DataFrame\n        DataFrame containing training samples (for reference)\n    val_df : pd.DataFrame\n        DataFrame containing validation samples (for reference)\n    \"\"\"\n    \n    # Create a copy to avoid modifying the original DataFrame\n    data_df = data_df.copy()\n    \n    # Convert Path objects to strings if necessary\n    if len(data_df) > 0 and isinstance(data_df['paths'].iloc[0], Path):\n        print(\"Converting Path objects to strings...\")\n        data_df['paths'] = data_df['paths'].astype(str)\n    \n    # Convert integer labels to strings for binary classification\n    if data_df['label'].dtype in ['int64', 'int32', 'int16', 'int8']:\n        print(\"Converting integer labels to strings...\")\n        data_df['label'] = data_df['label'].astype(str)\n    \n    # Perform stratified train-validation split\n    train_df, val_df = train_test_split(\n        data_df,\n        test_size=validation_size,\n        stratify=data_df['label'],\n        random_state=random_state\n    )\n    \n    # Reset indices to avoid issues with flow_from_dataframe\n    train_df = train_df.reset_index(drop=True)\n    val_df = val_df.reset_index(drop=True)\n    \n    # Print split information\n    print(f\"\\nDataset split:\")\n    print(f\"  Total samples: {len(data_df)}\")\n    print(f\"  Training samples: {len(train_df)} ({100*(1-validation_size):.0f}%)\")\n    print(f\"  Validation samples: {len(val_df)} ({100*validation_size:.0f}%)\")\n    print(f\"\\nClass distribution:\")\n    print(f\"  Training: {dict(train_df['label'].value_counts().sort_index())}\")\n    print(f\"  Validation: {dict(val_df['label'].value_counts().sort_index())}\")\n    \n    # Define augmentation for training data\n    train_datagen = ImageDataGenerator(\n        rescale=1./255,\n        rotation_range=90,  # 90-degree rotations for histopathology\n        horizontal_flip=True,\n        vertical_flip=True,\n        brightness_range=[0.8, 1.2],\n        zoom_range=0.1,\n        fill_mode='reflect'\n    )\n    \n    # Validation data should only be rescaled\n    val_datagen = ImageDataGenerator(rescale=1./255)\n    \n    # Create training generator\n    print(f\"\\nCreating generators with batch_size={batch_size}...\")\n    print(\"Note: Skipping file validation for faster initialization...\")\n    \n    train_generator = train_datagen.flow_from_dataframe(\n        dataframe=train_df,\n        x_col='paths',\n        y_col='label',\n        target_size=(96, 96),\n        batch_size=batch_size,\n        class_mode='binary',\n        shuffle=True,\n        seed=42,\n        validate_filenames=False  # Skip file existence validation\n    )\n    \n    # Create validation generator\n    val_generator = val_datagen.flow_from_dataframe(\n        dataframe=val_df,\n        x_col='paths',\n        y_col='label',\n        target_size=(96, 96),\n        batch_size=batch_size,\n        class_mode='binary',\n        shuffle=False,  # Don't shuffle validation data\n        validate_filenames=False  # Skip file existence validation\n    )\n    \n    # Calculate steps per epoch for reference\n    steps_per_epoch = len(train_df) // batch_size\n    validation_steps = len(val_df) // batch_size\n    \n    print(f\"\\nGenerator information:\")\n    print(f\"  Steps per epoch (training): {steps_per_epoch}\")\n    print(f\"  Steps per epoch (validation): {validation_steps}\")\n    print(f\"  Class indices: {train_generator.class_indices}\")\n    \n    return train_generator, val_generator, train_df, val_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T16:07:14.614324Z","iopub.execute_input":"2025-06-17T16:07:14.614581Z","iopub.status.idle":"2025-06-17T16:07:14.625041Z","shell.execute_reply.started":"2025-06-17T16:07:14.614541Z","shell.execute_reply":"2025-06-17T16:07:14.624494Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create generators with default 80/20 split\ntrain_gen, val_gen, train_df, val_df = create_generators(\n    labels_df, \n    validation_size=0.33,\n    batch_size=128\n)\n\n# Verify generator output\nbatch_images, batch_labels = next(train_gen)\nprint(f\"\\nBatch verification:\")\nprint(f\"  Images shape: {batch_images.shape}\")\nprint(f\"  Labels shape: {batch_labels.shape}\")\nprint(f\"  Image value range: [{batch_images.min():.3f}, {batch_images.max():.3f}]\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T16:19:20.402326Z","iopub.execute_input":"2025-06-17T16:19:20.402809Z","iopub.status.idle":"2025-06-17T16:19:21.889345Z","shell.execute_reply.started":"2025-06-17T16:19:20.402785Z","shell.execute_reply":"2025-06-17T16:19:21.888564Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Tune Hyperparameters","metadata":{}},{"cell_type":"code","source":"bayes.search(train_gen, epochs = 10,steps_per_epoch = 100,validation_steps = 50, validation_data = val_gen, callbacks = [early])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T16:30:38.855008Z","iopub.execute_input":"2025-06-17T16:30:38.855287Z","iopub.status.idle":"2025-06-17T19:16:36.266518Z","shell.execute_reply.started":"2025-06-17T16:30:38.855268Z","shell.execute_reply":"2025-06-17T19:16:36.265953Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_model = bayes.get_best_models()[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T19:25:48.207401Z","iopub.execute_input":"2025-06-17T19:25:48.208076Z","iopub.status.idle":"2025-06-17T19:25:52.739168Z","shell.execute_reply.started":"2025-06-17T19:25:48.208051Z","shell.execute_reply":"2025-06-17T19:25:52.738599Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_model.fit(train_gen, epochs = 10,steps_per_epoch = 1000, validation_steps = 100, validation_data = val_gen, callbacks = [early])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T19:26:42.877955Z","iopub.execute_input":"2025-06-17T19:26:42.878514Z","iopub.status.idle":"2025-06-17T20:12:10.248328Z","shell.execute_reply.started":"2025-06-17T19:26:42.878493Z","shell.execute_reply":"2025-06-17T20:12:10.247604Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null}]}