{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":10338,"databundleVersionId":862042,"sourceType":"competition"},{"sourceId":11713291,"sourceType":"datasetVersion","datasetId":7352392}],"dockerImageVersionId":31011,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pydicom\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom keras.models import Sequential\nfrom keras.layers import Conv2D, MaxPooling2D, Flatten, Dense\nfrom keras.applications import MobileNetV2\nfrom keras.optimizers import Adam\nfrom keras import mixed_precision\n\n# Use mixed precision\npolicy = mixed_precision.Policy('mixed_float16')\nmixed_precision.set_global_policy(policy)\n\n# Read DICOM file\ndef load_dicom_image(file_path):\n ds = pydicom.dcmread(file_path)\n img = ds.pixel_array\n img = img.astype(np.float32)\n img = (img - np.min(img)) / (np.max(img) - np.min(img)) # Normalize\n return img\n\n# Read label data\nlabels_df = pd.read_csv('/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv')\ndetailed_info_df = pd.read_csv('/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv')\n\n# Select target columns and merge information\nlabels_df = labels_df[['patientId', 'Target']]\ndetailed_info_df = detailed_info_df[['patientId', 'class']]\nmerged_df = pd.merge(labels_df, detailed_info_df, on='patientId', how='left')\n\n# Read images and labels\nimages = []\ntargets = []\n\nfor index, row in merged_df.iterrows():\n image_path = f'/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images/{row[\"patientId\"]}.dcm'\n try:\n img = load_dicom_image(image_path)\n images.append(img)\n targets.append(row['Target'])\n \n except Exception as e:\n print(f\"Error loading {image_path}: {e}\")\n\n# Convert to NumPy arrays and reshape\nX = np.array(images).reshape(-1, images[0].shape[0], images[0].shape[1], 1)\ny = np.array(targets)\n\n# Split dataset\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Build a lightweight convolutional neural network model\nbase_model = MobileNetV2(input_shape=(X.shape[1], X.shape[2], 1), include_top=False, weights=None)\nbase_model.trainable = False # Freeze layers of the base model\n\nmodel = Sequential()\nmodel.add(base_model)\nmodel.add(Flatten())\nmodel.add(Dense(64, activation='relu'))\nmodel.add(Dense(1, activation='sigmoid')) # Binary classification\n\n# Compile model\noptimizer = Adam(learning_rate=0.001)\nmodel.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])\n\n# Train model\nmodel.fit(X_train, y_train, epochs=10, batch_size=16, validation_data=(X_test, y_test))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T07:31:30.396834Z","iopub.execute_input":"2025-05-10T07:31:30.397512Z","execution_failed":"2025-05-10T07:33:21.54Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!python -V","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T07:57:08.98964Z","iopub.execute_input":"2025-05-10T07:57:08.989893Z","iopub.status.idle":"2025-05-10T07:57:09.119274Z","shell.execute_reply.started":"2025-05-10T07:57:08.989875Z","shell.execute_reply":"2025-05-10T07:57:09.118373Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Replace it with a feature map","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.layers import Flatten, Dense\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.preprocessing.image import load_img, img_to_array\n\n# Read label data\nlabels_df = pd.read_csv('/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv')\ndetailed_info_df = pd.read_csv('/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv')\n\n# Select target columns and merge information\nlabels_df = labels_df[['patientId', 'Target']]\ndetailed_info_df = detailed_info_df[['patientId', 'class']]\nmerged_df = pd.merge(labels_df, detailed_info_df, on='patientId', how='left')\n\n# Read images and labels\nimages = []\ntargets = []\n\n# Image directory path\nimage_dir = '/kaggle/input/rsna-feature/'\n\nfor index, row in merged_df.iterrows():\n image_filename = f\"{row['patientId']}_feature1.png\"\n image_path = os.path.join(image_dir, image_filename)\n try:\n img = load_img(image_path, target_size=(224, 224), color_mode='rgb') # Resize the image\n img = img_to_array(img) / 255.0 # Normalize\n images.append(img)\n targets.append(row['Target'])\n except Exception as e:\n print(f\"Error loading {image_path}: {e}\")\n\n# Convert to NumPy arrays\nX = np.array(images)\ny = np.array(targets)\n\n# Split dataset\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Build a lightweight convolutional neural network model\nbase_model = MobileNetV2(input_shape=(224, 224, 3), include_top=False, weights=None)\nbase_model.trainable = False # Freeze layers of the base model\n\nmodel = Sequential()\nmodel.add(base_model)\nmodel.add(Flatten())\nmodel.add(Dense(64, activation='relu'))\nmodel.add(Dense(1, activation='sigmoid')) # Binary classification\n\n# Compile model\noptimizer = Adam(learning_rate=0.001)\nmodel.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])\n\n# Train model\nmodel.fit(X_train, y_train, epochs=10, batch_size=16, validation_data=(X_test, y_test))","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport glob\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.layers import Flatten, Dense\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.preprocessing.image import load_img, img_to_array\n\n# Read label data\nlabels_df = pd.read_csv('/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv')\ndetailed_info_df = pd.read_csv('/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv')\n\n# Select target columns and merge information\nlabels_df = labels_df[['patientId', 'Target']]\ndetailed_info_df = detailed_info_df[['patientId', 'class']]\nmerged_df = pd.merge(labels_df, detailed_info_df, on='patientId', how='left')\n\n# Read images and labels\nimages = []\ntargets = []\n\n# Image directory path\nimage_dir = '/kaggle/input/rsna-feature/'\n\nfor index, row in merged_df.iterrows():\n patient_id = row['patientId']\n # Find all feature images\n image_files = glob.glob(os.path.join(image_dir, f\"{patient_id}_feature*.png\"))\n \n for image_path in image_files:\n try:\n img = load_img(image_path, color_mode='rgb') # Load image\n img = img_to_array(img) / 255.0 # Normalize\n images.append(img)\n targets.append(row['Target'])\n break # Exit loop after successful loading\n except Exception as e:\n print(f\"Error loading {image_path}: {e}\")\n\n# Convert to NumPy arrays\nX = np.array(images)\ny = np.array(targets)\n\n# Split dataset\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Build a lightweight convolutional neural network model\nbase_model = MobileNetV2(input_shape=(224, 224, 3), include_top=False, weights=None)\nbase_model.trainable = False # Freeze layers of the base model\n\nmodel = Sequential()\nmodel.add(base_model)\nmodel.add(Flatten())\nmodel.add(Dense(64, activation='relu'))\nmodel.add(Dense(1, activation='sigmoid')) # Binary classification\n\n# Compile model\noptimizer = Adam(learning_rate=0.001)\nmodel.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])\n\n# Train model\nmodel.fit(X_train, y_train, epochs=10, batch_size=16, validation_data=(X_test, y_test))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T07:48:27.035386Z","iopub.execute_input":"2025-05-10T07:48:27.036173Z","execution_failed":"2025-05-10T07:49:54.017Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Train only 100 pictures","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport glob\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.layers import Flatten, Dense\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.preprocessing.image import load_img, img_to_array\n\n# Read label data\nlabels_df = pd.read_csv('/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv')\ndetailed_info_df = pd.read_csv('/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv')\n\n# Select target columns and merge information\nlabels_df = labels_df[['patientId', 'Target']]\ndetailed_info_df = detailed_info_df[['patientId', 'class']]\nmerged_df = pd.merge(labels_df, detailed_info_df, on='patientId', how='left')\n\n# Read images and labels\nimages = []\ntargets = []\n\n# Image directory path\nimage_dir = '/kaggle/input/rsna-feature/'\n\n# Counter\ncount = 0\nmax_images = 100 # Limit the number of images to 100\n\nfor index, row in merged_df.iterrows():\n if count >= max_images:\n break # Exit loop after reaching the maximum image count\n \n patient_id = row['patientId']\n # Find all feature images\n image_files = glob.glob(os.path.join(image_dir, f\"{patient_id}_feature*.png\"))\n \n for image_path in image_files:\n try:\n img = load_img(image_path, target_size=(224, 224), color_mode='rgb') # Resize the image\n img = img_to_array(img) / 255.0 # Normalize\n images.append(img)\n targets.append(row['Target'])\n count += 1 # Increment counter\n break # Exit loop after successful loading\n except Exception as e:\n print(f\"Error loading {image_path}: {e}\")\n\n# Convert to NumPy arrays\nX = np.array(images)\ny = np.array(targets)\n\n# Split dataset\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Build a lightweight convolutional neural network model\nbase_model = MobileNetV2(input_shape=(224, 224, 3), include_top=False, weights=None)\nbase_model.trainable = False # Freeze layers of the base model\n\nmodel = Sequential()\nmodel.add(base_model)\nmodel.add(Flatten())\nmodel.add(Dense(64, activation='relu'))\nmodel.add(Dense(1, activation='sigmoid')) # Binary classification\n\n# Compile model\noptimizer = Adam(learning_rate=0.001)\nmodel.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])\n\n# Train model\nmodel.fit(X_train, y_train, epochs=10, batch_size=16, validation_data=(X_test, y_test))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T07:51:05.09859Z","iopub.execute_input":"2025-05-10T07:51:05.098849Z","iopub.status.idle":"2025-05-10T07:51:42.905837Z","shell.execute_reply.started":"2025-05-10T07:51:05.098823Z","shell.execute_reply":"2025-05-10T07:51:42.90523Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Do not resize images\n# -*- coding: utf-8 -*-\n\"\"\"\nVariable size image training example:\n- No scaling or padding of images\n- Only take the first 100 images\n- Use MobileNetV2 (dynamic input) + GAP\n- Manually train sample by sample without batching\n\"\"\"\nimport numpy as np\nimport pandas as pd\nimport os\nimport glob\nimport tensorflow as tf\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dense\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.preprocessing.image import load_img, img_to_array\n\n# Configuration\nMAX_IMAGES = 100 # Only read the first 100 images\nEPOCHS = 2 # Number of iterations\nLEARNING_RATE = 1e-3\n\n# Read CSV labels\nlabels_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv'\n)[['patientId', 'Target']]\ndi_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv'\n)[['patientId', 'class']]\nmerged = pd.merge(labels_df, di_df, on='patientId', how='left')\n\n# Collect images and labels\nimages = []\ntargets = []\nimage_dir = '/kaggle/input/rsna-feature/'\ncount = 0\nfor _, row in merged.iterrows():\n if count >= MAX_IMAGES:\n break\n pid = row['patientId']\n for p in glob.glob(os.path.join(image_dir, f\"{pid}_feature*.png\")):\n try:\n # Load without resizing\n img = load_img(p, color_mode='rgb')\n arr = img_to_array(img) / 255.0\n images.append(arr)\n targets.append(row['Target'])\n count += 1\n break # Take only one image per patient\n except Exception:\n continue\n\n# Manually split into training/validation\nsplit = int(len(images) * 0.8)\ntrain_imgs, val_imgs = images[:split], images[split:]\ntrain_lbls, val_lbls = targets[:split], targets[split:]\n\n# Build model (dynamic input)\nbase = MobileNetV2(input_shape=(None, None, 3), include_top=False, weights=None)\nbase.trainable = False\nmodel = Sequential([\n base,\n GlobalAveragePooling2D(),\n Dense(64, activation='relu'),\n Dense(1, activation='sigmoid')\n])\n\noptimizer = Adam(learning_rate=LEARNING_RATE)\nmodel.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])\n\n# Sample by sample training\nfor epoch in range(1, EPOCHS + 1):\n print(f\"Epoch {epoch}/{EPOCHS}\")\n # Training\n for img, lbl in zip(train_imgs, train_lbls):\n x = np.expand_dims(img, axis=0)\n y = np.array([lbl])\n loss, acc = model.train_on_batch(x, y)\n # Validation\n val_losses, val_accs = [], []\n for img, lbl in zip(val_imgs, val_lbls):\n x = np.expand_dims(img, axis=0)\n y = np.array([lbl])\n metrics = model.test_on_batch(x, y)\n val_losses.append(metrics[0])\n val_accs.append(metrics[1])\n print(f\" val_loss: {np.mean(val_losses):.4f}, val_acc: {np.mean(val_accs):.4f}\\n\")\n\nprint(\"Training complete\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T10:26:41.556075Z","iopub.execute_input":"2025-05-10T10:26:41.556686Z","iopub.status.idle":"2025-05-10T10:30:27.667915Z","shell.execute_reply.started":"2025-05-10T10:26:41.556661Z","shell.execute_reply":"2025-05-10T10:30:27.667113Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Add timing\n# -*- coding: utf-8 -*-\n\"\"\"\nVariable size image training example:\n- No scaling or padding of images\n- Only take the first 100 images\n- Use MobileNetV2 (dynamic input) + GAP\n- Manually train sample by sample without batching\n\"\"\"\nimport numpy as np\nimport pandas as pd\nimport os\nimport glob\nimport tensorflow as tf\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dense\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.preprocessing.image import load_img, img_to_array\nimport time\n\n# Configuration\nMAX_IMAGES = 100 # Only read the first 100 images\nEPOCHS = 2 # Number of iterations\nLEARNING_RATE = 1e-3\n\n# Read CSV labels\nlabels_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv'\n)[['patientId', 'Target']]\ndi_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv'\n)[['patientId', 'class']]\nmerged = pd.merge(labels_df, di_df, on='patientId', how='left')\n\n# Collect images and labels\nimages = []\ntargets = []\nimage_dir = '/kaggle/input/rsna-feature/'\ncount = 0\nfor _, row in merged.iterrows():\n if count >= MAX_IMAGES:\n break\n pid = row['patientId']\n for p in glob.glob(os.path.join(image_dir, f\"{pid}_feature*.png\")):\n try:\n # Load without resizing\n img = load_img(p, color_mode='rgb')\n arr = img_to_array(img) / 255.0\n images.append(arr)\n targets.append(row['Target'])\n count += 1\n break # Take only one image per patient\n except Exception:\n continue\n\n# Manually split into training/validation\nsplit = int(len(images) * 0.8)\ntrain_imgs, val_imgs = images[:split], images[split:]\ntrain_lbls, val_lbls = targets[:split], targets[split:]\n\n# Build model (dynamic input)\nbase = MobileNetV2(input_shape=(None, None, 3), include_top=False, weights=None)\nbase.trainable = False\nmodel = Sequential([\n base,\n GlobalAveragePooling2D(),\n Dense(64, activation='relu'),\n Dense(1, activation='sigmoid')\n])\n\noptimizer = Adam(learning_rate=LEARNING_RATE)\nmodel.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])\n\n# Sample by sample training\nfor epoch in range(1, EPOCHS + 1):\n print(f\"Epoch {epoch}/{EPOCHS}\")\n \n # Record training time\n train_start_time = time.time()\n \n # Training\n for img, lbl in zip(train_imgs, train_lbls):\n x = np.expand_dims(img, axis=0)\n y = np.array([lbl])\n loss, acc = model.train_on_batch(x, y)\n \n train_end_time = time.time()\n train_duration = train_end_time - train_start_time\n \n # Record validation time\n val_start_time = time.time()\n \n val_losses, val_accs = [], []\n for img, lbl in zip(val_imgs, val_lbls):\n x = np.expand_dims(img, axis=0)\n y = np.array([lbl])\n metrics = model.test_on_batch(x, y)\n val_losses.append(metrics[0])\n val_accs.append(metrics[1])\n \n val_end_time = time.time()\n val_duration = val_end_time - val_start_time\n \n print(f\" Train time: {train_duration:.2f}s, val_loss: {np.mean(val_losses):.4f}, val_acc: {np.mean(val_accs):.4f}, Val time: {val_duration:.2f}s\\n\")\n\nprint(\"Training complete\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T10:32:55.323667Z","iopub.execute_input":"2025-05-10T10:32:55.324389Z","iopub.status.idle":"2025-05-10T10:36:36.921536Z","shell.execute_reply.started":"2025-05-10T10:32:55.32436Z","shell.execute_reply":"2025-05-10T10:36:36.920745Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Here’s the modified code that incorporates mixed precision, XLA JIT compilation, layer freezing, and gradient disabling to optimize GPU usage:\n# -*- coding: utf-8 -*-\n\"\"\"\nVariable size image training example (Kaggle):\n- No resizing of images\n- Only take the first 100 images\n- Supports AMP (mixed precision)\n- Use XLA JIT compilation\n- Freeze the first half of the base model's layers and disable gradient flow\n- TF Data Pipeline + prefetch to improve GPU utilization\n\"\"\"\nimport numpy as np\nimport pandas as pd\nimport os\nimport glob\nimport tensorflow as tf\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dense\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\n\n# ---------- Configuration ----------\nMAX_IMAGES = 100 # Only take the first 100\nEPOCHS = 3 # Number of epochs\nLEARNING_RATE = 1e-3\nBATCH_SIZE = 1 # Variable size, batch=1\nAUTOTUNE = tf.data.AUTOTUNE\n\n# ---------- Mixed Precision & XLA ----------\nfrom tensorflow.keras import mixed_precision\nmixed_precision.set_global_policy('mixed_float16') # Enable mixed precision\ntf.config.optimizer.set_jit(True) # Enable XLA JIT\n\n# ---------- Read and Merge Labels ----------\nlabels = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv', usecols=['patientId','Target']\n)\ninfo = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv', usecols=['patientId','class']\n)\ndf = pd.merge(labels, info, on='patientId', how='left')\n\n# ---------- Load First 100 Images & Build Dataset ----------\nimage_dir = '/kaggle/input/rsna-feature/'\npaths, targets = [], []\nfor pid, tgt in zip(df['patientId'], df['Target']):\n if len(paths) >= MAX_IMAGES:\n break\n files = glob.glob(os.path.join(image_dir, f\"{pid}_feature*.png\"))\n if not files: continue\n paths.append(files[0])\n targets.append(tgt)\n\n# Create TF Dataset\ndef load_and_preprocess(path, label):\n img = tf.io.read_file(path)\n img = tf.image.decode_png(img, channels=3)\n img = tf.image.convert_image_dtype(img, tf.float32)\n return img, label\n\ndataset = tf.data.Dataset.from_tensor_slices((paths, targets))\n# 80/20 Split\nn = len(paths)\ntrain_ds = dataset.take(int(n*0.8))\nval_ds = dataset.skip(int(n*0.8))\n\n# Pipeline: map, batch, prefetch\ntrain_ds = (\n train_ds.map(load_and_preprocess, num_parallel_calls=AUTOTUNE)\n .batch(BATCH_SIZE)\n .prefetch(AUTOTUNE)\n)\nval_ds = (\n val_ds.map(load_and_preprocess, num_parallel_calls=AUTOTUNE)\n .batch(BATCH_SIZE)\n .prefetch(AUTOTUNE)\n)\n\n# ---------- Build Model ----------\nbase = MobileNetV2(input_shape=(None, None, 3), include_top=False, weights=None)\n# Freeze the first half of layers and disable gradient flow\ntotal_layers = len(base.layers)\nfreeze_idx = total_layers // 2\nfor i, layer in enumerate(base.layers):\n if i < freeze_idx:\n layer.trainable = False # Freeze layer\n else:\n layer.trainable = False # Disable gradient updates\n layer._trainable_weights = [] # Disable gradient flow\n\nmodel = Sequential([\n base,\n GlobalAveragePooling2D(),\n Dense(64, activation='relu'),\n # Output as float16, convert back to float32\n Dense(1, activation='sigmoid', dtype='float32')\n])\n\n# Compile\nopt = Adam(learning_rate=LEARNING_RATE)\nmodel.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])\n\n# ---------- Train ----------\nmodel.fit(\n train_ds,\n epochs=EPOCHS,\n validation_data=val_ds,\n verbose=1\n)\n\nprint(\"Training complete\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T10:39:30.465449Z","iopub.execute_input":"2025-05-10T10:39:30.466225Z","iopub.status.idle":"2025-05-10T10:41:09.410566Z","shell.execute_reply.started":"2025-05-10T10:39:30.4662Z","shell.execute_reply":"2025-05-10T10:41:09.409942Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 时间统计\n# -*- coding: utf-8 -*-\n\"\"\"\n可变尺寸图像训练示例（Kaggle）：\n- 不调整图像大小\n- 仅取前100张图\n- 支持 AMP（混合精度）\n- 使用 XLA JIT 编译\n- 冻结基础模型前半部分层，并禁用梯度流\n- TF Data Pipeline + prefetch 提升 GPU 利用率\n\"\"\"\nimport numpy as np\nimport pandas as pd\nimport os\nimport glob\nimport tensorflow as tf\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dense\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.preprocessing.image import load_img, img_to_array\nimport time\n\n# ---------- 配置 ----------\nMAX_IMAGES = 100 # 只取前100张\nEPOCHS = 3 # 迭代轮数\nLEARNING_RATE = 1e-3\nBATCH_SIZE = 1 # 可变尺寸，batch=1\nAUTOTUNE = tf.data.AUTOTUNE\n\n# ---------- 混合精度 & XLA ----------\nfrom tensorflow.keras import mixed_precision\nmixed_precision.set_global_policy('mixed_float16')\n# 启用 XLA JIT\ntf.config.optimizer.set_jit(True)\n\n# ---------- 读取并合并标签 ----------\nlabels = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv', usecols=['patientId','Target']\n)\ninfo = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv', usecols=['patientId','class']\n)\ndf = pd.merge(labels, info, on='patientId', how='left')\n\n# ---------- 加载前100张图 & 构建 Dataset ----------\nimage_dir = '/kaggle/input/rsna-feature/'\npaths, targets = [], []\nfor pid, tgt in zip(df['patientId'], df['Target']):\n if len(paths) >= MAX_IMAGES:\n break\n files = glob.glob(os.path.join(image_dir, f\"{pid}_feature*.png\"))\n if not files: continue\n paths.append(files[0])\n targets.append(tgt)\n\n# 创建 TF Dataset\ndef load_and_preprocess(path, label):\n img = tf.io.read_file(path)\n img = tf.image.decode_png(img, channels=3)\n img = tf.image.convert_image_dtype(img, tf.float32)\n return img, label\n\ndataset = tf.data.Dataset.from_tensor_slices((paths, targets))\n# 80/20 划分\nn = len(paths)\ntrain_ds = dataset.take(int(n*0.8))\nval_ds = dataset.skip(int(n*0.8))\n\n# Pipeline: map, batch, prefetch\ntrain_ds = (\n train_ds.map(load_and_preprocess, num_parallel_calls=AUTOTUNE)\n .batch(BATCH_SIZE)\n .prefetch(AUTOTUNE)\n)\nval_ds = (\n val_ds.map(load_and_preprocess, num_parallel_calls=AUTOTUNE)\n .batch(BATCH_SIZE)\n .prefetch(AUTOTUNE)\n)\n\n# ---------- 构建模型 ----------\nbase = MobileNetV2(input_shape=(None, None, 3), include_top=False, weights=None)\n# 冻结前半部分层并禁用梯度流\ntotal_layers = len(base.layers)\nfreeze_idx = total_layers // 2\nfor i, layer in enumerate(base.layers):\n if i < freeze_idx:\n layer.trainable = False\n else:\n # 禁用梯度更新\n layer.trainable = False\n layer._trainable_weights = []\n\nmodel = Sequential([\n base,\n GlobalAveragePooling2D(),\n Dense(64, activation='relu'),\n # 输出为 float16，需要转换回 float32\n Dense(1, activation='sigmoid', dtype='float32')\n])\n\n# 编译\nopt = Adam(learning_rate=LEARNING_RATE)\nmodel.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])\n\n# ---------- 训练 ----------\nstart_time = time.time() # 记录开始时间\n\nmodel.fit(\n train_ds,\n epochs=EPOCHS,\n validation_data=val_ds,\n verbose=1\n)\n\nend_time = time.time() # 记录结束时间\ntotal_duration = end_time - start_time\n\nprint(f\"训练完成，耗时: {total_duration:.2f}秒\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T10:44:47.23592Z","iopub.execute_input":"2025-05-10T10:44:47.236601Z","iopub.status.idle":"2025-05-10T10:46:24.512693Z","shell.execute_reply.started":"2025-05-10T10:44:47.236574Z","shell.execute_reply":"2025-05-10T10:46:24.512106Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# -*- coding: utf-8 -*-\n\"\"\"\nVariable size image training example (Kaggle):\n- No resizing of images\n- Read all images\n- Supports AMP (mixed precision)\n- Use XLA JIT compilation\n- Freeze the first half of the base model's layers and disable gradient flow\n- TF Data Pipeline + prefetch to improve GPU utilization\n\"\"\"\nimport numpy as np\nimport pandas as pd\nimport os\nimport glob\nimport tensorflow as tf\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dense\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\nimport time\n\n# ---------- Configuration ----------\nMAX_IMAGES = None # Read all images\nEPOCHS = 1 # Number of epochs\nLEARNING_RATE = 1e-3\nBATCH_SIZE = 1 # Variable size, batch=1\nAUTOTUNE = tf.data.AUTOTUNE\n\n# ---------- Mixed Precision & XLA ----------\nfrom tensorflow.keras import mixed_precision\nmixed_precision.set_global_policy('mixed_float16') # Enable mixed precision\ntf.config.optimizer.set_jit(True) # Enable XLA JIT\n\n# ---------- Read and Merge Labels ----------\nlabels = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv', usecols=['patientId', 'Target']\n)\ninfo = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv', usecols=['patientId', 'class']\n)\ndf = pd.merge(labels, info, on='patientId', how='left')\n\n# ---------- Load All Images & Build Dataset ----------\nimage_dir = '/kaggle/input/rsna-feature/'\npaths, targets = [], []\nfor pid, tgt in zip(df['patientId'], df['Target']):\n files = glob.glob(os.path.join(image_dir, f\"{pid}_feature*.png\"))\n if not files: continue\n paths.append(files[0]) # Take the first image for each patient\n targets.append(tgt)\n\n# Create TF Dataset\ndef load_and_preprocess(path, label):\n img = tf.io.read_file(path)\n img = tf.image.decode_png(img, channels=3)\n img = tf.image.convert_image_dtype(img, tf.float32)\n return img, label\n\ndataset = tf.data.Dataset.from_tensor_slices((paths, targets))\n# 80/20 Split\nn = len(paths)\ntrain_ds = dataset.take(int(n * 0.8))\nval_ds = dataset.skip(int(n * 0.8))\n\n# Pipeline: map, batch, prefetch\ntrain_ds = (\n train_ds.map(load_and_preprocess, num_parallel_calls=AUTOTUNE)\n .batch(BATCH_SIZE)\n .prefetch(AUTOTUNE)\n)\nval_ds = (\n val_ds.map(load_and_preprocess, num_parallel_calls=AUTOTUNE)\n .batch(BATCH_SIZE)\n .prefetch(AUTOTUNE)\n)\n\n# ---------- Build Model ----------\nbase = MobileNetV2(input_shape=(None, None, 3), include_top=False, weights=None)\n# Freeze the first half of layers and disable gradient flow\ntotal_layers = len(base.layers)\nfreeze_idx = total_layers // 2\nfor i, layer in enumerate(base.layers):\n if i < freeze_idx:\n layer.trainable = False # Freeze layer\n else:\n layer.trainable = False # Disable gradient updates\n layer._trainable_weights = [] # Disable gradient flow\n\nmodel = Sequential([\n base,\n GlobalAveragePooling2D(),\n Dense(64, activation='relu'),\n # Output as float16, convert back to float32\n Dense(1, activation='sigmoid', dtype='float32')\n])\n\n# Compile\nopt = Adam(learning_rate=LEARNING_RATE)\nmodel.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])\n\n# ---------- Train ----------\nstart_time = time.time() # Record start time\n\nmodel.fit(\n train_ds,\n epochs=EPOCHS,\n validation_data=val_ds,\n verbose=1\n)\n\nend_time = time.time() # Record end time\ntotal_duration = end_time - start_time\n\nprint(f\"Training complete, duration: {total_duration:.2f} seconds\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T10:49:45.015362Z","iopub.execute_input":"2025-05-10T10:49:45.015923Z","iopub.status.idle":"2025-05-10T10:58:25.363631Z","shell.execute_reply.started":"2025-05-10T10:49:45.015897Z","shell.execute_reply":"2025-05-10T10:58:25.362697Z"},"jupyter":{"source_hidden":true,"outputs_hidden":true},"collapsed":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"It's also quite strange here that 9555 is not the previous abnormal 9551","metadata":{}},{"cell_type":"code","source":"import os\nimport glob\n\n# Specify the image directory\nimage_dir = '/kaggle/input/rsna-feature/'\n\n# Use glob to get all matching files\nfiles = glob.glob(os.path.join(image_dir, '*_feature*.png'))\n\n# Calculate the number of files\nnum_files = len(files)\n\nprint(f\"Found {num_files} image files in the directory '{image_dir}'.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T10:59:51.461461Z","iopub.execute_input":"2025-05-10T10:59:51.461754Z","iopub.status.idle":"2025-05-10T10:59:51.488048Z","shell.execute_reply.started":"2025-05-10T10:59:51.461733Z","shell.execute_reply":"2025-05-10T10:59:51.487273Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Only traverse the unique patientId and do not add the same image repeatedly.\n\nThe variable name \"labels_df\" has been corrected to ensure that tgt can index correctly.\n\nThe.cache () has been added to the data pipeline to avoid repeated decoding in each epoch within the same process.","metadata":{}},{"cell_type":"code","source":"# Run all images 1---- to solve the problem of too many files\n# -*- coding: utf-8 -*-\n\"\"\"\nVariable size image training example (Kaggle):\n- No resizing of images\n- Use all images (unique patientId)\n- Supports AMP (mixed precision)\n- Use XLA JIT compilation\n- Freeze the first half of the base model's layers and disable gradient flow\n- TF Data Pipeline + prefetch to improve GPU utilization\n\"\"\"\nimport numpy as np\nimport pandas as pd\nimport os\nimport glob\nimport tensorflow as tf\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dense\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\nimport time\n\n# ---------- Configuration ----------\nEPOCHS = 1\nLEARNING_RATE = 1e-3\nBATCH_SIZE = 1 # Variable size, batch=1\nAUTOTUNE = tf.data.AUTOTUNE\n\n# ---------- Mixed Precision & XLA ----------\nfrom tensorflow.keras import mixed_precision\nmixed_precision.set_global_policy('mixed_float16') # Enable mixed precision\ntf.config.optimizer.set_jit(True) # Enable XLA JIT\n\n# ---------- Read and Merge Labels ----------\nlabels_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv',\n usecols=['patientId', 'Target']\n)\ninfo_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv',\n usecols=['patientId', 'class']\n)\ndf = pd.merge(labels_df, info_df, on='patientId', how='left')\n\n# ---------- Paths and Labels (unique patientId) ----------\nimage_dir = '/kaggle/input/rsna-feature/'\npaths, targets = [], []\nfor pid in df['patientId'].unique():\n tgt = labels_df.loc[labels_df.patientId == pid, 'Target'].iloc[0]\n files = glob.glob(os.path.join(image_dir, f\"{pid}_feature*.png\"))\n if not files:\n continue\n paths.append(files[0]) # Take the first image for each patient\n targets.append(tgt)\n\n# Confirm the count\nprint(f\"Found a total of {len(paths)} unique patientId images\")\n\n# ---------- Create TF Dataset ----------\ndef load_and_preprocess(path, label):\n img = tf.io.read_file(path)\n img = tf.image.decode_png(img, channels=3)\n img = tf.image.convert_image_dtype(img, tf.float32)\n return img, label\n\ndataset = tf.data.Dataset.from_tensor_slices((paths, targets))\n\n# 80/20 Split\nn = len(paths)\ntrain_ds = dataset.take(int(n * 0.8))\nval_ds = dataset.skip(int(n * 0.8))\n\n# Pipeline: map, cache, batch, prefetch\ntrain_ds = (\n train_ds.map(load_and_preprocess, num_parallel_calls=AUTOTUNE)\n .cache()\n .batch(BATCH_SIZE)\n .prefetch(AUTOTUNE)\n)\nval_ds = (\n val_ds.map(load_and_preprocess, num_parallel_calls=AUTOTUNE)\n .cache()\n .batch(BATCH_SIZE)\n .prefetch(AUTOTUNE)\n)\n\n# ---------- Build Model ----------\nbase = MobileNetV2(input_shape=(None, None, 3), include_top=False, weights=None)\ntotal_layers = len(base.layers)\nfreeze_idx = total_layers // 2\nfor i, layer in enumerate(base.layers):\n layer.trainable = False # Freeze layer\n layer._trainable_weights = [] # Disable gradient flow\n\nmodel = Sequential([\n base,\n GlobalAveragePooling2D(),\n Dense(64, activation='relu'),\n Dense(1, activation='sigmoid', dtype='float32')\n])\n\n# Compile\nopt = Adam(learning_rate=LEARNING_RATE)\nmodel.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])\n\n# ---------- Train ----------\nstart_time = time.time()\nmodel.fit(\n train_ds,\n epochs=EPOCHS,\n validation_data=val_ds,\n verbose=1\n)\n\ntotal_duration = time.time() - start_time\nprint(f\"Training complete, duration: {total_duration:.2f} seconds\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T11:05:08.35256Z","iopub.execute_input":"2025-05-10T11:05:08.353273Z","iopub.status.idle":"2025-05-10T11:11:46.04006Z","shell.execute_reply.started":"2025-05-10T11:05:08.35325Z","shell.execute_reply":"2025-05-10T11:11:46.038976Z"},"collapsed":true,"jupyter":{"source_hidden":true,"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Accelerate 1- Increase batch-size\n# -*- coding: utf-8 -*-\n\"\"\"\nVariable size image training example (Kaggle):\n- No resizing of images\n- Use all images (unique patientId)\n- Supports AMP (mixed precision)\n- Use XLA JIT compilation\n- Freeze the first half of the base model's layers and disable gradient flow\n- TF Data Pipeline + prefetch to improve GPU utilization\n\"\"\"\nimport numpy as np\nimport pandas as pd\nimport os\nimport glob\nimport tensorflow as tf\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dense\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\nimport time\n\n# ---------- Configuration ----------\nEPOCHS = 1\nLEARNING_RATE = 1e-3\nBATCH_SIZE = 4 # Increased batch size for better performance\nAUTOTUNE = tf.data.AUTOTUNE\n\n# ---------- Mixed Precision & XLA ----------\nfrom tensorflow.keras import mixed_precision\nmixed_precision.set_global_policy('mixed_float16') # Enable mixed precision\ntf.config.optimizer.set_jit(True) # Enable XLA JIT\n\n# ---------- Read and Merge Labels ----------\nlabels_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv',\n usecols=['patientId', 'Target']\n)\ninfo_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv',\n usecols=['patientId', 'class']\n)\ndf = pd.merge(labels_df, info_df, on='patientId', how='left')\n\n# ---------- Paths and Labels (unique patientId) ----------\nimage_dir = '/kaggle/input/rsna-feature/'\npaths, targets = [], []\nfor pid in df['patientId'].unique():\n tgt = labels_df.loc[labels_df.patientId == pid, 'Target'].iloc[0]\n files = glob.glob(os.path.join(image_dir, f\"{pid}_feature*.png\"))\n if not files:\n continue\n paths.append(files[0]) # Take the first image for each patient\n targets.append(tgt)\n\n# Confirm the count\nprint(f\"Found a total of {len(paths)} unique patientId images\")\n\n# ---------- Create TF Dataset ----------\ndef load_and_preprocess(path, label):\n img = tf.io.read_file(path)\n img = tf.image.decode_png(img, channels=3)\n img = tf.image.convert_image_dtype(img, tf.float32)\n return img, label\n\ndataset = tf.data.Dataset.from_tensor_slices((paths, targets))\n\n# 80/20 Split\nn = len(paths)\ntrain_ds = dataset.take(int(n * 0.8))\nval_ds = dataset.skip(int(n * 0.8))\n\n# Pipeline: map, cache, batch, prefetch\ntrain_ds = (\n train_ds.map(load_and_preprocess, num_parallel_calls=AUTOTUNE)\n .cache()\n .batch(BATCH_SIZE)\n .prefetch(AUTOTUNE)\n)\nval_ds = (\n val_ds.map(load_and_preprocess, num_parallel_calls=AUTOTUNE)\n .cache()\n .batch(BATCH_SIZE)\n .prefetch(AUTOTUNE)\n)\n\n# ---------- Build Model ----------\nbase = MobileNetV2(input_shape=(None, None, 3), include_top=False, weights=None)\ntotal_layers = len(base.layers)\nfreeze_idx = total_layers // 2\nfor i, layer in enumerate(base.layers):\n layer.trainable = False # Freeze layer\n layer._trainable_weights = [] # Disable gradient flow\n\nmodel = Sequential([\n base,\n GlobalAveragePooling2D(),\n Dense(64, activation='relu'),\n Dense(1, activation='sigmoid', dtype='float32')\n])\n\n# Compile\nopt = Adam(learning_rate=LEARNING_RATE)\nmodel.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])\n\n# ---------- Train ----------\nstart_time = time.time()\nmodel.fit(\n train_ds,\n epochs=EPOCHS,\n validation_data=val_ds,\n verbose=1\n)\n\ntotal_duration = time.time() - start_time\nprint(f\"Training complete, duration: {total_duration:.2f} seconds\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T11:12:27.458879Z","iopub.execute_input":"2025-05-10T11:12:27.459476Z","iopub.status.idle":"2025-05-10T11:16:38.982919Z","shell.execute_reply.started":"2025-05-10T11:12:27.45945Z","shell.execute_reply":"2025-05-10T11:16:38.981732Z"},"jupyter":{"source_hidden":true,"outputs_hidden":true},"collapsed":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"When you change batch_size from 1 to 4, the reason for the error is:\n\n“Cannot batch tensors with different shapes”\nTensorFlow aims to combine all the samples in a batch into a tensor (shape [batch, H, W, C]). If variable-resolution images are directly packaged, they will fail due to inconsistent height and width.","metadata":{}},{"cell_type":"code","source":"# Acceleration 1.1- Bucketing by Size\n# -*- coding: utf-8 -*-\n\"\"\"\nVariable size image training example (Kaggle):\n- No resizing of images\n- Use all images (unique patientId)\n- Supports AMP (mixed precision)\n- Use XLA JIT compilation\n- Freeze the first half of the base model's layers and disable gradient flow\n- TF Data Pipeline + prefetch to improve GPU utilization\n\"\"\"\nimport numpy as np\nimport pandas as pd\nimport os\nimport glob\nimport tensorflow as tf\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dense\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\nimport time\n\n# ---------- Configuration ----------\nEPOCHS = 1\nLEARNING_RATE = 1e-3\nBATCH_SIZE = 4 # Increased batch size\nAUTOTUNE = tf.data.AUTOTUNE\n\n# ---------- Mixed Precision & XLA ----------\nfrom tensorflow.keras import mixed_precision\nmixed_precision.set_global_policy('mixed_float16') # Enable mixed precision\ntf.config.optimizer.set_jit(True) # Enable XLA JIT\n\n# ---------- Read and Merge Labels ----------\nlabels_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv',\n usecols=['patientId', 'Target']\n)\ninfo_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv',\n usecols=['patientId', 'class']\n)\ndf = pd.merge(labels_df, info_df, on='patientId', how='left')\n\n# ---------- Paths and Labels (unique patientId) ----------\nimage_dir = '/kaggle/input/rsna-feature/'\npaths, targets = [], []\nfor pid in df['patientId'].unique():\n tgt = labels_df.loc[labels_df.patientId == pid, 'Target'].iloc[0]\n files = glob.glob(os.path.join(image_dir, f\"{pid}_feature*.png\"))\n if not files:\n continue\n paths.append(files[0]) # Take the first image for each patient\n targets.append(tgt)\n\n# Confirm the count\nprint(f\"Found a total of {len(paths)} unique patientId images\")\n\n# ---------- Create TF Dataset ----------\ndef load_and_preprocess(path, label):\n img = tf.io.read_file(path)\n img = tf.image.decode_png(img, channels=3)\n img = tf.image.convert_image_dtype(img, tf.float32)\n return img, label\n\n# Bucketing function\ndef key_fn(path, label):\n img = tf.io.decode_png(tf.io.read_file(path), channels=3)\n h, w = tf.shape(img)[0], tf.shape(img)[1]\n bucket_h = h // 32\n bucket_w = w // 32\n return bucket_h * 100 + bucket_w, img, label\n\ndef reduce_fn(bucket_id, elems):\n return elems.batch(BATCH_SIZE, drop_remainder=True)\n\n# Create dataset and apply bucketing\ndataset = tf.data.Dataset.from_tensor_slices((paths, targets))\ndataset = dataset.apply(\n tf.data.experimental.group_by_window(\n key_func=lambda p, l: key_fn(p, l)[0],\n reduce_func=reduce_fn,\n window_size=BATCH_SIZE\n )\n)\n\n# 80/20 Split\nn = len(paths)\ntrain_ds = dataset.take(int(n * 0.8))\nval_ds = dataset.skip(int(n * 0.8))\n\n# Pipeline: map, cache, batch, prefetch\ntrain_ds = (\n train_ds.map(load_and_preprocess, num_parallel_calls=AUTOTUNE)\n .cache()\n .prefetch(AUTOTUNE)\n)\nval_ds = (\n val_ds.map(load_and_preprocess, num_parallel_calls=AUTOTUNE)\n .cache()\n .prefetch(AUTOTUNE)\n)\n\n# ---------- Build Model ----------\nbase = MobileNetV2(input_shape=(None, None, 3), include_top=False, weights=None)\ntotal_layers = len(base.layers)\nfreeze_idx = total_layers // 2\nfor i, layer in enumerate(base.layers):\n layer.trainable = False # Freeze layer\n layer._trainable_weights = [] # Disable gradient flow\n\nmodel = Sequential([\n base,\n GlobalAveragePooling2D(),\n Dense(64, activation='relu'),\n Dense(1, activation='sigmoid', dtype='float32')\n])\n\n# Compile\nopt = Adam(learning_rate=LEARNING_RATE)\nmodel.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])\n\n# ---------- Train ----------\nstart_time = time.time()\nmodel.fit(\n train_ds,\n epochs=EPOCHS,\n validation_data=val_ds,\n verbose=1\n)\n\ntotal_duration = time.time() - start_time\nprint(f\"Training complete, duration: {total_duration:.2f} seconds\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T11:25:09.946147Z","iopub.execute_input":"2025-05-10T11:25:09.946892Z","iopub.status.idle":"2025-05-10T11:29:23.352746Z","shell.execute_reply.started":"2025-05-10T11:25:09.946866Z","shell.execute_reply":"2025-05-10T11:29:23.351408Z"},"collapsed":true,"jupyter":{"source_hidden":true,"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The pipeline segmented by size was rewritten:\n\ndataset.group_by_window: key_fn returns the int64 bucket number, and reduce_fn performs map decoding and batch(BATCH_SIZE) for each bucket.\n\nRemove the additional '.batch ': Complete the batch processing within' reduce_fn ', and no longer call '.batch 'on the outer layer to avoid duplication.\n\n80/20 division: Split the training/validation set based on the total number of batches (aligned with BATCH_SIZE).","metadata":{}},{"cell_type":"code","source":"# There was no time to wait for him to finish running - it still reported an error\n# -*- coding: utf-8 -*-\n\"\"\"\nBucketing training example:\n- No resizing of images\n- Use all images (unique patientId)\n- Supports AMP & XLA\n- Freeze all layers and disable gradients\n- Bucket by shape and batch to improve GPU utilization\n\"\"\"\nimport os\nimport glob\nimport time\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dense\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\n\n# ---------- Configuration ----------\nEPOCHS = 1\nLEARNING_RATE = 1e-3\nBATCH_SIZE = 4 # Bucket batch size\nAUTOTUNE = tf.data.AUTOTUNE\n\n# ---------- Mixed Precision & XLA ----------\nfrom tensorflow.keras import mixed_precision\nmixed_precision.set_global_policy('mixed_float16') # Enable mixed precision\ntf.config.optimizer.set_jit(True) # Enable XLA JIT\n\n# ---------- Read Labels & Paths ----------\nlabels_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv',\n usecols=['patientId', 'Target']\n)\ninfo_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv',\n usecols=['patientId', 'class']\n)\ndf = pd.merge(labels_df, info_df, on='patientId', how='left')\n\nimage_dir = '/kaggle/input/rsna-feature/'\npaths, targets = [], []\nfor pid in df['patientId'].unique():\n files = glob.glob(os.path.join(image_dir, f\"{pid}_feature*.png\"))\n if not files:\n continue\n paths.append(files[0])\n targets.append(int(labels_df.loc[labels_df.patientId == pid, 'Target'].iloc[0]))\n\nprint(f\"Found a total of {len(paths)} unique patientId images\")\n\n# ---------- Dataset ----------\ndataset = tf.data.Dataset.from_tensor_slices((paths, targets))\n\n# Key function for bucketing\ndef key_fn(path, label):\n img = tf.image.decode_png(tf.io.read_file(path), channels=3)\n shape = tf.shape(img)\n bucket_h = shape[0] // 32\n bucket_w = shape[1] // 32\n key = bucket_h * 100 + bucket_w\n return tf.cast(key, tf.int64)\n\n# Reduce function for batching\ndef reduce_fn(key, ds):\n return (ds.map(lambda p, l: (\n tf.image.convert_image_dtype(tf.image.decode_png(tf.io.read_file(p), channels=3), tf.float32), l),\n num_parallel_calls=AUTOTUNE)\n .batch(BATCH_SIZE, drop_remainder=True))\n\n# Apply bucketing\ndataset = dataset.apply(\n tf.data.experimental.group_by_window(\n key_func=key_fn,\n reduce_func=reduce_fn,\n window_size=BATCH_SIZE\n )\n)\n\n# 80/20 Split\nn = len(paths) // BATCH_SIZE * BATCH_SIZE # Round down to the nearest batch size\ntrain_ds = dataset.take(int(n * 0.8 / BATCH_SIZE))\nval_ds = dataset.skip(int(n * 0.8 / BATCH_SIZE))\n\n# ---------- Build & Compile Model ----------\nbase = MobileNetV2(input_shape=(None, None, 3), include_top=False, weights=None)\nfor layer in base.layers:\n layer.trainable = False # Freeze all layers\n\nmodel = Sequential([\n base,\n GlobalAveragePooling2D(),\n Dense(64, activation='relu'),\n Dense(1, activation='sigmoid', dtype='float32')\n])\nopt = Adam(learning_rate=LEARNING_RATE)\nmodel.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])\n\n# ---------- Train ----------\nstart = time.time()\nmodel.fit(train_ds, epochs=EPOCHS, validation_data=val_ds)\nprint(f\"Training complete, duration: {time.time() - start:.2f} seconds\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T22:02:04.637139Z","iopub.execute_input":"2025-05-10T22:02:04.637379Z","iopub.status.idle":"2025-05-10T22:06:02.065975Z","shell.execute_reply.started":"2025-05-10T22:02:04.637354Z","shell.execute_reply":"2025-05-10T22:06:02.064755Z"},"jupyter":{"source_hidden":true,"outputs_hidden":true},"collapsed":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Unexpectedly, after unifying the size, the speed has actually increased and the memory usage has also decreased\n# -*- coding: utf-8 -*-\n\"\"\"\nVariable size image training example (Kaggle) - Resize + Multithreading + Distributed Strategy:\n- Global resize to smaller dimensions (e.g., 64×64) to enhance parallelism\n- Supports AMP (mixed precision) & XLA\n- Multithreaded configuration\n- Multi-GPU distributed strategy (MirroredStrategy)\n\"\"\"\nimport os\nimport glob\nimport time\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dense\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\n\n# ---------- Hyperparameters ----------\nIMAGE_SIZE = (64, 64) # Target resize dimensions\nBATCH_SIZE = 16 # Increase batch size for better GPU utilization\nEPOCHS = 3\nLEARNING_RATE = 1e-3\nAUTOTUNE = tf.data.AUTOTUNE\n\n# ---------- Mixed Precision & XLA ----------\nfrom tensorflow.keras import mixed_precision\nmixed_precision.set_global_policy('mixed_float16') # Enable mixed precision\ntf.config.optimizer.set_jit(True) # Enable XLA JIT\n\n# ---------- Distributed Strategy ----------\nstrategy = tf.distribute.MirroredStrategy()\nprint(f\"Number of GPUs: {strategy.num_replicas_in_sync}\")\n\nwith strategy.scope():\n # ---------- Build Model ----------\n base = MobileNetV2(input_shape=(*IMAGE_SIZE, 3), include_top=False, weights=None)\n base.trainable = False\n model = Sequential([\n base,\n GlobalAveragePooling2D(),\n Dense(64, activation='relu'),\n Dense(1, activation='sigmoid', dtype='float32')\n ])\n opt = Adam(learning_rate=LEARNING_RATE)\n model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])\n\n# ---------- Read Labels & Paths ----------\nlabels_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv',\n usecols=['patientId', 'Target']\n)\ninfo_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv',\n usecols=['patientId', 'class']\n)\ndf = pd.merge(labels_df, info_df, on='patientId', how='left')\n\nimage_dir = '/kaggle/input/rsna-feature/'\npaths, targets = [], []\nfor pid in df['patientId'].unique():\n files = glob.glob(os.path.join(image_dir, f\"{pid}_feature*.png\"))\n if not files:\n continue\n paths.append(files[0])\n targets.append(int(labels_df.loc[labels_df.patientId == pid, 'Target'].iloc[0]))\n\nprint(f\"Found a total of {len(paths)} unique patientId images\")\n\n# ---------- Dataset Pipeline ----------\ndef load_and_preprocess(path, label):\n img = tf.io.read_file(path)\n img = tf.image.decode_png(img, channels=3)\n img = tf.image.resize(img, IMAGE_SIZE) # Resize to uniform small size\n img = tf.image.convert_image_dtype(img, tf.float32)\n return img, label\n\n# Create dataset\ndataset = tf.data.Dataset.from_tensor_slices((paths, targets))\n# Shuffle & split\ndataset = dataset.shuffle(buffer_size=len(paths))\ntrain_size = int(0.8 * len(paths))\ntrain_ds = dataset.take(train_size)\nval_ds = dataset.skip(train_size)\n\n# Preprocessing pipeline\ndef prepare(ds):\n return (ds.map(load_and_preprocess, num_parallel_calls=AUTOTUNE)\n .batch(BATCH_SIZE)\n .cache()\n .prefetch(AUTOTUNE))\n\ntrain_ds = prepare(train_ds)\nval_ds = prepare(val_ds)\n\n# ---------- Train ----------\nstart = time.time()\nmodel.fit(train_ds, epochs=EPOCHS, validation_data=val_ds)\nprint(f\"Training complete, duration: {time.time() - start:.2f} seconds\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T22:12:19.504711Z","iopub.execute_input":"2025-05-10T22:12:19.504941Z","iopub.status.idle":"2025-05-10T22:16:56.274491Z","shell.execute_reply.started":"2025-05-10T22:12:19.504912Z","shell.execute_reply":"2025-05-10T22:16:56.273793Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# image size changed to 224x224:\n# -*- coding: utf-8 -*-\n\"\"\"\nVariable size image training example (Kaggle) - Resize + Multithreading + Distributed Strategy:\n- Global resize to smaller dimensions (e.g., 224×224) to enhance parallelism\n- Supports AMP (mixed precision) & XLA\n- Multithreaded configuration\n- Multi-GPU distributed strategy (MirroredStrategy)\n\"\"\"\nimport os\nimport glob\nimport time\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dense\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\n\n# ---------- Hyperparameters ----------\nIMAGE_SIZE = (224, 224) # Target resize dimensions\nBATCH_SIZE = 16 # Increase batch size for better GPU utilization\nEPOCHS = 3\nLEARNING_RATE = 1e-3\nAUTOTUNE = tf.data.AUTOTUNE\n\n# ---------- Mixed Precision & XLA ----------\nfrom tensorflow.keras import mixed_precision\nmixed_precision.set_global_policy('mixed_float16') # Enable mixed precision\ntf.config.optimizer.set_jit(True) # Enable XLA JIT\n\n# ---------- Distributed Strategy ----------\nstrategy = tf.distribute.MirroredStrategy()\nprint(f\"Number of GPUs: {strategy.num_replicas_in_sync}\")\n\nwith strategy.scope():\n # ---------- Build Model ----------\n base = MobileNetV2(input_shape=(*IMAGE_SIZE, 3), include_top=False, weights=None)\n base.trainable = False\n model = Sequential([\n base,\n GlobalAveragePooling2D(),\n Dense(64, activation='relu'),\n Dense(1, activation='sigmoid', dtype='float32')\n ])\n opt = Adam(learning_rate=LEARNING_RATE)\n model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])\n\n# ---------- Read Labels & Paths ----------\nlabels_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv',\n usecols=['patientId', 'Target']\n)\ninfo_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv',\n usecols=['patientId', 'class']\n)\ndf = pd.merge(labels_df, info_df, on='patientId', how='left')\n\nimage_dir = '/kaggle/input/rsna-feature/'\npaths, targets = [], []\nfor pid in df['patientId'].unique():\n files = glob.glob(os.path.join(image_dir, f\"{pid}_feature*.png\"))\n if not files:\n continue\n paths.append(files[0])\n targets.append(int(labels_df.loc[labels_df.patientId == pid, 'Target'].iloc[0]))\n\nprint(f\"Found a total of {len(paths)} unique patientId images\")\n\n# ---------- Dataset Pipeline ----------\ndef load_and_preprocess(path, label):\n img = tf.io.read_file(path)\n img = tf.image.decode_png(img, channels=3)\n img = tf.image.resize(img, IMAGE_SIZE) # Resize to uniform size\n img = tf.image.convert_image_dtype(img, tf.float32)\n return img, label\n\n# Create dataset\ndataset = tf.data.Dataset.from_tensor_slices((paths, targets))\n# Shuffle & split\ndataset = dataset.shuffle(buffer_size=len(paths))\ntrain_size = int(0.8 * len(paths))\ntrain_ds = dataset.take(train_size)\nval_ds = dataset.skip(train_size)\n\n# Preprocessing pipeline\ndef prepare(ds):\n return (ds.map(load_and_preprocess, num_parallel_calls=AUTOTUNE)\n .batch(BATCH_SIZE)\n .cache()\n .prefetch(AUTOTUNE))\n\ntrain_ds = prepare(train_ds)\nval_ds = prepare(val_ds)\n\n# ---------- Train ----------\nstart = time.time()\nmodel.fit(train_ds, epochs=EPOCHS, validation_data=val_ds)\nprint(f\"Training complete, duration: {time.time() - start:.2f} seconds\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T22:23:04.930542Z","iopub.execute_input":"2025-05-10T22:23:04.931212Z","iopub.status.idle":"2025-05-10T22:27:12.675681Z","shell.execute_reply.started":"2025-05-10T22:23:04.931187Z","shell.execute_reply":"2025-05-10T22:27:12.675087Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ---------- Validate the Model ----------\nval_loss, val_accuracy = model.evaluate(val_ds)\nprint(f\"Validation Loss: {val_loss:.4f}, Validation Accuracy: {val_accuracy:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T22:41:04.731973Z","iopub.execute_input":"2025-05-10T22:41:04.732317Z","iopub.status.idle":"2025-05-10T22:41:05.736644Z","shell.execute_reply.started":"2025-05-10T22:41:04.732293Z","shell.execute_reply":"2025-05-10T22:41:05.736103Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"1. Overfitting\n\n2. The time is too short.","metadata":{}},{"cell_type":"code","source":"# 1. Regularization\n# -*- coding: utf-8 -*-\n\"\"\"\nVariable size image training example (Kaggle) - Resize + Multithreading + Distributed Strategy:\n- Global resize to smaller dimensions (e.g., 224×224) to enhance parallelism\n- Supports AMP (mixed precision) & XLA\n- Multithreaded configuration\n- Multi-GPU distributed strategy (MirroredStrategy)\n\"\"\"\nimport os\nimport glob\nimport time\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dense\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.regularizers import l2\n\n# ---------- Hyperparameters ----------\nIMAGE_SIZE = (224, 224) # Target resize dimensions\nBATCH_SIZE = 16 # Increase batch size for better GPU utilization\nEPOCHS = 3\nLEARNING_RATE = 1e-3\nAUTOTUNE = tf.data.AUTOTUNE\n\n# ---------- Mixed Precision & XLA ----------\nfrom tensorflow.keras import mixed_precision\nmixed_precision.set_global_policy('mixed_float16') # Enable mixed precision\ntf.config.optimizer.set_jit(True) # Enable XLA JIT\n\n# ---------- Distributed Strategy ----------\nstrategy = tf.distribute.MirroredStrategy()\nprint(f\"Number of GPUs: {strategy.num_replicas_in_sync}\")\n\nwith strategy.scope():\n # ---------- Build Model ----------\n base = MobileNetV2(input_shape=(*IMAGE_SIZE, 3), include_top=False, weights=None)\n base.trainable = False\n model = Sequential([\n base,\n GlobalAveragePooling2D(),\n Dense(64, activation='relu', kernel_regularizer=l2(0.01)), # L2 regularization\n Dense(1, activation='sigmoid', dtype='float32')\n ])\n opt = Adam(learning_rate=LEARNING_RATE)\n model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])\n\n# ---------- Read Labels & Paths ----------\nlabels_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv',\n usecols=['patientId', 'Target']\n)\ninfo_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv',\n usecols=['patientId', 'class']\n)\ndf = pd.merge(labels_df, info_df, on='patientId', how='left')\n\nimage_dir = '/kaggle/input/rsna-feature/'\npaths, targets = [], []\nfor pid in df['patientId'].unique():\n files = glob.glob(os.path.join(image_dir, f\"{pid}_feature*.png\"))\n if not files:\n continue\n paths.append(files[0])\n targets.append(int(labels_df.loc[labels_df.patientId == pid, 'Target'].iloc[0]))\n\nprint(f\"Found a total of {len(paths)} unique patientId images\")\n\n# ---------- Dataset Pipeline ----------\ndef load_and_preprocess(path, label):\n img = tf.io.read_file(path)\n img = tf.image.decode_png(img, channels=3)\n img = tf.image.resize(img, IMAGE_SIZE) # Resize to uniform size\n img = tf.image.convert_image_dtype(img, tf.float32)\n return img, label\n\n# Create dataset\ndataset = tf.data.Dataset.from_tensor_slices((paths, targets))\n# Shuffle & split\ndataset = dataset.shuffle(buffer_size=len(paths))\ntrain_size = int(0.8 * len(paths))\ntrain_ds = dataset.take(train_size)\nval_ds = dataset.skip(train_size)\n\n# Preprocessing pipeline\ndef prepare(ds):\n return (ds.map(load_and_preprocess, num_parallel_calls=AUTOTUNE)\n .batch(BATCH_SIZE)\n .cache()\n .prefetch(AUTOTUNE))\n\ntrain_ds = prepare(train_ds)\nval_ds = prepare(val_ds)\n\n# ---------- Train ----------\nstart = time.time()\nmodel.fit(train_ds, epochs=EPOCHS, validation_data=val_ds)\nprint(f\"Training complete, duration: {time.time() - start:.2f} seconds\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T22:36:35.800853Z","iopub.execute_input":"2025-05-10T22:36:35.801519Z","iopub.status.idle":"2025-05-10T22:40:35.20227Z","shell.execute_reply.started":"2025-05-10T22:36:35.801487Z","shell.execute_reply":"2025-05-10T22:40:35.201527Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 2. Dropout\n# -*- coding: utf-8 -*-\n\"\"\"\nVariable size image training example (Kaggle) - Resize + Multithreading + Distributed Strategy:\n- Global resize to smaller dimensions (e.g., 224×224) to enhance parallelism\n- Supports AMP (mixed precision) & XLA\n- Multithreaded configuration\n- Multi-GPU distributed strategy (MirroredStrategy)\n\"\"\"\nimport os\nimport glob\nimport time\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dense, Dropout\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\n\n# ---------- Hyperparameters ----------\nIMAGE_SIZE = (224, 224) # Target resize dimensions\nBATCH_SIZE = 16 # Increase batch size for better GPU utilization\nEPOCHS = 3\nLEARNING_RATE = 1e-3\nAUTOTUNE = tf.data.AUTOTUNE\n\n# ---------- Mixed Precision & XLA ----------\nfrom tensorflow.keras import mixed_precision\nmixed_precision.set_global_policy('mixed_float16') # Enable mixed precision\ntf.config.optimizer.set_jit(True) # Enable XLA JIT\n\n# ---------- Distributed Strategy ----------\nstrategy = tf.distribute.MirroredStrategy()\nprint(f\"Number of GPUs: {strategy.num_replicas_in_sync}\")\n\nwith strategy.scope():\n # ---------- Build Model ----------\n base = MobileNetV2(input_shape=(*IMAGE_SIZE, 3), include_top=False, weights=None)\n base.trainable = False\n model = Sequential([\n base,\n GlobalAveragePooling2D(),\n Dense(64, activation='relu'),\n Dropout(0.5), # Dropout layer for regularization\n Dense(1, activation='sigmoid', dtype='float32')\n ])\n opt = Adam(learning_rate=LEARNING_RATE)\n model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])\n\n# ---------- Read Labels & Paths ----------\nlabels_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv',\n usecols=['patientId', 'Target']\n)\ninfo_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv',\n usecols=['patientId', 'class']\n)\ndf = pd.merge(labels_df, info_df, on='patientId', how='left')\n\nimage_dir = '/kaggle/input/rsna-feature/'\npaths, targets = [], []\nfor pid in df['patientId'].unique():\n files = glob.glob(os.path.join(image_dir, f\"{pid}_feature*.png\"))\n if not files:\n continue\n paths.append(files[0])\n targets.append(int(labels_df.loc[labels_df.patientId == pid, 'Target'].iloc[0]))\n\nprint(f\"Found a total of {len(paths)} unique patientId images\")\n\n# ---------- Dataset Pipeline ----------\ndef load_and_preprocess(path, label):\n img = tf.io.read_file(path)\n img = tf.image.decode_png(img, channels=3)\n img = tf.image.resize(img, IMAGE_SIZE) # Resize to uniform size\n img = tf.image.convert_image_dtype(img, tf.float32)\n return img, label\n\n# Create dataset\ndataset = tf.data.Dataset.from_tensor_slices((paths, targets))\n# Shuffle & split\ndataset = dataset.shuffle(buffer_size=len(paths))\ntrain_size = int(0.8 * len(paths))\ntrain_ds = dataset.take(train_size)\nval_ds = dataset.skip(train_size)\n\n# Preprocessing pipeline\ndef prepare(ds):\n return (ds.map(load_and_preprocess, num_parallel_calls=AUTOTUNE)\n .batch(BATCH_SIZE)\n .cache()\n .prefetch(AUTOTUNE))\n\ntrain_ds = prepare(train_ds)\nval_ds = prepare(val_ds)\n\n# ---------- Train ----------\nstart = time.time()\nmodel.fit(train_ds, epochs=EPOCHS, validation_data=val_ds)\nprint(f\"Training complete, duration: {time.time() - start:.2f} seconds\")","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-05-10T22:41:35.965956Z","iopub.execute_input":"2025-05-10T22:41:35.966555Z","iopub.status.idle":"2025-05-10T22:45:39.890971Z","shell.execute_reply.started":"2025-05-10T22:41:35.966533Z","shell.execute_reply":"2025-05-10T22:45:39.890348Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 3. Data Augmentation\n# -*- coding: utf-8 -*-\n\"\"\"\nVariable size image training example (Kaggle) - Resize + Multithreading + Distributed Strategy:\n- Global resize to smaller dimensions (e.g., 224×224) to enhance parallelism\n- Supports AMP (mixed precision) & XLA\n- Multithreaded configuration\n- Multi-GPU distributed strategy (MirroredStrategy)\n\"\"\"\nimport os\nimport glob\nimport time\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dense\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\n\n# ---------- Hyperparameters ----------\nIMAGE_SIZE = (224, 224) # Target resize dimensions\nBATCH_SIZE = 16 # Increase batch size for better GPU utilization\nEPOCHS = 3\nLEARNING_RATE = 1e-3\nAUTOTUNE = tf.data.AUTOTUNE\n\n# ---------- Mixed Precision & XLA ----------\nfrom tensorflow.keras import mixed_precision\nmixed_precision.set_global_policy('mixed_float16') # Enable mixed precision\ntf.config.optimizer.set_jit(True) # Enable XLA JIT\n\n# ---------- Distributed Strategy ----------\nstrategy = tf.distribute.MirroredStrategy()\nprint(f\"Number of GPUs: {strategy.num_replicas_in_sync}\")\n\nwith strategy.scope():\n # ---------- Build Model ----------\n base = MobileNetV2(input_shape=(*IMAGE_SIZE, 3), include_top=False, weights=None)\n base.trainable = False\n model = Sequential([\n base,\n GlobalAveragePooling2D(),\n Dense(64, activation='relu'),\n Dense(1, activation='sigmoid', dtype='float32')\n ])\n opt = Adam(learning_rate=LEARNING_RATE)\n model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])\n\n# ---------- Read Labels & Paths ----------\nlabels_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv',\n usecols=['patientId', 'Target']\n)\ninfo_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv',\n usecols=['patientId', 'class']\n)\ndf = pd.merge(labels_df, info_df, on='patientId', how='left')\n\nimage_dir = '/kaggle/input/rsna-feature/'\npaths, targets = [], []\nfor pid in df['patientId'].unique():\n files = glob.glob(os.path.join(image_dir, f\"{pid}_feature*.png\"))\n if not files:\n continue\n paths.append(files[0])\n targets.append(int(labels_df.loc[labels_df.patientId == pid, 'Target'].iloc[0]))\n\nprint(f\"Found a total of {len(paths)} unique patientId images\")\n\n# ---------- Dataset Pipeline ----------\ndef load_and_preprocess(path, label):\n img = tf.io.read_file(path)\n img = tf.image.decode_png(img, channels=3)\n img = tf.image.resize(img, IMAGE_SIZE)\n\n # Data Augmentation\n img = tf.image.random_flip_left_right(img)\n img = tf.image.random_brightness(img, max_delta=0.1)\n\n img = tf.image.convert_image_dtype(img, tf.float32)\n return img, label\n\n# Create dataset\ndataset = tf.data.Dataset.from_tensor_slices((paths, targets))\n# Shuffle & split\ndataset = dataset.shuffle(buffer_size=len(paths))\ntrain_size = int(0.8 * len(paths))\ntrain_ds = dataset.take(train_size)\nval_ds = dataset.skip(train_size)\n\n# Preprocessing pipeline\ndef prepare(ds):\n return (ds.map(load_and_preprocess, num_parallel_calls=AUTOTUNE)\n .batch(BATCH_SIZE)\n .cache()\n .prefetch(AUTOTUNE))\n\ntrain_ds = prepare(train_ds)\nval_ds = prepare(val_ds)\n\n# ---------- Train ----------\nstart = time.time()\nmodel.fit(train_ds, epochs=EPOCHS, validation_data=val_ds)\nprint(f\"Training complete, duration: {time.time() - start:.2f} seconds\")","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-05-10T22:47:42.025451Z","iopub.execute_input":"2025-05-10T22:47:42.026293Z","iopub.status.idle":"2025-05-10T22:51:39.725315Z","shell.execute_reply.started":"2025-05-10T22:47:42.026267Z","shell.execute_reply":"2025-05-10T22:51:39.724615Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 4. Reduce the learning rate\n# -*- coding: utf-8 -*-\n\"\"\"\nVariable size image training example (Kaggle) - Resize + Multithreading + Distributed Strategy:\n- Global resize to smaller dimensions (e.g., 224×224) to enhance parallelism\n- Supports AMP (mixed precision) & XLA\n- Multithreaded configuration\n- Multi-GPU distributed strategy (MirroredStrategy)\n\"\"\"\nimport os\nimport glob\nimport time\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dense\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\n\n# ---------- Hyperparameters ----------\nIMAGE_SIZE = (224, 224) # Target resize dimensions\nBATCH_SIZE = 16 # Increase batch size for better GPU utilization\nEPOCHS = 10 # Increased epochs for better training\nLEARNING_RATE = 1e-5 # Reduced learning rate\nAUTOTUNE = tf.data.AUTOTUNE\n\n# ---------- Mixed Precision & XLA ----------\nfrom tensorflow.keras import mixed_precision\nmixed_precision.set_global_policy('mixed_float16') # Enable mixed precision\ntf.config.optimizer.set_jit(True) # Enable XLA JIT\n\n# ---------- Distributed Strategy ----------\nstrategy = tf.distribute.MirroredStrategy()\nprint(f\"Number of GPUs: {strategy.num_replicas_in_sync}\")\n\nwith strategy.scope():\n # ---------- Build Model ----------\n base = MobileNetV2(input_shape=(*IMAGE_SIZE, 3), include_top=False, weights=None)\n base.trainable = False\n model = Sequential([\n base,\n GlobalAveragePooling2D(),\n Dense(64, activation='relu'),\n Dense(1, activation='sigmoid', dtype='float32')\n ])\n opt = Adam(learning_rate=LEARNING_RATE)\n model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])\n\n# ---------- Read Labels & Paths ----------\nlabels_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv',\n usecols=['patientId', 'Target']\n)\ninfo_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv',\n usecols=['patientId', 'class']\n)\ndf = pd.merge(labels_df, info_df, on='patientId', how='left')\n\nimage_dir = '/kaggle/input/rsna-feature/'\npaths, targets = [], []\nfor pid in df['patientId'].unique():\n files = glob.glob(os.path.join(image_dir, f\"{pid}_feature*.png\"))\n if not files:\n continue\n paths.append(files[0])\n targets.append(int(labels_df.loc[labels_df.patientId == pid, 'Target'].iloc[0]))\n\nprint(f\"Found a total of {len(paths)} unique patientId images\")\n\n# ---------- Dataset Pipeline ----------\ndef load_and_preprocess(path, label):\n img = tf.io.read_file(path)\n img = tf.image.decode_png(img, channels=3)\n img = tf.image.resize(img, IMAGE_SIZE) # Resize to uniform size\n img = tf.image.convert_image_dtype(img, tf.float32)\n return img, label\n\n# Create dataset\ndataset = tf.data.Dataset.from_tensor_slices((paths, targets))\n# Shuffle & split\ndataset = dataset.shuffle(buffer_size=len(paths))\ntrain_size = int(0.8 * len(paths))\ntrain_ds = dataset.take(train_size)\nval_ds = dataset.skip(train_size)\n\n# Preprocessing pipeline\ndef prepare(ds):\n return (ds.map(load_and_preprocess, num_parallel_calls=AUTOTUNE)\n .batch(BATCH_SIZE)\n .cache()\n .prefetch(AUTOTUNE))\n\ntrain_ds = prepare(train_ds)\nval_ds = prepare(val_ds)\n\n# ---------- Train ----------\nstart = time.time()\nmodel.fit(train_ds, epochs=EPOCHS, validation_data=val_ds)\nprint(f\"Training complete, duration: {time.time() - start:.2f} seconds\")","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-05-10T22:51:49.149188Z","iopub.execute_input":"2025-05-10T22:51:49.149844Z","iopub.status.idle":"2025-05-10T22:56:14.200228Z","shell.execute_reply.started":"2025-05-10T22:51:49.149822Z","shell.execute_reply":"2025-05-10T22:56:14.199619Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The error you encountered was due to the mismatch between the shape of the input image and the shape expected by the model. Specifically, the expected input shape of the MobileNetV2 model is (None, 224, 224, 3), while the input shape you provided is (8, 214, 214, 3).","metadata":{}},{"cell_type":"code","source":"# GPT V1\nimport os, glob, time\nimport numpy as np\nimport pandas as pd\nimport pydicom\nimport tensorflow as tf\nfrom tensorflow.keras import mixed_precision\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dense, Dropout\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import (\n LearningRateScheduler,\n ReduceLROnPlateau,\n EarlyStopping\n)\n\n\n# ---------- 超参数 ----------\nIMAGE_SIZE = (224, 224)\nBATCH_SIZE = 16\nEPOCHS = 20\nINIT_LR = 1e-3\nAUTOTUNE = tf.data.AUTOTUNE\n\n# ---------- 混合精度 & XLA ----------\nmixed_precision.set_global_policy('mixed_float16')\ntf.config.optimizer.set_jit(True)\n\n# ---------- 分布式策略 ----------\nstrategy = tf.distribute.MirroredStrategy()\nprint(\"Number of GPUs:\", strategy.num_replicas_in_sync)\n\n# 自定义 Focal Loss\ndef focal_loss(gamma=2.0, alpha=0.25):\n def loss(y_true, y_pred):\n # 防止 log(0)\n y_pred = tf.clip_by_value(y_pred, tf.keras.backend.epsilon(), 1. - tf.keras.backend.epsilon())\n # 交叉熵\n ce = - (y_true * tf.math.log(y_pred) + (1 - y_true) * tf.math.log(1 - y_pred))\n # 权重项\n p_t = y_true * y_pred + (1 - y_true) * (1 - y_pred)\n weight = alpha * y_true * tf.pow(1 - y_pred, gamma) + (1 - alpha) * (1 - y_true) * tf.pow(y_pred, gamma)\n return tf.reduce_mean(weight * ce)\n return loss\n\n# … strategy.scope() 里 …\nwith strategy.scope():\n # （省略 model 定义部分）\n loss_fn = focal_loss(gamma=2.0, alpha=0.25)\n opt = Adam(learning_rate=INIT_LR)\n model.compile(optimizer=opt,\n loss=loss_fn,\n metrics=['accuracy'])\n \nwith strategy.scope():\n # 使用 ImageNet 预训练权重\n base = MobileNetV2(input_shape=(*IMAGE_SIZE,3), include_top=False, weights='imagenet')\n base.trainable = False # 先冻结全部层\n\n model = Sequential([\n base,\n GlobalAveragePooling2D(),\n Dropout(0.5),\n Dense(128, activation='relu'),\n Dropout(0.3),\n Dense(1, activation='sigmoid', dtype='float32')\n ])\n\n # 使用 Focal Loss 以平衡正负样本\n # loss_fn = SigmoidFocalCrossEntropy(alpha=0.25, gamma=2.0)\n opt = Adam(learning_rate=INIT_LR)\n model.compile(optimizer=opt,\n loss=loss_fn,\n metrics=['accuracy'])\n\n# ---------- 读取标签 & 路径 ----------\nlabels = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv',\n usecols=['patientId','Target']\n)\ninfo = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv',\n usecols=['patientId','class']\n)\ndf = pd.merge(labels, info, on='patientId', how='left')\n\nimage_dir = '/kaggle/input/rsna-feature/'\npaths, targets = [], []\nfor pid in df['patientId'].unique():\n f = glob.glob(os.path.join(image_dir, f\"{pid}_feature*.png\"))\n if f:\n paths.append(f[0])\n targets.append(int(labels.loc[labels.patientId==pid,'Target'].iloc[0]))\nprint(f\"Found {len(paths)} training images\")\n\n# ---------- 数据管道 ----------\ndef load_and_preprocess(path, label, training=True):\n img = tf.io.read_file(path)\n img = tf.image.decode_png(img, channels=3)\n img = tf.image.resize(img, IMAGE_SIZE)\n if training:\n img = tf.image.random_flip_left_right(img)\n img = tf.image.random_flip_up_down(img)\n img = tf.image.random_brightness(img, 0.1)\n img = tf.image.random_contrast(img, 0.1,0.2)\n img = tf.image.convert_image_dtype(img, tf.float32)\n return img, label\n\ndef prepare_ds(paths, labs, training):\n ds = tf.data.Dataset.from_tensor_slices((paths, labs))\n if training:\n ds = ds.shuffle(len(paths))\n ds = ds.map(lambda p,l: load_and_preprocess(p,l,training),\n num_parallel_calls=AUTOTUNE)\n ds = ds.batch(BATCH_SIZE).cache().prefetch(AUTOTUNE)\n return ds\n\n \ntrain_size = int(0.8 * len(paths))\ntrain_ds = prepare_ds(paths[:train_size], targets[:train_size], True)\nval_ds = prepare_ds(paths[train_size:], targets[train_size:], False)\n\n# ---------- Callbacks ----------\ncallbacks = [\n LearningRateScheduler(lambda e: INIT_LR * 0.1**(e//10)),\n ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, verbose=1),\n EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)\n]\n\n# ---------- 训练 Phase1: 只训练 Head ----------\nmodel.fit(train_ds, validation_data=val_ds,\n epochs=5, callbacks=callbacks)\n\n# ---------- 解冻并微调 Phase2 ----------\nwith strategy.scope():\n base.trainable = True\n # 只微调最后 30 层\n for layer in base.layers[:-30]:\n layer.trainable = False\n # opt.lr = INIT_LR * 0.1\n # 实例化优化器并调整学习率\n opt = Adam(learning_rate=INIT_LR * 0.1)\n \n # 把 focal_loss 实例化后作为 loss_fn\n loss_fn = focal_loss(gamma=2.0, alpha=0.25)\n model.compile(optimizer=opt, loss=loss_fn, metrics=['accuracy'])\n\nmodel.fit(train_ds, validation_data=val_ds,\n epochs=EPOCHS, callbacks=callbacks)\n\n# ---------- 构造 Test Dataset ----------\ntest_dir = '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images/'\ntest_paths = glob.glob(os.path.join(test_dir, '*.dcm'))\n\n# 建立 sliding-window patch list\ndef extract_patches(img, patch_size=224, stride=112):\n patches = []\n h, w, _ = img.shape\n for y in range(0, h-patch_size+1, stride):\n for x in range(0, w-patch_size+1, stride):\n patches.append(img[y:y+patch_size, x:x+patch_size, :])\n return np.array(patches)\n\ndef preprocess_dicom(path):\n ds = pydicom.dcmread(path)\n img = ds.pixel_array.astype(np.float32)\n img = (img/np.max(img))*255.0\n img = np.stack([img]*3, -1)\n return tf.image.resize(img, IMAGE_SIZE).numpy()\n\n# 用 sliding-window + TTA 预测\ndef predict_with_tta(path):\n img = preprocess_dicom(path)\n # 裁剪 patch\n patches = extract_patches(img, patch_size=224, stride=112)\n # 原图也用于一次预测\n all_imgs = np.vstack([patches, img[None,...]])\n preds = model.predict(all_imgs, batch_size=BATCH_SIZE)\n return preds.mean()\n\n# 评估\ny_true = []\ny_pred = []\nfor p in test_paths:\n pid = os.path.basename(p).replace('.dcm','')\n y_true.append(int(labels.loc[labels.patientId==pid,'Target'].values or [0])[0])\n y_pred.append(predict_with_tta(p) > 0.5)\n\ny_true = np.array(y_true)\ny_pred = np.array(y_pred)\nacc = (y_true==y_pred).mean()\nprint(f\"Test Accuracy: {acc:.4f}\")\n","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-05-11T07:44:02.456353Z","iopub.execute_input":"2025-05-11T07:44:02.456945Z","iopub.status.idle":"2025-05-11T07:44:03.861416Z","shell.execute_reply.started":"2025-05-11T07:44:02.45692Z","shell.execute_reply":"2025-05-11T07:44:03.860465Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Although there are some problems with the display, the training results are good","metadata":{}},{"cell_type":"code","source":"import glob\nimport os\nimport numpy as np\nimport pydicom\nimport tensorflow as tf\n\n# Assuming labels_df already exists with structure ['patientId', 'Target']\ntest_dir = '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images/'\n\n# 1. Collect test DICOM paths\ntest_paths = glob.glob(os.path.join(test_dir, '*.dcm'))\nprint(f\"Found {len(test_paths)} test DICOM files\")\n\n# 2. Extract patientId from filenames and map to labels\ntest_labels = []\nfor p in test_paths:\n # Example filename: 0000a175-0e68-4ca4-b1af-167204a7e0bc.dcm\n pid = os.path.basename(p).replace('.dcm', '')\n # Skip if this test patient's ID is not in the training labels\n if pid not in labels_df.patientId.values:\n test_labels.append(0) # Assign a default label or handle as needed\n else:\n test_labels.append(int(labels_df.loc[labels_df.patientId == pid, 'Target'].iloc[0]))\n\n# 3. Define DICOM preprocessing function\ndef load_and_preprocess_dicom(path, label):\n # a) Read DICOM to get pixel matrix\n ds = pydicom.dcmread(path.numpy().decode('utf-8'))\n img = ds.pixel_array\n # b) Convert to three channels (repeat grayscale to RGB)\n img = np.stack([img] * 3, axis=-1)\n # c) Normalize to [0, 1]\n img = img.astype(np.float32) / np.max(img)\n # d) Resize to the same IMAGE_SIZE used during training\n img = tf.image.resize(img, IMAGE_SIZE)\n return img, label\n\n# 4. Wrap using tf.data.Dataset.from_tensor_slices + map \ndef tf_load_and_preprocess_dicom(path, label):\n img, lbl = tf.py_function(\n load_and_preprocess_dicom,\n inp=[path, label],\n Tout=[tf.float32, tf.int32]\n )\n img.set_shape((*IMAGE_SIZE, 3))\n lbl.set_shape(())\n return img, lbl\n\ntest_ds = tf.data.Dataset.from_tensor_slices((test_paths, test_labels))\ntest_ds = (test_ds\n .map(tf_load_and_preprocess_dicom, num_parallel_calls=tf.data.AUTOTUNE)\n .batch(BATCH_SIZE)\n .prefetch(tf.data.AUTOTUNE)\n )\n\n# 5. Evaluate\nresults = model.evaluate(test_ds)\nprint(\"Test loss, Test accuracy:\", results)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-11T07:27:41.370143Z","iopub.execute_input":"2025-05-11T07:27:41.370433Z","iopub.status.idle":"2025-05-11T07:37:30.089101Z","shell.execute_reply.started":"2025-05-11T07:27:41.370408Z","shell.execute_reply":"2025-05-11T07:37:30.08844Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import glob\nimport os\nimport numpy as np\nimport pydicom\nimport tensorflow as tf\n\n# Assuming labels_df already exists with structure ['patientId', 'Target']\ntest_dir = '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images/'\n\n# 1. Collect test DICOM paths\ntest_paths = glob.glob(os.path.join(test_dir, '*.dcm'))\nprint(f\"Found {len(test_paths)} test DICOM files\")\n\n# 2. Extract patientId from filenames and map to labels\ntest_labels = []\nfor p in test_paths:\n # Example filename: 0000a175-0e68-4ca4-b1af-167204a7e0bc.dcm\n pid = os.path.basename(p).replace('.dcm', '')\n # Skip if this test patient's ID is not in the training labels\n if pid not in labels_df.patientId.values:\n test_labels.append(0) # Assign a default label or handle as needed\n else:\n test_labels.append(int(labels_df.loc[labels_df.patientId == pid, 'Target'].iloc[0]))\n\n# 3. Define DICOM preprocessing function\ndef load_and_preprocess_dicom(path, label):\n # a) Read DICOM to get pixel matrix\n ds = pydicom.dcmread(path.numpy().decode('utf-8'))\n img = ds.pixel_array\n # b) Convert to three channels (repeat grayscale to RGB)\n img = np.stack([img] * 3, axis=-1)\n # c) Normalize to [0, 1]\n img = img.astype(np.float32) / np.max(img)\n # d) Resize to the same IMAGE_SIZE used during training\n img = tf.image.resize(img, IMAGE_SIZE)\n return img, label\n\n# 4. Wrap using tf.data.Dataset.from_tensor_slices + map \ndef tf_load_and_preprocess_dicom(path, label):\n img, lbl = tf.py_function(\n load_and_preprocess_dicom,\n inp=[path, label],\n Tout=[tf.float32, tf.int32]\n )\n img.set_shape((*IMAGE_SIZE, 3))\n lbl.set_shape(())\n return img, lbl\n\ntest_ds = tf.data.Dataset.from_tensor_slices((test_paths, test_labels))\ntest_ds = (test_ds\n .map(tf_load_and_preprocess_dicom, num_parallel_calls=tf.data.AUTOTUNE)\n .batch(BATCH_SIZE)\n .prefetch(tf.data.AUTOTUNE)\n )\n\n# 5. Evaluate\nresults = model.evaluate(test_ds)\nprint(\"Test loss, Test accuracy:\", results)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T22:57:56.040022Z","iopub.execute_input":"2025-05-10T22:57:56.040829Z","iopub.status.idle":"2025-05-10T23:08:13.836764Z","shell.execute_reply.started":"2025-05-10T22:57:56.040805Z","shell.execute_reply":"2025-05-10T23:08:13.836205Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.metrics import roc_auc_score, precision_recall_fscore_support, confusion_matrix\n\n# 1. Predict on the entire test dataset\ny_true = []\ny_pred = []\nfor imgs, labels in test_ds:\n preds = model.predict(imgs)\n y_true.extend(labels.numpy())\n y_pred.extend(preds.flatten())\n\ny_true = np.array(y_true)\ny_pred = np.array(y_pred)\n\n# 2. Calculate AUC for binary classification\nauc = roc_auc_score(y_true, y_pred)\n\n# 3. Calculate precision, recall, F1, and confusion matrix using a threshold of 0.5\ny_hat = (y_pred >= 0.5).astype(int)\nprecision, recall, f1, _ = precision_recall_fscore_support(y_true, y_hat, average='binary')\ncm = confusion_matrix(y_true, y_hat)\n\nprint(f\"Test AUC: {auc:.4f}\")\nprint(f\"Precision: {precision:.4f}, Recall: {recall:.4f}, F1: {f1:.4f}\")\nprint(\"Confusion matrix:\\n\", cm)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T23:08:34.241771Z","iopub.execute_input":"2025-05-10T23:08:34.242572Z","iopub.status.idle":"2025-05-10T23:14:34.973244Z","shell.execute_reply.started":"2025-05-10T23:08:34.242548Z","shell.execute_reply":"2025-05-10T23:14:34.972009Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.metrics import roc_auc_score, precision_recall_fscore_support, confusion_matrix\n\n# 1. Predict on the entire test dataset\ny_true = []\ny_pred = []\nfor imgs, labels in test_ds:\n preds = model.predict(imgs, verbose=0) # Set verbose=0 to avoid progress bar output\n y_true.extend(labels.numpy())\n y_pred.extend(preds.flatten())\n\ny_true = np.array(y_true)\ny_pred = np.array(y_pred)\n\n# 2. Calculate AUC for binary classification\nauc = roc_auc_score(y_true, y_pred)\n\n# 3. Calculate precision, recall, F1, and confusion matrix using a threshold of 0.5\ny_hat = (y_pred >= 0.5).astype(int)\nprecision, recall, f1, _ = precision_recall_fscore_support(y_true, y_hat, average='binary')\ncm = confusion_matrix(y_true, y_hat)\n\nprint(f\"Test AUC: {auc:.4f}\")\nprint(f\"Precision: {precision:.4f}, Recall: {recall:.4f}, F1: {f1:.4f}\")\nprint(\"Confusion matrix:\\n\", cm)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T23:16:05.969917Z","iopub.execute_input":"2025-05-10T23:16:05.97049Z","iopub.status.idle":"2025-05-10T23:26:43.032529Z","shell.execute_reply.started":"2025-05-10T23:16:05.970466Z","shell.execute_reply":"2025-05-10T23:26:43.031746Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Let's see if the ACC can be improved","metadata":{}},{"cell_type":"code","source":"import glob\nimport os\nimport numpy as np\nimport pydicom\nimport tensorflow as tf\n\n# Assuming labels_df already exists with structure ['patientId', 'Target']\ntest_dir = '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images/'\n\n# 1. Collect test DICOM paths\ntest_paths = glob.glob(os.path.join(test_dir, '*.dcm'))\nprint(f\"Found {len(test_paths)} test DICOM files\")\n\n# 2. Extract patientId from filenames and map to labels\ntest_labels = []\nfor p in test_paths:\n # Example filename: 0000a175-0e68-4ca4-b1af-167204a7e0bc.dcm\n pid = os.path.basename(p).replace('.dcm', '')\n # Skip if this test patient's ID is not in the training labels\n if pid not in labels_df.patientId.values:\n test_labels.append(0) # Assign a default label or handle as needed\n else:\n test_labels.append(int(labels_df.loc[labels_df.patientId == pid, 'Target'].iloc[0]))\n\n# 3. Define DICOM preprocessing function\ndef load_and_preprocess_dicom(path, label):\n # a) Read DICOM to get pixel matrix\n ds = pydicom.dcmread(path.numpy().decode('utf-8'))\n img = ds.pixel_array\n # b) Convert to three channels (repeat grayscale to RGB)\n img = np.stack([img] * 3, axis=-1)\n # c) Normalize to [0, 1]\n img = img.astype(np.float32) / np.max(img)\n # d) Resize to the same IMAGE_SIZE used during training\n img = tf.image.resize(img, IMAGE_SIZE)\n return img, label\n\n# 4. Wrap using tf.data.Dataset.from_tensor_slices + map \ndef tf_load_and_preprocess_dicom(path, label):\n img, lbl = tf.py_function(\n load_and_preprocess_dicom,\n inp=[path, label],\n Tout=[tf.float32, tf.int32]\n )\n img.set_shape((*IMAGE_SIZE, 3))\n lbl.set_shape(())\n return img, lbl\n\ntest_ds = tf.data.Dataset.from_tensor_slices((test_paths, test_labels))\ntest_ds = (test_ds\n .map(tf_load_and_preprocess_dicom, num_parallel_calls=tf.data.AUTOTUNE)\n .batch(BATCH_SIZE)\n .prefetch(tf.data.AUTOTUNE)\n )\n\n# 5. Evaluate\nresults = model.evaluate(test_ds)\nprint(\"Test loss, Test accuracy:\", results)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-11T00:46:49.564798Z","iopub.execute_input":"2025-05-11T00:46:49.56557Z","iopub.status.idle":"2025-05-11T00:57:34.643261Z","shell.execute_reply.started":"2025-05-11T00:46:49.565541Z","shell.execute_reply":"2025-05-11T00:57:34.642451Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"adjust lr","metadata":{}},{"cell_type":"code","source":"# # Adjust the learning rate 1 to 20 rounds -4\n# -*- coding: utf-8 -*-\n\"\"\"\nVariable size image training example (Kaggle) - Resize + Multithreading + Distributed Strategy:\n- Global resize to smaller dimensions (e.g., 224×224) to enhance parallelism\n- Supports AMP (mixed precision) & XLA\n- Multithreaded configuration\n- Multi-GPU distributed strategy (MirroredStrategy)\n\"\"\"\nimport os\nimport glob\nimport time\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dense, Dropout\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\n\n# ---------- Hyperparameters ----------\nIMAGE_SIZE = (224, 224) # Target resize dimensions\nBATCH_SIZE = 16 # Increase batch size for better GPU utilization\nEPOCHS = 20 # Increased epochs for better training\nLEARNING_RATE = 1e-4 # Initial learning rate\nAUTOTUNE = tf.data.AUTOTUNE\n\n# ---------- Mixed Precision & XLA ----------\nfrom tensorflow.keras import mixed_precision\nmixed_precision.set_global_policy('mixed_float16') # Enable mixed precision\ntf.config.optimizer.set_jit(True) # Enable XLA JIT\n\n# ---------- Distributed Strategy ----------\nstrategy = tf.distribute.MirroredStrategy()\nprint(f\"Number of GPUs: {strategy.num_replicas_in_sync}\")\n\nwith strategy.scope():\n base = MobileNetV2(input_shape=(*IMAGE_SIZE, 3), include_top=False, weights=None)\n base.trainable = True # Unfreeze all layers\n for layer in base.layers[-30:]:\n layer.trainable = True # Unfreeze only the last 30 layers\n\n model = Sequential([\n base,\n GlobalAveragePooling2D(),\n Dropout(0.5), # Add Dropout\n Dense(128, activation='relu'), # Increase the number of units\n Dense(1, activation='sigmoid', dtype='float32')\n ])\n opt = Adam(learning_rate=LEARNING_RATE)\n model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])\n\n# ---------- Read Labels & Paths ----------\nlabels_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv',\n usecols=['patientId', 'Target']\n)\ninfo_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv',\n usecols=['patientId', 'class']\n)\ndf = pd.merge(labels_df, info_df, on='patientId', how='left')\n\nimage_dir = '/kaggle/input/rsna-feature/'\npaths, targets = [], []\nfor pid in df['patientId'].unique():\n files = glob.glob(os.path.join(image_dir, f\"{pid}_feature*.png\"))\n if not files:\n continue\n paths.append(files[0])\n targets.append(int(labels_df.loc[labels_df.patientId == pid, 'Target'].iloc[0]))\n\nprint(f\"Found a total of {len(paths)} unique patientId images\")\n\n# ---------- Dataset Pipeline ----------\n# Data Augmentation\ndef load_and_preprocess(path, label):\n img = tf.io.read_file(path)\n img = tf.image.decode_png(img, channels=3)\n img = tf.image.resize(img, IMAGE_SIZE)\n\n # Data Augmentation\n img = tf.image.random_flip_left_right(img)\n img = tf.image.random_brightness(img, max_delta=0.1)\n img = tf.image.random_contrast(img, 0.1, 0.2)\n img = tf.image.random_flip_up_down(img) # Add vertical flip\n img = tf.image.random_crop(img, size=[IMAGE_SIZE[0] - 10, IMAGE_SIZE[1] - 10, 3]) # Random crop\n\n img = tf.image.convert_image_dtype(img, tf.float32)\n return img, label\n\ndataset = tf.data.Dataset.from_tensor_slices((paths, targets))\n# Shuffle & split\ndataset = dataset.shuffle(buffer_size=len(paths))\ntrain_size = int(0.8 * len(paths))\ntrain_ds = dataset.take(train_size)\nval_ds = dataset.skip(train_size)\n\n# Preprocessing pipeline\ndef prepare(ds):\n return (ds.map(load_and_preprocess, num_parallel_calls=AUTOTUNE)\n .batch(BATCH_SIZE)\n .cache()\n .prefetch(AUTOTUNE))\n\ntrain_ds = prepare(train_ds)\nval_ds = prepare(val_ds)\n\n# ---------- Train ----------\nstart = time.time()\n# Learning rate schedule\nlr_schedule = tf.keras.callbacks.LearningRateScheduler(lambda epoch: 1e-3 * 0.1**(epoch // 10))\nmodel.fit(train_ds, epochs=EPOCHS, validation_data=val_ds, callbacks=[lr_schedule])\nprint(f\"Training complete, duration: {time.time() - start:.2f} seconds\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-11T02:57:45.858006Z","iopub.execute_input":"2025-05-11T02:57:45.858352Z","iopub.status.idle":"2025-05-11T03:09:47.061622Z","shell.execute_reply.started":"2025-05-11T02:57:45.858326Z","shell.execute_reply":"2025-05-11T03:09:47.060967Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import glob\nimport os\nimport numpy as np\nimport pydicom\nimport tensorflow as tf\n\n# Assuming labels_df already exists with structure ['patientId', 'Target']\ntest_dir = '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images/'\n\n# 1. Collect test DICOM paths\ntest_paths = glob.glob(os.path.join(test_dir, '*.dcm'))\nprint(f\"Found {len(test_paths)} test DICOM files\")\n\n# 2. Extract patientId from filenames and map to labels\ntest_labels = []\nfor p in test_paths:\n # Example filename: 0000a175-0e68-4ca4-b1af-167204a7e0bc.dcm\n pid = os.path.basename(p).replace('.dcm', '')\n # Skip if this test patient's ID is not in the training labels\n if pid not in labels_df.patientId.values:\n test_labels.append(0) # Assign a default label or handle as needed\n else:\n test_labels.append(int(labels_df.loc[labels_df.patientId == pid, 'Target'].iloc[0]))\n\n# 3. Define DICOM preprocessing function\ndef load_and_preprocess_dicom(path, label):\n # a) Read DICOM to get pixel matrix\n ds = pydicom.dcmread(path.numpy().decode('utf-8'))\n img = ds.pixel_array\n # b) Convert to three channels (repeat grayscale to RGB)\n img = np.stack([img] * 3, axis=-1)\n # c) Normalize to [0, 1]\n img = img.astype(np.float32) / np.max(img)\n # d) Resize to the same IMAGE_SIZE used during training\n img = tf.image.resize(img, IMAGE_SIZE)\n return img, label\n\n# 4. Wrap using tf.data.Dataset.from_tensor_slices + map \ndef tf_load_and_preprocess_dicom(path, label):\n img, lbl = tf.py_function(\n load_and_preprocess_dicom,\n inp=[path, label],\n Tout=[tf.float32, tf.int32]\n )\n img.set_shape((*IMAGE_SIZE, 3))\n lbl.set_shape(())\n return img, lbl\n\ntest_ds = tf.data.Dataset.from_tensor_slices((test_paths, test_labels))\ntest_ds = (test_ds\n .map(tf_load_and_preprocess_dicom, num_parallel_calls=tf.data.AUTOTUNE)\n .batch(BATCH_SIZE)\n .prefetch(tf.data.AUTOTUNE)\n )\n\n# 5. Evaluate\nresults = model.evaluate(test_ds)\nprint(\"Test loss, Test accuracy:\", results)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-11T03:34:08.336935Z","iopub.execute_input":"2025-05-11T03:34:08.337228Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.metrics import roc_auc_score, precision_recall_fscore_support, confusion_matrix\n\n# 1. Predict on the entire test dataset\ny_true = []\ny_pred = []\nfor imgs, labels in test_ds:\n preds = model.predict(imgs, verbose=0) # Set verbose=0 to avoid progress bar output\n y_true.extend(labels.numpy())\n y_pred.extend(preds.flatten())\n\ny_true = np.array(y_true)\ny_pred = np.array(y_pred)\n\n# 2. Calculate AUC for binary classification\nauc = roc_auc_score(y_true, y_pred)\n\n# 3. Calculate precision, recall, F1, and confusion matrix using a threshold of 0.5\ny_hat = (y_pred >= 0.5).astype(int)\nprecision, recall, f1, _ = precision_recall_fscore_support(y_true, y_hat, average='binary')\ncm = confusion_matrix(y_true, y_hat)\n\nprint(f\"Test AUC: {auc:.4f}\")\nprint(f\"Precision: {precision:.4f}, Recall: {recall:.4f}, F1: {f1:.4f}\")\nprint(\"Confusion matrix:\\n\", cm)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-11T03:11:36.298287Z","iopub.execute_input":"2025-05-11T03:11:36.298845Z","iopub.status.idle":"2025-05-11T03:22:47.641158Z","shell.execute_reply.started":"2025-05-11T03:11:36.298821Z","shell.execute_reply":"2025-05-11T03:22:47.640372Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Adjust the learning rate 1 to 40 rounds -5\n# -*- coding: utf-8 -*-\n\"\"\"\nVariable size image training example (Kaggle) - Resize + Multithreading + Distributed Strategy:\n- Global resize to smaller dimensions (e.g., 224×224) to enhance parallelism\n- Supports AMP (mixed precision) & XLA\n- Multithreaded configuration\n- Multi-GPU distributed strategy (MirroredStrategy)\n\"\"\"\nimport os\nimport glob\nimport time\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dense, Dropout\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\n\n# ---------- Hyperparameters ----------\nIMAGE_SIZE = (224, 224) # Target resize dimensions\nBATCH_SIZE = 16 # Increase batch size for better GPU utilization\nEPOCHS = 40 # Increased epochs for better training\nLEARNING_RATE = 1e-5 # Initial learning rate\nAUTOTUNE = tf.data.AUTOTUNE\n\n# ---------- Mixed Precision & XLA ----------\nfrom tensorflow.keras import mixed_precision\nmixed_precision.set_global_policy('mixed_float16') # Enable mixed precision\ntf.config.optimizer.set_jit(True) # Enable XLA JIT\n\n# ---------- Distributed Strategy ----------\nstrategy = tf.distribute.MirroredStrategy()\nprint(f\"Number of GPUs: {strategy.num_replicas_in_sync}\")\n\nwith strategy.scope():\n base = MobileNetV2(input_shape=(*IMAGE_SIZE, 3), include_top=False, weights=None)\n base.trainable = True # Unfreeze all layers\n for layer in base.layers[-30:]:\n layer.trainable = True # Unfreeze only the last 30 layers\n\n model = Sequential([\n base,\n GlobalAveragePooling2D(),\n Dropout(0.5), # Add Dropout\n Dense(128, activation='relu'), # Increase the number of units\n Dense(1, activation='sigmoid', dtype='float32')\n ])\n opt = Adam(learning_rate=LEARNING_RATE)\n model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])\n\n# ---------- Read Labels & Paths ----------\nlabels_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv',\n usecols=['patientId', 'Target']\n)\ninfo_df = pd.read_csv(\n '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv',\n usecols=['patientId', 'class']\n)\ndf = pd.merge(labels_df, info_df, on='patientId', how='left')\n\nimage_dir = '/kaggle/input/rsna-feature/'\npaths, targets = [], []\nfor pid in df['patientId'].unique():\n files = glob.glob(os.path.join(image_dir, f\"{pid}_feature*.png\"))\n if not files:\n continue\n paths.append(files[0])\n targets.append(int(labels_df.loc[labels_df.patientId == pid, 'Target'].iloc[0]))\n\nprint(f\"Found a total of {len(paths)} unique patientId images\")\n\n# ---------- Dataset Pipeline ----------\n# Data Augmentation\ndef load_and_preprocess(path, label):\n img = tf.io.read_file(path)\n img = tf.image.decode_png(img, channels=3)\n img = tf.image.resize(img, IMAGE_SIZE)\n\n # Data Augmentation\n img = tf.image.random_flip_left_right(img)\n img = tf.image.random_brightness(img, max_delta=0.1)\n img = tf.image.random_contrast(img, 0.1, 0.2)\n img = tf.image.random_flip_up_down(img) # Add vertical flip\n img = tf.image.random_crop(img, size=[IMAGE_SIZE[0] - 10, IMAGE_SIZE[1] - 10, 3]) # Random crop\n\n img = tf.image.convert_image_dtype(img, tf.float32)\n return img, label\n\ndataset = tf.data.Dataset.from_tensor_slices((paths, targets))\n# Shuffle & split\ndataset = dataset.shuffle(buffer_size=len(paths))\ntrain_size = int(0.8 * len(paths))\ntrain_ds = dataset.take(train_size)\nval_ds = dataset.skip(train_size)\n\n# Preprocessing pipeline\ndef prepare(ds):\n return (ds.map(load_and_preprocess, num_parallel_calls=AUTOTUNE)\n .batch(BATCH_SIZE)\n .cache()\n .prefetch(AUTOTUNE))\n\ntrain_ds = prepare(train_ds)\nval_ds = prepare(val_ds)\n\n# ---------- Train ----------\nstart = time.time()\n# Learning rate schedule\nlr_schedule = tf.keras.callbacks.LearningRateScheduler(lambda epoch: 1e-3 * 0.1**(epoch // 10))\nmodel.fit(train_ds, epochs=EPOCHS, validation_data=val_ds, callbacks=[lr_schedule])\nprint(f\"Training complete, duration: {time.time() - start:.2f} seconds\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import glob\nimport os\nimport numpy as np\nimport pydicom\nimport tensorflow as tf\n\n# Assuming labels_df already exists with structure ['patientId', 'Target']\ntest_dir = '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images/'\n\n# 1. Collect test DICOM paths\ntest_paths = glob.glob(os.path.join(test_dir, '*.dcm'))\nprint(f\"Found {len(test_paths)} test DICOM files\")\n\n# 2. Extract patientId from filenames and map to labels\ntest_labels = []\nfor p in test_paths:\n # Example filename: 0000a175-0e68-4ca4-b1af-167204a7e0bc.dcm\n pid = os.path.basename(p).replace('.dcm', '')\n # Skip if this test patient's ID is not in the training labels\n if pid not in labels_df.patientId.values:\n test_labels.append(0) # Assign a default label or handle as needed\n else:\n test_labels.append(int(labels_df.loc[labels_df.patientId == pid, 'Target'].iloc[0]))\n\n# 3. Define DICOM preprocessing function\ndef load_and_preprocess_dicom(path, label):\n # a) Read DICOM to get pixel matrix\n ds = pydicom.dcmread(path.numpy().decode('utf-8'))\n img = ds.pixel_array\n # b) Convert to three channels (repeat grayscale to RGB)\n img = np.stack([img] * 3, axis=-1)\n # c) Normalize to [0, 1]\n img = img.astype(np.float32) / np.max(img)\n # d) Resize to the same IMAGE_SIZE used during training\n img = tf.image.resize(img, IMAGE_SIZE)\n return img, label\n\n# 4. Wrap using tf.data.Dataset.from_tensor_slices + map \ndef tf_load_and_preprocess_dicom(path, label):\n img, lbl = tf.py_function(\n load_and_preprocess_dicom,\n inp=[path, label],\n Tout=[tf.float32, tf.int32]\n )\n img.set_shape((*IMAGE_SIZE, 3))\n lbl.set_shape(())\n return img, lbl\n\ntest_ds = tf.data.Dataset.from_tensor_slices((test_paths, test_labels))\ntest_ds = (test_ds\n .map(tf_load_and_preprocess_dicom, num_parallel_calls=tf.data.AUTOTUNE)\n .batch(BATCH_SIZE)\n .prefetch(tf.data.AUTOTUNE)\n )\n\n# 5. Evaluate\nresults = model.evaluate(test_ds)\nprint(\"Test loss, Test accuracy:\", results)","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null}]}