{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"sourceType":"competition"}],"dockerImageVersionId":31011,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Melanoma Classification: Streamlined Model Training (GPU)\n#\nThis notebook focuses on training a MobileNetV2 model for melanoma classification using TFRecords on a Kaggle GPU environment. It's a simplified version of the original Notebook III.","metadata":{"_uuid":"2a2ec211-de2e-4df5-8774-92fdd486b126","_cell_guid":"a9131435-c891-46b5-8938-e662955fc7a8","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"## 1. Setup and Imports","metadata":{"_uuid":"61a0ec88-9b0c-4fbe-a9a0-590b60773445","_cell_guid":"f2d82295-4775-48a9-85b8-e1c710722b21","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"import os\nimport re\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom kaggle_datasets import KaggleDatasets\nfrom functools import partial\nfrom sklearn.model_selection import train_test_split\nfrom tqdm.notebook import tqdm # Use tqdm.notebook for Kaggle\nimport gc","metadata":{"_uuid":"e90412d0-2e87-437a-bd14-88b27941a1d0","_cell_guid":"4b5b8ff9-25b6-430e-89b0-592456a7f395","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-05-08T00:17:27.957710Z","iopub.execute_input":"2025-05-08T00:17:27.958025Z","iopub.status.idle":"2025-05-08T00:17:40.604733Z","shell.execute_reply.started":"2025-05-08T00:17:27.958005Z","shell.execute_reply":"2025-05-08T00:17:40.604145Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Configuration","metadata":{"_uuid":"ca6db9ee-89a8-4df8-9b48-23f09f22a60b","_cell_guid":"357fa05e-8dc5-46e3-89f2-157effda2dc2","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# Environment Setup\nprint(\"TensorFlow Version:\", tf.__version__)\ngpus = tf.config.list_physical_devices('GPU')\nif gpus:\n    try:\n        # Currently, memory growth needs to be the same across GPUs\n        for gpu in gpus:\n            tf.config.experimental.set_memory_growth(gpu, True)\n        logical_gpus = tf.config.list_logical_devices('GPU')\n        print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n    except RuntimeError as e:\n        # Memory growth must be set before GPUs have been initialized\n        print(e)\nelse:\n    print(\"No GPU detected. Running on CPU.\")\n\n# TPU setup removed, using default strategy for GPU/CPU\nstrategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)\n\n# Constants\nGCS_PATH = KaggleDatasets().get_gcs_path(\"siim-isic-melanoma-classification\")\nKAGGLE_PATH = '/kaggle/input/siim-isic-melanoma-classification'\n# Adjust BATCH_SIZE for GPU memory (original was 16 * replicas for TPU)\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync # Start with 32 or 64 per replica (GPU)\nIMAGE_SIZE = [256, 256] # Target size for the model\nAUTOTUNE = tf.data.experimental.AUTOTUNE\n\nprint(\"Batch Size:\", BATCH_SIZE)\nprint(\"Image Size:\", IMAGE_SIZE)\nprint(\"GCS Path:\", GCS_PATH)","metadata":{"_uuid":"6e12314d-29e0-4ee5-acb7-24ccf12f04f0","_cell_guid":"b1ca9e3e-dfbb-49e8-b731-7df44ebeb4ba","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-05-08T00:17:40.605777Z","iopub.execute_input":"2025-05-08T00:17:40.606343Z","iopub.status.idle":"2025-05-08T00:17:42.033770Z","shell.execute_reply.started":"2025-05-08T00:17:40.606323Z","shell.execute_reply":"2025-05-08T00:17:42.033149Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. Load Data Paths and Split","metadata":{"_uuid":"85ff6702-f9d9-40c9-ae12-dabefd5b08bb","_cell_guid":"00093f7b-f5ad-4e86-a3f4-d1ac77fdbc4f","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# Get TFRecord file paths\nALL_TRAINING_FILENAMES = tf.io.gfile.glob(KAGGLE_PATH + \"/tfrecords/train*.tfrec\")\nTEST_FILENAMES = tf.io.gfile.glob(KAGGLE_PATH + \"/tfrecords/test*.tfrec\")\n\n# Split training files for validation\nTRAIN_FILENAMES, VAL_FILENAMES = train_test_split(\n    ALL_TRAINING_FILENAMES,\n    test_size=0.1, # 10% for validation\n    random_state=42 # Use a fixed random state for reproducibility\n)\n\nprint(\"Number of training TFRecord files:\", len(TRAIN_FILENAMES))\nprint(\"Number of validation TFRecord files:\", len(VAL_FILENAMES))\nprint(\"Number of test TFRecord files:\", len(TEST_FILENAMES))","metadata":{"_uuid":"add50565-047a-4426-a72f-fc5d4f249e59","_cell_guid":"f83cedea-10f3-43e9-8b1c-ca2979138c0d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-05-08T00:17:42.034466Z","iopub.execute_input":"2025-05-08T00:17:42.034739Z","iopub.status.idle":"2025-05-08T00:17:42.077803Z","shell.execute_reply.started":"2025-05-08T00:17:42.034712Z","shell.execute_reply":"2025-05-08T00:17:42.077273Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. Data Pipeline Functions","metadata":{"_uuid":"02f4cdd3-1675-469a-83f6-fafd7eb7c9f3","_cell_guid":"9df21b29-c2d9-4b2b-b572-d3f5e0e3f8a2","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def decode_image(image_data):\n    \"\"\"Decodes JPEG image, casts to float32, and normalizes.\"\"\"\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # normalize to [0,1]\n    # No initial resize here, will be done in augmentation/preprocessing step\n    return image\n\ndef read_tfrecord(example, labeled):\n    \"\"\"Parses a single TFRecord example.\"\"\"\n    if labeled:\n        tfrecord_format = {\n            \"image\": tf.io.FixedLenFeature([], tf.string),\n            \"target\": tf.io.FixedLenFeature([], tf.int64)\n        }\n    else:\n        tfrecord_format = {\n            \"image\": tf.io.FixedLenFeature([], tf.string),\n            \"image_name\": tf.io.FixedLenFeature([], tf.string)\n        }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    if labeled:\n        label = tf.cast(example['target'], tf.int32)\n        return image, label\n    else:\n        image_name = example['image_name']\n        return image, image_name\n\ndef preprocess_image(image, label=None, is_training=False):\n    \"\"\"Resizes and optionally augments the image.\"\"\"\n    image = tf.image.resize(image, IMAGE_SIZE)\n    if is_training:\n        # Basic augmentation\n        image = tf.image.random_flip_left_right(image)\n        # image = tf.image.random_flip_up_down(image) # Optional\n        # image = tf.image.random_saturation(image, 0.8, 1.2) # Optional\n        # image = tf.image.random_brightness(image, 0.1) # Optional\n        # image = tf.image.random_contrast(image, 0.8, 1.2) # Optional\n    if label is None:\n        return image\n    else:\n        return image, label\n\n\ndef load_dataset(filenames, labeled=True, ordered=False, is_training=False):\n    \"\"\"Loads TFRecords, preprocesses, and batches the dataset.\"\"\"\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)\n    # Apply preprocessing and augmentation\n    dataset = dataset.map(partial(preprocess_image, is_training=is_training), num_parallel_calls=AUTOTUNE)\n\n    if is_training:\n        dataset = dataset.shuffle(2048) # Shuffle buffer size\n        dataset = dataset.repeat()\n\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE) # prefetch next batch while training\n    return dataset","metadata":{"_uuid":"8cc6e355-ebc9-430a-a77b-bdb40deb6184","_cell_guid":"c3815022-3879-472b-8405-8f273bdf5c27","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-05-08T00:17:42.079094Z","iopub.execute_input":"2025-05-08T00:17:42.079321Z","iopub.status.idle":"2025-05-08T00:17:42.087800Z","shell.execute_reply.started":"2025-05-08T00:17:42.079305Z","shell.execute_reply":"2025-05-08T00:17:42.087064Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. Create Datasets and Calculate Steps","metadata":{"_uuid":"de3915aa-2030-4d56-a1b9-ba8e2bbf5b44","_cell_guid":"a71afb81-4396-4df9-8763-2da9b3782a55","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"train_dataset = load_dataset(TRAIN_FILENAMES, labeled=True, ordered=False, is_training=True)\nval_dataset = load_dataset(VAL_FILENAMES, labeled=True, ordered=False, is_training=False) # No augmentation/repeat/shuffle for val\ntest_dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=True, is_training=False) # Ordered for submission\n\n# Calculate number of images and steps\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nnum_training_images = count_data_items(TRAIN_FILENAMES)\nnum_validation_images = count_data_items(VAL_FILENAMES)\nnum_test_images = count_data_items(TEST_FILENAMES)\n\nSTEPS_PER_EPOCH_TRAIN = num_training_images // BATCH_SIZE\n\nprint(f\"Training images: {num_training_images}, Steps/epoch: {STEPS_PER_EPOCH_TRAIN}\")\nprint(f\"Test images: {num_test_images}\")","metadata":{"_uuid":"5cde4bfb-4c7e-418b-b7a7-958c02d0e4b9","_cell_guid":"19bb766e-6185-4747-a910-82410334bba6","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-05-08T00:17:42.088539Z","iopub.execute_input":"2025-05-08T00:17:42.088733Z","iopub.status.idle":"2025-05-08T00:17:42.398955Z","shell.execute_reply.started":"2025-05-08T00:17:42.088718Z","shell.execute_reply":"2025-05-08T00:17:42.398195Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6. (Optional) Visualize a Batch","metadata":{"_uuid":"b3c4a18b-cf0e-4bef-b610-7fe4810dcb3e","_cell_guid":"562fd7de-cb71-4ab3-87eb-a8344bb7aabf","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# def show_batch(image_batch, label_batch):\n#     plt.figure(figsize=(15, 15))\n#     for n in range(min(8, BATCH_SIZE)): # Show up to 8 images\n#         ax = plt.subplot(2, 4, n + 1)\n#         plt.imshow(image_batch[n])\n#         if label_batch[n] == 0:\n#             plt.title(\"BENIGN\")\n#         else:\n#             plt.title(\"MALIGNANT\")\n#         plt.axis(\"off\")\n#     plt.tight_layout()\n#     plt.show()\n\n# # Fetch a batch from the training dataset to visualize\n# image_batch, label_batch = next(iter(train_dataset))\n# show_batch(image_batch.numpy(), label_batch.numpy())\n\n# # Clean up memory\n# del image_batch, label_batch\n# gc.collect()","metadata":{"_uuid":"3ab28763-4b55-4eab-bf32-b97dc57fbd12","_cell_guid":"f6d9ed08-8119-4ee5-b91b-51083f22f15e","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-05-08T00:17:42.399804Z","iopub.execute_input":"2025-05-08T00:17:42.400035Z","iopub.status.idle":"2025-05-08T00:17:42.403632Z","shell.execute_reply.started":"2025-05-08T00:17:42.400018Z","shell.execute_reply":"2025-05-08T00:17:42.402921Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7. Model Building (MobileNetV2)","metadata":{"_uuid":"2557c140-d721-45a8-abc4-79439e4a1708","_cell_guid":"1837e50f-fd6d-4b72-a0fc-85333cd8fa24","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# --- Class Weights and Bias Initialization (Requires train.csv) ---\n# Load train CSV briefly to calculate weights\ntrain_df = pd.read_csv(\"/kaggle/input/siim-isic-melanoma-classification/train.csv\")\nmalignant_count = train_df['target'].sum()\ntotal_count = len(train_df)\nbenign_count = total_count - malignant_count\n\nweight_malignant = (total_count / malignant_count) / 2.0\nweight_benign = (total_count / benign_count) / 2.0\nclass_weight = {0: weight_benign, 1: weight_malignant}\ninitial_bias = np.log([malignant_count / benign_count]) # Calculate initial bias\n\nprint(f\"Benign cases: {benign_count}, Malignant cases: {malignant_count}\")\nprint(f\"Weight for class 0 (Benign): {class_weight[0]:.2f}\")\nprint(f\"Weight for class 1 (Malignant): {class_weight[1]:.2f}\")\nprint(f\"Initial bias: {initial_bias[0]:.2f}\")\n\ndel train_df # Free memory\ngc.collect()\n# --- End Class Weights ---\n\n\n# --- Model Definition ---\n# No strategy.scope() needed for default strategy\nbase_model = tf.keras.applications.MobileNetV2(\n    input_shape=(*IMAGE_SIZE, 3),\n    include_top=False, # Exclude the final classification layer\n    weights='imagenet' # Use pre-trained ImageNet weights\n)\nbase_model.trainable = False # Freeze the base model layers initially\n\nmodel = tf.keras.Sequential([\n    base_model,\n    tf.keras.layers.GlobalAveragePooling2D(),\n    tf.keras.layers.Dense(20, activation=\"relu\"), # Smaller dense layers from original\n    tf.keras.layers.Dropout(0.4),                 # Dropout for regularization\n    tf.keras.layers.Dense(10, activation=\"relu\"),\n    tf.keras.layers.Dropout(0.3),\n    tf.keras.layers.Dense(1, activation='sigmoid', # Output layer for binary classification\n                          bias_initializer=tf.keras.initializers.Constant(initial_bias)) # Set initial bias\n])\n\n# Compile the model\nmodel.compile(\n    optimizer=tf.keras.optimizers.Adam(learning_rate=1e-3), # Standard Adam optimizer\n    loss='binary_crossentropy', # Suitable for binary classification\n    metrics=[tf.keras.metrics.AUC(name='auc')] # Competition metric\n)\n\nmodel.summary()\n# --- End Model Definition ---","metadata":{"_uuid":"a52f5505-55ca-4a3a-b5d2-333572573227","_cell_guid":"8d243858-e942-4a33-9a66-51f301ebb414","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-05-08T00:17:42.404423Z","iopub.execute_input":"2025-05-08T00:17:42.404888Z","iopub.status.idle":"2025-05-08T00:17:46.656735Z","shell.execute_reply.started":"2025-05-08T00:17:42.404862Z","shell.execute_reply":"2025-05-08T00:17:46.656165Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 8. Define Callbacks","metadata":{"_uuid":"bee54ab6-acf3-47d7-9a6d-bdc89c3dd799","_cell_guid":"398fc823-8054-4b5e-a475-e02cbdac7139","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# Callbacks for training\ncallback_early_stopping = tf.keras.callbacks.EarlyStopping(\n    monitor='val_auc', # Monitor validation AUC\n    patience=15,         # Stop after 15 epochs with no improvement\n    mode='max',         # Maximize AUC\n    verbose=1,\n    restore_best_weights=True # Restore weights from the epoch with the best val_auc\n)\n\ncallback_lr_reduce = tf.keras.callbacks.ReduceLROnPlateau(\n    monitor='val_auc', # Monitor validation AUC\n    factor=0.1,        # Reduce LR by factor of 10\n    patience=5,        # Reduce after 5 epochs with no improvement\n    mode='max',        # Maximize AUC\n    verbose=1,\n    min_lr=1e-6        # Minimum learning rate\n)\n\n# Checkpoint saving the best weights based on validation AUC\ncallback_checkpoint = tf.keras.callbacks.ModelCheckpoint(\n    \"melanoma_best.weights.h5\", # File path\n    monitor='val_auc',          # Monitor validation AUC\n    mode='max',                 # Maximize AUC\n    save_best_only=True,        # Only save the best model\n    save_weights_only=True,     # Save only the weights\n    verbose=0                   # Less verbose output\n)","metadata":{"_uuid":"e8074eaa-64b1-4881-b37a-07731c743c7c","_cell_guid":"3bc9bdce-87d0-4ef6-9e48-2b9129b3f9b4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-05-08T00:17:46.657410Z","iopub.execute_input":"2025-05-08T00:17:46.657719Z","iopub.status.idle":"2025-05-08T00:17:46.662493Z","shell.execute_reply.started":"2025-05-08T00:17:46.657700Z","shell.execute_reply":"2025-05-08T00:17:46.661805Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 9. Train the Model","metadata":{"_uuid":"3476c799-4ff9-4327-b4b9-c0ac3b383b95","_cell_guid":"a3d586ca-244a-44c0-97c7-392c2afd7fa3","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"EPOCHS = 40 # Set a reasonable number of epochs, EarlyStopping will likely stop it sooner\n\nhistory = model.fit(\n    train_dataset,\n    epochs=EPOCHS,\n    steps_per_epoch=STEPS_PER_EPOCH_TRAIN,\n    validation_data=val_dataset,\n    validation_steps=None,\n    callbacks=[callback_early_stopping, callback_lr_reduce, callback_checkpoint],\n    class_weight=class_weight, # Use calculated class weights\n    verbose=1 # Show progress bar\n)","metadata":{"_uuid":"ca87afba-5496-4e2f-b260-129a7a6d7b72","_cell_guid":"c2cb3d36-a304-4917-994e-5bd5adb35a45","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-05-08T00:56:21.667476Z","iopub.execute_input":"2025-05-08T00:56:21.667871Z","iopub.status.idle":"2025-05-08T01:14:32.218385Z","shell.execute_reply.started":"2025-05-08T00:56:21.667854Z","shell.execute_reply":"2025-05-08T01:14:32.217755Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 10. Plot Training History","metadata":{"_uuid":"73ba11f2-06c6-442c-917c-4c6a1b916128","_cell_guid":"76c6f951-a193-4288-aadd-d81cf4b206c2","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def plot_history(history):\n    hist = history.history\n    epochs = range(1, len(hist['loss']) + 1)\n\n    plt.figure(figsize=(12, 5))\n\n    # Plot Loss\n    plt.subplot(1, 2, 1)\n    plt.plot(epochs, hist['loss'], 'bo-', label='Training loss')\n    plt.plot(epochs, hist['val_loss'], 'ro-', label='Validation loss')\n    plt.title('Training and Validation Loss')\n    plt.xlabel('Epochs')\n    plt.ylabel('Loss')\n    plt.legend()\n    plt.grid(True)\n\n    # Plot AUC\n    plt.subplot(1, 2, 2)\n    plt.plot(epochs, hist['auc'], 'bo-', label='Training AUC')\n    plt.plot(epochs, hist['val_auc'], 'ro-', label='Validation AUC')\n    plt.title('Training and Validation AUC')\n    plt.xlabel('Epochs')\n    plt.ylabel('AUC')\n    plt.legend()\n    plt.grid(True)\n\n    plt.tight_layout()\n    plt.show()\n\n# Plot the results (EarlyStopping restores best weights)\nplot_history(history)","metadata":{"_uuid":"aa7d4148-6863-420e-87c2-fb9e593d1d2c","_cell_guid":"6631da1f-2f74-4e85-bf28-fec49c2ae21a","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-05-08T01:14:32.219193Z","iopub.execute_input":"2025-05-08T01:14:32.219444Z","iopub.status.idle":"2025-05-08T01:14:32.573602Z","shell.execute_reply.started":"2025-05-08T01:14:32.219428Z","shell.execute_reply":"2025-05-08T01:14:32.572894Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 11. Evaluate Model on Full Validation Set\n#\nAfter training, and with EarlyStopping having restored the best weights (or loading them from the checkpoint),\nwe evaluate the model's performance on the *entire* validation dataset.","metadata":{"_uuid":"e249669c-72cd-4ed0-a109-994e4309ed32","_cell_guid":"d530154d-0dd5-4850-8867-e7f81859dd25","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score, accuracy_score, precision_score, recall_score, f1_score, confusion_matrix\nimport seaborn as sns # For a prettier confusion matrix\n\n# --- Important: Ensure val_dataset for evaluation covers all data and is ordered ---\n# If your original val_dataset was created with ordered=False, it's best to create\n# a new one for evaluation to ensure labels and predictions align perfectly.\n# Also, ensure we use all validation images.\n\nprint(\"Re-creating validation dataset for full evaluation (ordered)...\")\n# Use the same BATCH_SIZE as training, or adjust if needed for prediction memory/speed\n# For evaluation, is_training should be False, and ordered should be True.\nEVAL_BATCH_SIZE = BATCH_SIZE # Can be different from training BATCH_SIZE if desired\nval_eval_dataset = load_dataset(VAL_FILENAMES, labeled=True, ordered=True, is_training=False)\n\n# Calculate the correct number of steps to cover the entire validation set\nnum_validation_images = count_data_items(VAL_FILENAMES)\nVAL_EVAL_STEPS = (num_validation_images + EVAL_BATCH_SIZE - 1) // EVAL_BATCH_SIZE\nprint(f\"Total validation images: {num_validation_images}\")\nprint(f\"Evaluation batch size: {EVAL_BATCH_SIZE}\")\nprint(f\"Validation evaluation steps: {VAL_EVAL_STEPS}\")\n\n# --- Get True Labels ---\nprint(\"Extracting true labels from the validation dataset...\")\ny_true_val = []\n# .take(VAL_EVAL_STEPS) ensures we iterate through the entire dataset once\nfor images, labels in tqdm(val_eval_dataset.take(VAL_EVAL_STEPS), total=VAL_EVAL_STEPS):\n    y_true_val.extend(labels.numpy())\ny_true_val = np.array(y_true_val)\nprint(f\"Extracted {len(y_true_val)} true labels.\")\n\n# --- Get Model Predictions (Probabilities) ---\n# The model should already have the best weights loaded if EarlyStopping's restore_best_weights=True\n# Or, if you saved weights: model.load_weights(\"melanoma_best.weights.h5\")\nprint(\"Generating predictions on the full validation dataset...\")\n# model.predict will iterate through the dataset.\n# Providing 'steps' ensures it processes the correct amount of data if the dataset could be infinite (though ours is not here).\ny_pred_probs_val = model.predict(val_eval_dataset, steps=VAL_EVAL_STEPS, verbose=1)\n# Ensure predictions match the number of true labels\ny_pred_probs_val = y_pred_probs_val[:len(y_true_val)] # Trim if predict gives more due to batching\nprint(f\"Generated {len(y_pred_probs_val)} predictions.\")\n\n\n# --- Calculate Metrics ---\n# For AUC, we use the probabilities\nauc_val = roc_auc_score(y_true_val, y_pred_probs_val)\n\n# For other metrics, we need binary predictions (threshold at 0.5)\nTHRESHOLD = 0.5\ny_pred_binary_val = (y_pred_probs_val > THRESHOLD).astype(int).flatten() # flatten in case of (N,1) shape\n\naccuracy_val = accuracy_score(y_true_val, y_pred_binary_val)\nprecision_val = precision_score(y_true_val, y_pred_binary_val)\nrecall_val = recall_score(y_true_val, y_pred_binary_val)\nf1_val = f1_score(y_true_val, y_pred_binary_val)\ncm_val = confusion_matrix(y_true_val, y_pred_binary_val)\n\nprint(\"\\n--- Validation Set Evaluation Results ---\")\nprint(f\"AUC: {auc_val:.4f}\")\nprint(f\"Accuracy: {accuracy_val:.4f}\")\nprint(f\"Precision: {precision_val:.4f}\")\nprint(f\"Recall: {recall_val:.4f}\")\nprint(f\"F1-Score: {f1_val:.4f}\")\n\n# Plot Confusion Matrix\nplt.figure(figsize=(8, 6))\nsns.heatmap(cm_val, annot=True, fmt='d', cmap='Blues',\n            xticklabels=['Benign (0)', 'Malignant (1)'],\n            yticklabels=['Benign (0)', 'Malignant (1)'])\nplt.title('Confusion Matrix - Validation Set')\nplt.xlabel('Predicted Label')\nplt.ylabel('True Label')\nplt.show()\n\n# ## End of Notebook","metadata":{"_uuid":"bb4ec626-f5a3-4ad7-b5b5-daef0532e460","_cell_guid":"0a953e5d-f123-4dbf-a80c-72e9cd0afd07","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-05-08T00:55:42.439410Z","iopub.execute_input":"2025-05-08T00:55:42.439591Z","iopub.status.idle":"2025-05-08T00:56:21.666227Z","shell.execute_reply.started":"2025-05-08T00:55:42.439577Z","shell.execute_reply":"2025-05-08T00:56:21.665605Z"}},"outputs":[],"execution_count":null}]}