{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"2499f17f-9b68-45d3-b995-4b8ffd6f333d","cell_type":"markdown","source":"# Task 3 - XAI Checkpoint Audit\n\nThis notebook implements Task 3 in\n`yeu_cau_dieu_chinh_thuc_nghiem_camera_ready.md`.\n\nIt does **not** train or fine-tune a model. It identifies the EfficientNet-B3\ncheckpoints used by channel occlusion and Grad-CAM, evaluates each checkpoint on\nits exact test split, records checkpoint hashes and test sample IDs, and explains\nwhy a single XAI checkpoint does not equal the three-run mean in the main table.\n\n","metadata":{}},{"id":"a89ee4b2-fc5d-466c-9520-d6201b779111","cell_type":"code","source":"# ============================================================\n# CELL 1: Imports and audit configuration\n# ============================================================\nimport glob\nimport hashlib\nimport json\nimport logging\nimport os\nimport sys\nfrom pathlib import Path\n\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\nos.environ[\"OMP_NUM_THREADS\"] = \"2\"\nos.environ[\"MKL_NUM_THREADS\"] = \"2\"\n\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom sklearn.metrics import f1_score\nfrom tqdm.auto import tqdm\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import datasets, models, transforms\n\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nAMP_ENABLED = DEVICE.type == \"cuda\"\n\nDATASETS_TO_RUN = [\"microsoft_rgb\", \"malimg_rgb\"]\n\nDATACONFIG_ROOT = \"/kaggle/input/datasets/vnhtbo/dataconfig\"\nMICROSOFT_CKPT_OVERRIDE = f\"{DATACONFIG_ROOT}/best_eb3_teacher_run1_microsoft.pt\"\nMALIMG_CKPT_OVERRIDE = f\"{DATACONFIG_ROOT}/best_eb3_teacher_run1_malimg.pt\"\nMICROSOFT_SPLIT_OVERRIDE = f\"{DATACONFIG_ROOT}/split_microsoft_rgb.json\"\nMALIMG_SPLIT_OVERRIDE = f\"{DATACONFIG_ROOT}/split_malimg_rgb.json\"\n\n# Best epochs are taken from the corresponding run-1 histories in dataconfig.\nMICROSOFT_BEST_EPOCH_OVERRIDE = 20\nMALIMG_BEST_EPOCH_OVERRIDE = 8\nREQUIRE_COMPLETE_EPOCH_METADATA = True\n\nSELECTION_RULE = (\n    \"highest validation F1-Macro within fixed training run 1; \"\n    \"checkpoint selected before test evaluation\"\n)\nARCHITECTURE = \"EfficientNet-B3\"\nTRAINING_RUN = 1\nSPLIT_SEED = 42\nTRAINING_SEED = SPLIT_SEED + TRAINING_RUN\nVAL_RATIO = 0.15\nTEST_RATIO = 0.15\nIMG_SIZE = 300\nBATCH_SIZE = 32\nNUM_WORKERS = 2\n\nDATASET_CONFIGS = {\n    \"microsoft_rgb\": {\n        \"dataset_label\": \"BIG-2015\",\n        \"description\": \"Microsoft BIG-2015\",\n        \"path\": \"/kaggle/input/datasets/vnhtbo/microsoft/train_rgb/train_rgb\",\n        \"csv_path\": \"/kaggle/input/competitions/malware-classification/trainLabels.csv\",\n        \"num_classes\": 9,\n        \"checkpoint_override\": MICROSOFT_CKPT_OVERRIDE,\n        \"checkpoint_candidates\": [\n            \"/kaggle/input/datasets/vnhtbo/eb3-train/best_eb3_teacher_run1.pt\",\n            \"/kaggle/working/best_eb3_teacher_run1.pt\",\n        ],\n        \"checkpoint_patterns\": [\n            \"*/eb3-train/best_eb3_teacher_run1.pt\",\n            \"best_eb3_teacher_run1.pt\",\n        ],\n        \"split_override\": MICROSOFT_SPLIT_OVERRIDE,\n        \"split_candidates\": [\n            \"/kaggle/working/checkpoints/split_microsoft_rgb.json\",\n            \"/kaggle/input/datasets/vnhtbo/eb3-train/split_microsoft_rgb.json\",\n        ],\n        \"split_pattern\": \"split_microsoft_rgb.json\",\n        \"best_epoch_override\": MICROSOFT_BEST_EPOCH_OVERRIDE,\n        \"training_result_candidates\": [\n            f\"{DATACONFIG_ROOT}/results_eb3_teacher_microsoft.json\",\n            \"/kaggle/input/datasets/vnhtbo/eb3-train/results_eb3_teacher.json\",\n            \"/kaggle/working/results_eb3_teacher.json\",\n        ],\n        \"training_result_patterns\": [\n            \"*/eb3-train/results_eb3_teacher*.json\",\n        ],\n        \"expected_num_test_samples\": 1630,\n        \"expected_xai_accuracy\": 0.9908,\n        \"expected_xai_f1\": 0.9817,\n        \"epoch_evidence\": (\n            \"results_eb3_teacher_microsoft.json: run 1 maximum validation \"\n            \"F1-Macro at epoch 20\"\n        ),\n    },\n    \"malimg_rgb\": {\n        \"dataset_label\": \"Malimg\",\n        \"description\": \"Malimg\",\n        \"path\": \"/kaggle/input/datasets/dongquan/malimg-rgb/kaggle/working/malimg_rgb\",\n        \"csv_path\": None,\n        \"num_classes\": 25,\n        \"checkpoint_override\": MALIMG_CKPT_OVERRIDE,\n        \"checkpoint_candidates\": [\n            \"/kaggle/input/datasets/vnhtbo/eb3-train-malimg/best_eb3_teacher_run1 (1).pt\",\n            \"/kaggle/input/datasets/vnhtbo/eb3-train-malimg/best_eb3_teacher_run1.pt\",\n        ],\n        \"checkpoint_patterns\": [\n            \"*/eb3-train-malimg/best_eb3_teacher_run1*.pt\",\n        ],\n        \"split_override\": MALIMG_SPLIT_OVERRIDE,\n        \"split_candidates\": [\n            \"/kaggle/working/checkpoints/split_malimg_rgb.json\",\n            \"/kaggle/input/datasets/vnhtbo/eb3-train-malimg/split_malimg_rgb.json\",\n        ],\n        \"split_pattern\": \"split_malimg_rgb.json\",\n        \"best_epoch_override\": MALIMG_BEST_EPOCH_OVERRIDE,\n        \"training_result_candidates\": [\n            f\"{DATACONFIG_ROOT}/results_eb3_teacher_malimg.json\",\n            \"/kaggle/input/datasets/vnhtbo/eb3-train-malimg/results_eb3_teacher.json\",\n            \"/kaggle/input/datasets/vnhtbo/eb3-train-malimg/results_eb3_teacher_malimg.json\",\n            \"/kaggle/working/results_eb3_teacher_malimg.json\",\n        ],\n        \"training_result_patterns\": [\n            \"*/eb3-train-malimg/results*eb3*teacher*.json\",\n        ],\n        \"expected_num_test_samples\": 1400,\n        \"expected_xai_accuracy\": 0.9943,\n        \"expected_xai_f1\": 0.9833,\n        \"epoch_evidence\": (\n            \"results_eb3_teacher_malimg.json: run 1 maximum validation \"\n            \"F1-Macro at epoch 8\"\n        ),\n    },\n}\n\n# Both repository XAI notebooks configure these files and checkpoints:\n# - eb3-xai.ipynb -> channel_attribution_heatmap.png\n# - gradcam.ipynb -> entropy_attention_correlation.png\nATTRIBUTION_SOURCE_NOTEBOOK = \"Src/eb3-xai.ipynb\"\nCORRELATION_SOURCE_NOTEBOOK = \"Src/gradcam.ipynb\"\nATTRIBUTION_FIGURE = \"channel_attribution_heatmap.png\"\nCORRELATION_FIGURE = \"entropy_attention_correlation.png\"\n\nAUDIT_ROOT = Path(\"/kaggle/working/camera_ready_experiment_audit\")\nCHECKPOINT_DIR = AUDIT_ROOT / \"checkpoint\"\nLOG_DIR = AUDIT_ROOT / \"logs\"\nCHECKPOINT_DIR.mkdir(parents=True, exist_ok=True)\nLOG_DIR.mkdir(parents=True, exist_ok=True)\n\nMETADATA_CSV = CHECKPOINT_DIR / \"xai_checkpoint_metadata.csv\"\nEVALUATION_JSON = CHECKPOINT_DIR / \"xai_checkpoint_evaluation.json\"\nPREDICTIONS_CSV = CHECKPOINT_DIR / \"xai_checkpoint_predictions.csv\"\nPROVENANCE_CSV = CHECKPOINT_DIR / \"xai_figure_provenance.csv\"\nCONSISTENCY_REPORT = CHECKPOINT_DIR / \"xai_checkpoint_consistency_report.md\"\nREADME_PATH = CHECKPOINT_DIR / \"README.md\"\nCHECKPOINT_LOG = LOG_DIR / \"checkpoint_evaluation.log\"\n\nTASK2_METADATA_PATH = (\n    AUDIT_ROOT / \"attribution\" / \"channel_occlusion_run_metadata.json\"\n)\n\n\ndef configure_logger():\n    logger = logging.getLogger(\"xai_checkpoint_audit\")\n    logger.setLevel(logging.INFO)\n    for handler in list(logger.handlers):\n        handler.close()\n        logger.removeHandler(handler)\n    formatter = logging.Formatter(\"%(asctime)s | %(levelname)s | %(message)s\")\n    file_handler = logging.FileHandler(CHECKPOINT_LOG, mode=\"a\", encoding=\"utf-8\")\n    file_handler.setFormatter(formatter)\n    stream_handler = logging.StreamHandler(sys.stdout)\n    stream_handler.setFormatter(formatter)\n    logger.addHandler(file_handler)\n    logger.addHandler(stream_handler)\n    return logger\n\n\nLOGGER = configure_logger()\nLOGGER.info(\"Task 3 audit configured on device=%s, AMP=%s\", DEVICE, AMP_ENABLED)\n\n","metadata":{},"outputs":[],"execution_count":null},{"id":"11ec72be-0d50-4a15-a363-8f6100f4720d","cell_type":"markdown","source":"## Provenance preflight\n\nCheckpoint, class count, split, epoch evidence, and test sample IDs are resolved\nbefore evaluation. The audit refuses to label a checkpoint \"best\" without the\nexplicit validation-F1 selection rule recorded above.\n\n","metadata":{}},{"id":"eb0424d0-dff1-4cce-8d38-fe06ddcd126c","cell_type":"code","source":"# ============================================================\n# CELL 2: Hash, path, checkpoint, epoch, dataset, and split helpers\n# ============================================================\ndef sha256_file(path, chunk_size=1024 * 1024):\n    digest = hashlib.sha256()\n    with open(path, \"rb\") as handle:\n        while True:\n            chunk = handle.read(chunk_size)\n            if not chunk:\n                break\n            digest.update(chunk)\n    return digest.hexdigest()\n\n\ndef sha256_json(value):\n    payload = json.dumps(value, sort_keys=True, separators=(\",\", \":\")).encode(\"utf-8\")\n    return hashlib.sha256(payload).hexdigest()\n\n\ndef resolve_file(override, candidates, patterns, label, required=True):\n    if override:\n        if not os.path.isfile(override):\n            raise FileNotFoundError(f\"{label} override does not exist: {override}\")\n        return override\n    for path in candidates:\n        if path and os.path.isfile(path):\n            return path\n    for root in (\"/kaggle/working\", \"/kaggle/input\"):\n        if not os.path.isdir(root):\n            continue\n        for pattern in patterns:\n            matches = sorted(glob.glob(os.path.join(root, \"**\", pattern), recursive=True))\n            if matches:\n                return matches[0]\n    if required:\n        raise FileNotFoundError(\n            f\"Could not resolve {label}. Attach the original artifact or set its override.\"\n        )\n    return None\n\n\ndef unwrap_state_dict(checkpoint):\n    state = checkpoint\n    if isinstance(state, dict) and \"model\" in state:\n        state = state[\"model\"]\n    if isinstance(state, dict) and \"state_dict\" in state:\n        state = state[\"state_dict\"]\n    if not isinstance(state, dict):\n        raise TypeError(\"Checkpoint does not contain a state dictionary\")\n    cleaned = {}\n    for key, value in state.items():\n        if key.startswith(\"module.\"):\n            key = key[len(\"module.\"):]\n        cleaned[key] = value\n    return cleaned\n\n\ndef checkpoint_num_classes(checkpoint_path):\n    checkpoint = torch.load(checkpoint_path, map_location=\"cpu\")\n    state = unwrap_state_dict(checkpoint)\n    weight = state.get(\"classifier.1.weight\")\n    if weight is None or not hasattr(weight, \"shape\") or len(weight.shape) != 2:\n        raise ValueError(f\"Cannot infer classifier shape from {checkpoint_path}\")\n    return int(weight.shape[0])\n\n\ndef resolve_best_epoch(config):\n    if config[\"best_epoch_override\"] is not None:\n        return int(config[\"best_epoch_override\"]), (\n            config[\"epoch_evidence\"] or \"manual best-epoch override\"\n        )\n    result_path = resolve_file(\n        None,\n        config[\"training_result_candidates\"],\n        config[\"training_result_patterns\"],\n        f\"{config['dataset_label']} EB3 training result\",\n        required=False,\n    )\n    if not result_path:\n        return None, \"training result/history unavailable\"\n    with open(result_path, \"r\", encoding=\"utf-8\") as handle:\n        result = json.load(handle)\n    histories = result.get(\"histories\") or result.get(\"run_histories\")\n    if isinstance(histories, list) and len(histories) >= TRAINING_RUN:\n        history = histories[TRAINING_RUN - 1]\n        valid_rows = [\n            row for row in history\n            if isinstance(row, dict) and \"epoch\" in row and \"val_f1\" in row\n        ]\n        if valid_rows:\n            selected = max(valid_rows, key=lambda row: float(row[\"val_f1\"]))\n            return int(selected[\"epoch\"]), (\n                f\"derived from run-1 maximum val_f1 in {result_path}\"\n            )\n    runs = result.get(\"runs\", {})\n    run_record = runs.get(\"run_1\") or runs.get(\"1\")\n    if isinstance(run_record, dict) and run_record.get(\"best_epoch\") is not None:\n        return int(run_record[\"best_epoch\"]), f\"loaded from {result_path}\"\n    return None, f\"best epoch not present in {result_path}\"\n\n\ndef build_dataset_records(dataset_key, config):\n    root = config[\"path\"]\n    if not os.path.isdir(root):\n        raise FileNotFoundError(f\"Dataset directory not found: {root}\")\n    records = []\n    if config[\"csv_path\"]:\n        if not os.path.isfile(config[\"csv_path\"]):\n            raise FileNotFoundError(f\"Label CSV not found: {config['csv_path']}\")\n        labels = pd.read_csv(config[\"csv_path\"])\n        class_values = sorted(int(value) for value in labels[\"Class\"].unique())\n        class_to_index = {value: index for index, value in enumerate(class_values)}\n        classes = [str(value) for value in class_values]\n        existing = set(os.listdir(root))\n        for row in labels.itertuples(index=False):\n            sample_id = str(row.Id)\n            filename = f\"{sample_id}.png\"\n            if filename in existing:\n                records.append({\n                    \"path\": os.path.join(root, filename),\n                    \"sample_id\": sample_id,\n                    \"label_index\": class_to_index[int(row.Class)],\n                })\n    else:\n        base = datasets.ImageFolder(root)\n        classes = list(base.classes)\n        root_path = Path(root)\n        for path, label_index in base.samples:\n            sample_id = str(Path(path).relative_to(root_path).with_suffix(\"\"))\n            records.append({\n                \"path\": path,\n                \"sample_id\": sample_id,\n                \"label_index\": int(label_index),\n            })\n    if not records:\n        raise RuntimeError(f\"No samples found for {dataset_key}\")\n    if len({record[\"sample_id\"] for record in records}) != len(records):\n        raise ValueError(f\"Sample IDs are not unique for {dataset_key}\")\n    return records, classes\n\n\ndef resolve_split(dataset_key, config, num_samples):\n    split_path = resolve_file(\n        config[\"split_override\"],\n        config[\"split_candidates\"],\n        [config[\"split_pattern\"]],\n        f\"{dataset_key} split\",\n        required=False,\n    )\n    if split_path:\n        with open(split_path, \"r\", encoding=\"utf-8\") as handle:\n            split = json.load(handle)\n        source = f\"loaded:{split_path}\"\n    else:\n        n_test = int(num_samples * TEST_RATIO)\n        n_val = int(num_samples * VAL_RATIO)\n        n_train = num_samples - n_val - n_test\n        generator = torch.Generator().manual_seed(SPLIT_SEED)\n        permutation = torch.randperm(num_samples, generator=generator).tolist()\n        split = {\n            \"train\": permutation[:n_train],\n            \"val\": permutation[n_train:n_train + n_val],\n            \"test\": permutation[n_train + n_val:],\n            \"seed\": SPLIT_SEED,\n            \"total\": num_samples,\n        }\n        source = \"deterministically_recreated_from_seed_42_and_70_15_15_ratios\"\n        LOGGER.warning(\"%s split JSON unavailable; recreated deterministically\", dataset_key)\n    if int(split.get(\"total\", num_samples)) != num_samples:\n        raise ValueError(\n            f\"{dataset_key} split total={split.get('total')} does not match {num_samples}\"\n        )\n    test_indices = [int(index) for index in split[\"test\"]]\n    if not test_indices or min(test_indices) < 0 or max(test_indices) >= num_samples:\n        raise ValueError(f\"{dataset_key} split contains invalid test indices\")\n    split_hash = sha256_json(split)\n    snapshot = dict(split)\n    snapshot[\"audit_source\"] = source\n    snapshot_path = CHECKPOINT_DIR / f\"xai_split_{dataset_key}_snapshot.json\"\n    snapshot_path.write_text(json.dumps(snapshot, indent=2) + \"\\n\", encoding=\"utf-8\")\n    return split, source, split_hash, snapshot_path\n\n\nCONTEXTS = {}\nepoch_errors = []\nfor dataset_key in DATASETS_TO_RUN:\n    config = DATASET_CONFIGS[dataset_key]\n    records, classes = build_dataset_records(dataset_key, config)\n    split, split_source, split_hash, split_snapshot = resolve_split(\n        dataset_key, config, len(records)\n    )\n    checkpoint_path = resolve_file(\n        config[\"checkpoint_override\"],\n        config[\"checkpoint_candidates\"],\n        config[\"checkpoint_patterns\"],\n        f\"{dataset_key} EfficientNet-B3 checkpoint\",\n    )\n    output_classes = checkpoint_num_classes(checkpoint_path)\n    if output_classes != config[\"num_classes\"]:\n        raise ValueError(\n            f\"{dataset_key} checkpoint has {output_classes} classes; \"\n            f\"expected {config['num_classes']}: {checkpoint_path}\"\n        )\n    best_epoch, epoch_evidence = resolve_best_epoch(config)\n    if best_epoch is None:\n        epoch_errors.append(\n            f\"{config['dataset_label']}: set its BEST_EPOCH_OVERRIDE or attach the \"\n            \"original EB3 training result/history\"\n        )\n    test_records = [records[index] for index in split[\"test\"]]\n    test_ids_path = CHECKPOINT_DIR / f\"xai_test_sample_ids_{dataset_key}.csv\"\n    pd.DataFrame({\n        \"sample_id\": [record[\"sample_id\"] for record in test_records]\n    }).to_csv(test_ids_path, index=False)\n    CONTEXTS[dataset_key] = {\n        \"config\": config,\n        \"classes\": classes,\n        \"test_records\": test_records,\n        \"test_ids_path\": str(test_ids_path),\n        \"checkpoint_path\": checkpoint_path,\n        \"checkpoint_sha256\": sha256_file(checkpoint_path),\n        \"checkpoint_num_classes\": output_classes,\n        \"best_epoch\": best_epoch,\n        \"epoch_evidence\": epoch_evidence,\n        \"split_source\": split_source,\n        \"split_sha256\": split_hash,\n        \"split_snapshot\": str(split_snapshot),\n    }\n\nif epoch_errors and REQUIRE_COMPLETE_EPOCH_METADATA:\n    raise RuntimeError(\n        \"Best-epoch metadata is required before Task 3 can be called complete:\\n- \"\n        + \"\\n- \".join(epoch_errors)\n    )\n\nprint(pd.DataFrame([\n    {\n        \"dataset\": context[\"config\"][\"dataset_label\"],\n        \"checkpoint\": context[\"checkpoint_path\"],\n        \"checkpoint_sha256\": context[\"checkpoint_sha256\"],\n        \"best_epoch\": context[\"best_epoch\"],\n        \"epoch_evidence\": context[\"epoch_evidence\"],\n        \"split_source\": context[\"split_source\"],\n        \"num_test_samples\": len(context[\"test_records\"]),\n    }\n    for context in CONTEXTS.values()\n]).to_string(index=False))\n\n","metadata":{},"outputs":[],"execution_count":null},{"id":"0556ec4a-f43b-43d1-a1f1-01800fda7ff2","cell_type":"markdown","source":"## Checkpoint evaluation\n\nEvaluation uses the same 300 x 300 ImageNet-normalized PPS input and the exact\ntest indices used by the XAI notebooks. Predictions are exported so accuracy and\nF1-Macro can be independently recomputed.\n\n","metadata":{}},{"id":"9ca9b995-658b-4286-ae67-4799db645a27","cell_type":"code","source":"# ============================================================\n# CELL 3: Dataset, model, and evaluation\n# ============================================================\nVAL_TRANSFORM = transforms.Compose([\n    transforms.Resize((IMG_SIZE, IMG_SIZE)),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),\n])\n\n\nclass EvaluationDataset(Dataset):\n    def __init__(self, records):\n        self.records = records\n\n    def __len__(self):\n        return len(self.records)\n\n    def __getitem__(self, index):\n        record = self.records[index]\n        with Image.open(record[\"path\"]) as image:\n            tensor = VAL_TRANSFORM(image.convert(\"RGB\"))\n        return tensor, int(record[\"label_index\"]), str(record[\"sample_id\"])\n\n\ndef build_efficientnet_b3(num_classes):\n    model = models.efficientnet_b3(weights=None)\n    in_features = model.classifier[1].in_features\n    model.classifier = nn.Sequential(\n        nn.Dropout(p=0.3, inplace=True),\n        nn.Linear(in_features, num_classes),\n    )\n    return model.to(DEVICE)\n\n\ndef load_model(context):\n    model = build_efficientnet_b3(context[\"config\"][\"num_classes\"])\n    checkpoint = torch.load(context[\"checkpoint_path\"], map_location=DEVICE)\n    model.load_state_dict(unwrap_state_dict(checkpoint), strict=True)\n    model.eval()\n    return model\n\n\n@torch.no_grad()\ndef evaluate_checkpoint(dataset_key, context):\n    loader = DataLoader(\n        EvaluationDataset(context[\"test_records\"]),\n        batch_size=BATCH_SIZE,\n        shuffle=False,\n        num_workers=NUM_WORKERS,\n        pin_memory=DEVICE.type == \"cuda\",\n    )\n    model = load_model(context)\n    predictions = []\n    true_labels = []\n    prediction_rows = []\n    for images, labels, sample_ids in tqdm(\n        loader,\n        desc=f\"Evaluate {context['config']['dataset_label']}\",\n    ):\n        images = images.to(DEVICE, non_blocking=True)\n        labels = labels.to(DEVICE, non_blocking=True)\n        with torch.autocast(\n            device_type=\"cuda\",\n            dtype=torch.float16,\n            enabled=AMP_ENABLED,\n        ):\n            logits = model(images)\n        batch_predictions = logits.argmax(dim=1)\n        labels_np = labels.cpu().numpy()\n        predictions_np = batch_predictions.cpu().numpy()\n        true_labels.extend(labels_np.tolist())\n        predictions.extend(predictions_np.tolist())\n        for sample_id, true_index, predicted_index in zip(\n            sample_ids, labels_np, predictions_np\n        ):\n            prediction_rows.append({\n                \"dataset\": context[\"config\"][\"dataset_label\"],\n                \"sample_id\": str(sample_id),\n                \"true_label_index\": int(true_index),\n                \"predicted_label_index\": int(predicted_index),\n                \"correct\": bool(int(true_index) == int(predicted_index)),\n                \"checkpoint_sha256\": context[\"checkpoint_sha256\"],\n                \"split_sha256\": context[\"split_sha256\"],\n            })\n    true_labels = np.asarray(true_labels, dtype=np.int64)\n    predictions = np.asarray(predictions, dtype=np.int64)\n    accuracy = float(np.mean(true_labels == predictions))\n    macro_f1 = float(f1_score(\n        true_labels,\n        predictions,\n        average=\"macro\",\n        zero_division=0,\n    ))\n    del model, loader\n    if DEVICE.type == \"cuda\":\n        torch.cuda.empty_cache()\n    evaluation = {\n        \"dataset\": context[\"config\"][\"dataset_label\"],\n        \"checkpoint\": context[\"checkpoint_path\"],\n        \"checkpoint_hash\": context[\"checkpoint_sha256\"],\n        \"architecture\": ARCHITECTURE,\n        \"seed\": TRAINING_SEED,\n        \"training_run\": TRAINING_RUN,\n        \"selection_rule\": SELECTION_RULE,\n        \"epoch\": context[\"best_epoch\"],\n        \"epoch_evidence\": context[\"epoch_evidence\"],\n        \"split_source\": context[\"split_source\"],\n        \"split_hash\": context[\"split_sha256\"],\n        \"test_sample_ids_file\": context[\"test_ids_path\"],\n        \"num_test_samples\": int(len(true_labels)),\n        \"accuracy\": accuracy,\n        \"f1_macro\": macro_f1,\n        \"amp_enabled\": AMP_ENABLED,\n    }\n    return evaluation, prediction_rows\n\n\nEVALUATIONS = []\nPREDICTION_ROWS = []\nfor dataset_key, context in CONTEXTS.items():\n    evaluation, rows = evaluate_checkpoint(dataset_key, context)\n    EVALUATIONS.append(evaluation)\n    PREDICTION_ROWS.extend(rows)\n    LOGGER.info(\n        \"%s: n=%d accuracy=%.6f f1_macro=%.6f\",\n        evaluation[\"dataset\"],\n        evaluation[\"num_test_samples\"],\n        evaluation[\"accuracy\"],\n        evaluation[\"f1_macro\"],\n    )\n\nPREDICTIONS = pd.DataFrame(PREDICTION_ROWS)\nPREDICTIONS.to_csv(PREDICTIONS_CSV, index=False)\n\n","metadata":{},"outputs":[],"execution_count":null},{"id":"1dd4f9af-5fdf-450a-bc3d-c47b5b066a07","cell_type":"markdown","source":"## Metadata, figure provenance, and camera-ready statement\n\nThe final section checks the reproduced test metrics, cross-checks Task 2 hashes\nwhen available, and generates the exact CSV/JSON requested by the audit brief.\n\n","metadata":{}},{"id":"04e3a07d","cell_type":"code","source":"# ============================================================\n# CELL 4: Required outputs and consistency report\n# ============================================================\ndef find_figure(filename):\n    candidates = [\n        AUDIT_ROOT / \"attribution\" / filename,\n        Path(\"/kaggle/working\") / filename,\n    ]\n    for path in candidates:\n        if path.is_file():\n            return str(path), sha256_file(path)\n    return filename, None\n\n\ndef load_task2_metadata():\n    if not TASK2_METADATA_PATH.is_file():\n        return None\n    with open(TASK2_METADATA_PATH, \"r\", encoding=\"utf-8\") as handle:\n        return json.load(handle)\n\n\nTASK2_METADATA = load_task2_metadata()\nattribution_figure_path, attribution_figure_hash = find_figure(ATTRIBUTION_FIGURE)\ncorrelation_figure_path, correlation_figure_hash = find_figure(CORRELATION_FIGURE)\n\nmetadata_rows = []\nprovenance_rows = []\nfor evaluation in EVALUATIONS:\n    dataset_label = evaluation[\"dataset\"]\n    dataset_key = next(\n        key\n        for key, context in CONTEXTS.items()\n        if context[\"config\"][\"dataset_label\"] == dataset_label\n    )\n    context = CONTEXTS[dataset_key]\n    task2_hash = None\n    task2_match = None\n    if TASK2_METADATA:\n        task2_record = TASK2_METADATA.get(\"datasets\", {}).get(dataset_key, {})\n        task2_hash = task2_record.get(\"checkpoint_sha256\")\n        if task2_hash:\n            task2_match = task2_hash == context[\"checkpoint_sha256\"]\n    checkpoint_filename_match = os.path.basename(context[\"checkpoint_path\"]).startswith(\n        \"best_eb3_teacher_run1\"\n    )\n    reported_accuracy_reproduced = (\n        round(evaluation[\"accuracy\"], 4)\n        == context[\"config\"][\"expected_xai_accuracy\"]\n    )\n    source_checkpoint_consistent = bool(\n        checkpoint_filename_match and reported_accuracy_reproduced\n    )\n    metadata_rows.append({\n        \"dataset\": dataset_label,\n        \"checkpoint_path\": evaluation[\"checkpoint\"],\n        \"architecture\": ARCHITECTURE,\n        \"seed\": TRAINING_SEED,\n        \"selection_rule\": SELECTION_RULE,\n        \"epoch\": evaluation[\"epoch\"],\n        \"test_accuracy\": evaluation[\"accuracy\"],\n        \"test_f1_macro\": evaluation[\"f1_macro\"],\n        \"num_test_samples\": evaluation[\"num_test_samples\"],\n        \"attribution_figure\": ATTRIBUTION_FIGURE,\n        \"correlation_figure\": CORRELATION_FIGURE,\n    })\n    provenance_rows.append({\n        \"dataset\": dataset_label,\n        \"checkpoint_sha256\": context[\"checkpoint_sha256\"],\n        \"split_sha256\": context[\"split_sha256\"],\n        \"attribution_source_notebook\": ATTRIBUTION_SOURCE_NOTEBOOK,\n        \"attribution_figure_path\": attribution_figure_path,\n        \"attribution_figure_sha256\": attribution_figure_hash,\n        \"task2_checkpoint_sha256\": task2_hash,\n        \"task2_checkpoint_hash_match\": task2_match,\n        \"correlation_source_notebook\": CORRELATION_SOURCE_NOTEBOOK,\n        \"correlation_figure_path\": correlation_figure_path,\n        \"correlation_figure_sha256\": correlation_figure_hash,\n        \"source_checkpoint_filename_match\": checkpoint_filename_match,\n        \"reported_accuracy_reproduced\": reported_accuracy_reproduced,\n        \"source_checkpoint_consistent\": source_checkpoint_consistent,\n        \"confirmation_basis\": (\n            \"eb3-xai.ipynb and gradcam.ipynb configure the same run-1 checkpoint; \"\n            \"this audit records its hash and reproduces its test metrics\"\n        ),\n    })\n\nMETADATA = pd.DataFrame(metadata_rows)\nPROVENANCE = pd.DataFrame(provenance_rows)\nMETADATA.to_csv(METADATA_CSV, index=False)\nPROVENANCE.to_csv(PROVENANCE_CSV, index=False)\n\nevaluation_document = {\n    \"task\": \"Task 3 - XAI checkpoint audit\",\n    \"training_performed\": False,\n    \"checkpoint_selected_using_test_results\": False,\n    \"selection_rule\": SELECTION_RULE,\n    \"evaluations\": EVALUATIONS,\n}\nEVALUATION_JSON.write_text(\n    json.dumps(evaluation_document, indent=2) + \"\\n\",\n    encoding=\"utf-8\",\n)\n\n\ndef markdown_table(frame, float_digits=6):\n    formatted = frame.copy()\n    for column in formatted.select_dtypes(include=[np.number]).columns:\n        formatted[column] = formatted[column].map(\n            lambda value: f\"{value:.{float_digits}f}\" if pd.notna(value) else \"NA\"\n        )\n    headers = list(formatted.columns)\n    lines = [\n        \"| \" + \" | \".join(headers) + \" |\",\n        \"| \" + \" | \".join([\"---\"] * len(headers)) + \" |\",\n    ]\n    for row in formatted.astype(str).itertuples(index=False, name=None):\n        lines.append(\"| \" + \" | \".join(row) + \" |\")\n    return \"\\n\".join(lines)\n\n\ncomparison_rows = []\nfor evaluation in EVALUATIONS:\n    config = next(\n        context[\"config\"]\n        for context in CONTEXTS.values()\n        if context[\"config\"][\"dataset_label\"] == evaluation[\"dataset\"]\n    )\n    expected_f1 = config[\"expected_xai_f1\"]\n    comparison_rows.append({\n        \"dataset\": evaluation[\"dataset\"],\n        \"actual_num_test_samples\": evaluation[\"num_test_samples\"],\n        \"expected_num_test_samples\": config[\"expected_num_test_samples\"],\n        \"sample_count_match\": (\n            evaluation[\"num_test_samples\"] == config[\"expected_num_test_samples\"]\n        ),\n        \"actual_accuracy\": evaluation[\"accuracy\"],\n        \"expected_accuracy\": config[\"expected_xai_accuracy\"],\n        \"accuracy_rounding_match\": (\n            round(evaluation[\"accuracy\"], 4) == config[\"expected_xai_accuracy\"]\n        ),\n        \"actual_f1_macro\": evaluation[\"f1_macro\"],\n        \"expected_f1_macro\": expected_f1,\n        \"f1_rounding_match\": (\n            None if expected_f1 is None\n            else round(evaluation[\"f1_macro\"], 4) == expected_f1\n        ),\n    })\nCOMPARISON = pd.DataFrame(comparison_rows)\n\nstatement_lines = []\nfor evaluation in EVALUATIONS:\n    statement_lines.append(\n        f\"Phan phan tich giai thich tren {evaluation['dataset']} su dung checkpoint \"\n        f\"{ARCHITECTURE} cua hat giong {TRAINING_SEED} (run {TRAINING_RUN}), duoc \"\n        f\"chon tai epoch {evaluation['epoch']} theo F1-Macro cao nhat tren tap \"\n        f\"validation. Checkpoint nay dat accuracy {evaluation['accuracy']:.4f} va \"\n        f\"F1-Macro {evaluation['f1_macro']:.4f} tren {evaluation['num_test_samples']} \"\n        \"mau test. Cac so trong bang chinh la trung binh cua ba hat giong, nen \"\n        \"khong nhat thiet trung voi checkpoint don le dung de tao hinh.\"\n    )\n\nreport_lines = [\n    \"# XAI checkpoint consistency report\",\n    \"\",\n    \"## Scope\",\n    \"\",\n    \"No model was trained or fine-tuned. Each checkpoint was evaluated on the\",\n    \"recorded fixed test split, and all per-sample predictions were exported.\",\n    \"\",\n    \"## Checkpoint metadata\",\n    \"\",\n    markdown_table(METADATA),\n    \"\",\n    \"## Reproduction check\",\n    \"\",\n    markdown_table(COMPARISON),\n    \"\",\n    \"## Figure provenance\",\n    \"\",\n    markdown_table(PROVENANCE),\n    \"\",\n    \"A PNG does not embed its checkpoint. Confirmation therefore relies on the\",\n    \"checkpoint path configured in both source notebooks, the SHA-256 recorded by\",\n    \"this audit, the exact split hash, and reproduction of the reported test metrics.\",\n    \"When Task 2 metadata exists, its checkpoint hash is compared directly above.\",\n    \"\",\n    \"## Selection rule\",\n    \"\",\n    f\"`{SELECTION_RULE}`\",\n    \"\",\n    \"The checkpoint was selected using validation F1-Macro within a fixed run, not\",\n    \"by choosing the highest test result. It should be described as the run-1\",\n    \"validation-selected checkpoint, not as a global best model.\",\n    \"\",\n    \"## Suggested camera-ready wording\",\n    \"\",\n    *[f\"- {line}\" for line in statement_lines],\n    \"\",\n]\n\nall_counts_match = bool(COMPARISON[\"sample_count_match\"].all())\nall_accuracy_match = bool(COMPARISON[\"accuracy_rounding_match\"].all())\nknown_f1 = COMPARISON[\"f1_rounding_match\"].dropna()\nall_known_f1_match = bool(known_f1.all()) if len(known_f1) else True\nall_epochs_known = all(evaluation[\"epoch\"] is not None for evaluation in EVALUATIONS)\n\nreport_lines.extend([\n    \"## Completion status\",\n    \"\",\n    f\"- Test sample counts reproduced: `{all_counts_match}`\",\n    f\"- Reported XAI accuracies reproduced: `{all_accuracy_match}`\",\n    f\"- Known F1-Macro values reproduced: `{all_known_f1_match}`\",\n    f\"- Best epoch known for every checkpoint: `{all_epochs_known}`\",\n    \"- Training performed during audit: `False`\",\n    \"\",\n])\nCONSISTENCY_REPORT.write_text(\"\\n\".join(report_lines), encoding=\"utf-8\")\n\nREADME_PATH.write_text(\n    \"\"\"# Task 3 - XAI checkpoint audit\n\nRun `task3_xai_checkpoint_audit.ipynb` from top to bottom on a Kaggle GPU. Attach\nthe original BIG-2015 and Malimg PPS datasets, both run-1 EfficientNet-B3\ncheckpoints, and the `dataconfig` dataset containing the saved split and\ntraining-result JSON files.\n\nNo training occurs. The configured run-1 best epochs are 20 for BIG-2015 and 8\nfor Malimg, as recorded in their corresponding EB3 training-result JSON files.\n\"\"\",\n    encoding=\"utf-8\",\n)\n\nrequired_outputs = [\n    METADATA_CSV,\n    EVALUATION_JSON,\n    PREDICTIONS_CSV,\n    PROVENANCE_CSV,\n    CONSISTENCY_REPORT,\n    README_PATH,\n    CHECKPOINT_LOG,\n]\nfor dataset_key in DATASETS_TO_RUN:\n    required_outputs.append(\n        CHECKPOINT_DIR / f\"xai_test_sample_ids_{dataset_key}.csv\"\n    )\n    required_outputs.append(\n        CHECKPOINT_DIR / f\"xai_split_{dataset_key}_snapshot.json\"\n    )\n\nmissing_outputs = [str(path) for path in required_outputs if not path.exists()]\nif missing_outputs:\n    raise RuntimeError(\"Missing Task 3 outputs:\\n- \" + \"\\n- \".join(missing_outputs))\n\nLOGGER.info(\"Task 3 audit complete: %s\", CHECKPOINT_DIR)\nprint(METADATA.to_string(index=False))\nprint(\"\\n\" + \"\\n\".join(statement_lines))\nprint(\"\\nTask 3 outputs:\")\nfor output in required_outputs:\n    print(f\"- {output}\")\n","metadata":{},"outputs":[],"execution_count":null}]}