{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"f8c124a8","cell_type":"markdown","source":"# Phase 9 — Reproducibility / Benchmark Packaging (configs, seeds, fold defs, ablation reports)\n\nimmuneML-style packaging: YAML/JSON configs, deterministic seeds, fold definitions saved, feature lists saved, per-branch OOF predictions saved, ablation reports auto-generated. Builds the final reproducibility manifest.\n\n---\n\n## Kaggle inputs to add before running this notebook\n\nAdd the following as Kaggle inputs (via the \"Add Input\" button on the right\npanel of the notebook editor):\n- Phase 1 notebook output\n- Phase 2 notebook output\n- Phase 3 notebook output\n- Phase 4 notebook output\n- Phase 5 notebook output\n- Phase 6 notebook output\n- Phase 7 notebook output\n- Phase 8 notebook output\n\n## Workflow\n\n1. Run the **SETUP** cell — it creates `/kaggle/working/project/` and auto-merges\n   any previous-phase notebook outputs found under `/kaggle/input/`.\n2. Run each subsequent cell in order. The **Run** cell executes the phase's\n   training script; the **Inspect** cell prints a quick summary of the outputs.\n3. When the run completes, click **Save Version → Save & Run All (Commit)** so\n   the next phase can pick this phase's outputs up via \"Add Input\".\n","metadata":{}},{"id":"c4a378a8","cell_type":"code","source":"# ============================================================\n# SETUP — Initialize project + merge previous-phase inputs\n# ============================================================\n# This cell:\n#   1. Creates /kaggle/working/project/ fresh (idempotent re-runs).\n#   2. Auto-detects previous-phase notebook outputs under /kaggle/input/.\n#   3. Merges their project/ contents (src/, artifacts/, configs/) into\n#      /kaggle/working/project/ so this phase can build on them.\n#\n# Kaggle \"Add Input\" workflow:\n#   - Phase 1: add the AIRR-ML competition dataset only.\n#   - Phase N (N>=2): add the AIRR-ML competition dataset AND the previous\n#     phase notebook output(s). For Phase 9 add ALL of Phases 1..8.\n# ============================================================\n\nimport os\nimport shutil\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\n# Reset working project dir (idempotent re-runs)\nif PROJECT_ROOT.exists():\n    shutil.rmtree(PROJECT_ROOT)\nPROJECT_ROOT.mkdir(parents=True, exist_ok=True)\n(PROJECT_ROOT / \"src\").mkdir(parents=True, exist_ok=True)\n(PROJECT_ROOT / \"src\" / \"__init__.py\").write_text(\"\", encoding=\"utf-8\")\n(PROJECT_ROOT / \"configs\").mkdir(parents=True, exist_ok=True)\n(PROJECT_ROOT / \"artifacts\").mkdir(parents=True, exist_ok=True)\n\n# Auto-detect and merge previous-phase inputs.\n# Each previous-phase notebook output should contain a top-level `project/`\n# directory (created by Phase 1 and propagated through every later phase).\nprev_inputs_found = []\ninput_root = Path(\"/kaggle/input\")\nif input_root.exists():\n    for entry in sorted(input_root.iterdir()):\n        if not entry.is_dir():\n            continue\n        prev_project = entry / \"project\"\n        if not prev_project.is_dir():\n            continue\n        prev_inputs_found.append(entry.name)\n        for sub in [\"src\", \"artifacts\", \"configs\"]:\n            src_dir = prev_project / sub\n            if not src_dir.exists():\n                continue\n            for path in src_dir.rglob(\"*\"):\n                if path.is_file():\n                    rel = path.relative_to(src_dir)\n                    target = PROJECT_ROOT / sub / rel\n                    target.parent.mkdir(parents=True, exist_ok=True)\n                    shutil.copy2(path, target)\n\nprint(\"PROJECT_ROOT :\", PROJECT_ROOT)\nif prev_inputs_found:\n    print(\"Previous-phase inputs merged:\")\n    for name in prev_inputs_found:\n        print(\"  -\", name)\nelse:\n    print(\"Previous-phase inputs: (none — running fresh)\")\nprint(\"\\nExisting artifacts:\")\nart_dir = PROJECT_ROOT / \"artifacts\"\nif art_dir.exists():\n    for p in sorted(art_dir.glob(\"*\")):\n        print(\"  -\", p.name)\nelse:\n    print(\"  (none)\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-24T11:32:59.030661Z","iopub.execute_input":"2026-07-24T11:32:59.031153Z","iopub.status.idle":"2026-07-24T11:32:59.042820Z","shell.execute_reply.started":"2026-07-24T11:32:59.031124Z","shell.execute_reply":"2026-07-24T11:32:59.042076Z"}},"outputs":[],"execution_count":null},{"id":"53075078","cell_type":"code","source":"# Cell 1: Phase 9 packaging config file write\n\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nphase9_yaml = \"\"\"\nenabled: true\n\npaths:\n  output_root: /kaggle/working/project/artifacts/phase9_packaging\n  phase1_root: /kaggle/working/project/artifacts/phase1\n  phase2_root: /kaggle/working/project/artifacts/phase2_branch_a\n  phase3_root: /kaggle/working/project/artifacts/phase3_branch_b\n  phase4_root: /kaggle/working/project/artifacts/phase4_branch_c\n  phase5_root: /kaggle/working/project/artifacts/phase5_branch_d\n  phase6_root: /kaggle/working/project/artifacts/phase6_attribution\n  phase7_root: /kaggle/working/project/artifacts/phase7_fusion\n  phase8_root: /kaggle/working/project/artifacts/phase8_benchmarks\n\nruntime_flags:\n  USE_BRANCH_A: true\n  USE_BRANCH_B: true\n  USE_BRANCH_C: true\n  USE_BRANCH_D: true\n  SPLIT_MODE: group_stratified\n  N_SEEDS: 3\n  RANDOM_STATE: 42\n\"\"\"\n\n(PROJECT_ROOT / \"configs\" / \"phase9.yaml\").write_text(phase9_yaml.strip() + \"\\n\", encoding=\"utf-8\")\nprint(\"Written:\", PROJECT_ROOT / \"configs\" / \"phase9.yaml\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-24T11:32:59.044011Z","iopub.execute_input":"2026-07-24T11:32:59.044257Z","iopub.status.idle":"2026-07-24T11:32:59.058480Z","shell.execute_reply.started":"2026-07-24T11:32:59.044238Z","shell.execute_reply":"2026-07-24T11:32:59.057896Z"}},"outputs":[],"execution_count":null},{"id":"cc0c8d1f","cell_type":"code","source":"# Cell 2: Write src/repro.py and run_phase9.py\n\nfrom pathlib import Path\nfrom textwrap import dedent\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nrepro_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nimport hashlib\nimport os\nimport platform\nimport shutil\nimport subprocess\nimport sys\nfrom pathlib import Path\nfrom typing import Dict, List\n\nimport numpy as np\nimport pandas as pd\n\nfrom .utils import ensure_dir\n\n\ndef safe_read_csv(path: Path) -> pd.DataFrame:\n    try:\n        if path.exists():\n            return pd.read_csv(path)\n    except Exception:\n        pass\n    return pd.DataFrame()\n\n\ndef safe_read_parquet(path: Path) -> pd.DataFrame:\n    try:\n        if path.exists():\n            return pd.read_parquet(path)\n    except Exception:\n        pass\n    return pd.DataFrame()\n\n\ndef concat_csvs(paths: List[Path]) -> pd.DataFrame:\n    parts = []\n    for p in paths:\n        x = safe_read_csv(p)\n        if len(x):\n            parts.append(x)\n    if len(parts) == 0:\n        return pd.DataFrame()\n    return pd.concat(parts, ignore_index=True)\n\n\ndef concat_parquet_or_csv(parquet_paths: List[Path], csv_paths: List[Path]) -> pd.DataFrame:\n    parts = []\n    for p in parquet_paths:\n        x = safe_read_parquet(p)\n        if len(x):\n            parts.append(x)\n    for p in csv_paths:\n        x = safe_read_csv(p)\n        if len(x):\n            parts.append(x)\n    if len(parts) == 0:\n        return pd.DataFrame()\n    return pd.concat(parts, ignore_index=True)\n\n\ndef sha256_small(path: Path, chunk_size: int = 1024 * 1024) -> str:\n    try:\n        h = hashlib.sha256()\n        with open(path, \"rb\") as f:\n            while True:\n                chunk = f.read(chunk_size)\n                if not chunk:\n                    break\n                h.update(chunk)\n        return h.hexdigest()\n    except Exception:\n        return \"\"\n\n\ndef write_environment_file(out_path: Path):\n    lines = []\n    lines.append(f\"python_version: {sys.version}\")\n    lines.append(f\"platform: {platform.platform()}\")\n    lines.append(f\"executable: {sys.executable}\")\n    lines.append(\"\")\n\n    try:\n        res = subprocess.run([sys.executable, \"-m\", \"pip\", \"freeze\"], capture_output=True, text=True, timeout=60)\n        lines.append(\"[pip_freeze]\")\n        lines.append(res.stdout.strip())\n    except Exception as e:\n        lines.append(\"[pip_freeze]\")\n        lines.append(f\"ERROR: {e}\")\n\n    out_path.write_text(\"\\\\n\".join(lines) + \"\\\\n\", encoding=\"utf-8\")\n\n\ndef snapshot_configs(config_dir: Path, out_dir: Path):\n    out_dir = ensure_dir(out_dir)\n    for p in sorted(config_dir.glob(\"*.yaml\")):\n        shutil.copy2(p, out_dir / p.name)\n\n\ndef build_manifest(project_root: Path, out_path: Path):\n    rows = []\n    for p in sorted(project_root.rglob(\"*\")):\n        if p.is_file():\n            try:\n                rel = p.relative_to(project_root)\n            except Exception:\n                rel = p\n            rows.append({\n                \"relative_path\": str(rel),\n                \"size_bytes\": int(p.stat().st_size),\n                \"sha256\": sha256_small(p),\n            })\n    pd.DataFrame(rows).to_csv(out_path, index=False)\n\n\ndef write_runtime_flags_yaml(runtime_flags: dict, out_path: Path):\n    lines = []\n    for k, v in runtime_flags.items():\n        if isinstance(v, bool):\n            vv = \"true\" if v else \"false\"\n        else:\n            vv = v\n        lines.append(f\"{k}: {vv}\")\n    out_path.write_text(\"\\\\n\".join(lines) + \"\\\\n\", encoding=\"utf-8\")\n\n\ndef write_run_pipeline_skeleton(project_root: Path):\n    script = f'''from __future__ import annotations\n\nfrom pathlib import Path\nimport subprocess\nimport sys\nimport yaml\n\nPROJECT_ROOT = Path(\"{project_root}\")\nCONFIG_PATH = PROJECT_ROOT / \"configs\" / \"phase9.yaml\"\n\n\ndef run_script(path: Path):\n    print(f\"Running: {{path}}\")\n    res = subprocess.run([sys.executable, str(path)])\n    if res.returncode != 0:\n        raise SystemExit(res.returncode)\n\n\ndef main():\n    cfg = yaml.safe_load(CONFIG_PATH.read_text())\n    flags = cfg[\"runtime_flags\"]\n\n    # Phase 1\n    run_script(PROJECT_ROOT / \"run_phase1.py\")\n\n    # Branches\n    if flags.get(\"USE_BRANCH_A\", True):\n        run_script(PROJECT_ROOT / \"train_branch_a.py\")\n\n    if flags.get(\"USE_BRANCH_B\", True):\n        run_script(PROJECT_ROOT / \"train_branch_b.py\")\n\n    if flags.get(\"USE_BRANCH_C\", False):\n        run_script(PROJECT_ROOT / \"train_branch_c.py\")\n\n    if flags.get(\"USE_BRANCH_D\", True):\n        run_script(PROJECT_ROOT / \"train_branch_d.py\")\n\n    # Fusion + Benchmarks + Packaging\n    run_script(PROJECT_ROOT / \"train_fusion.py\")\n    run_script(PROJECT_ROOT / \"train_benchmarks.py\")\n    run_script(PROJECT_ROOT / \"run_phase9.py\")\n\n\nif __name__ == \"__main__\":\n    main()\n'''\n    (project_root / \"run_pipeline.py\").write_text(script, encoding=\"utf-8\")\n\n\ndef _best_rows(df: pd.DataFrame, metric_col: str, group_cols: List[str]) -> pd.DataFrame:\n    if len(df) == 0:\n        return pd.DataFrame()\n    x = df.dropna(subset=[metric_col]).copy()\n    if len(x) == 0:\n        return pd.DataFrame()\n    idx = x.groupby(group_cols)[metric_col].idxmax()\n    return x.loc[idx].sort_values(group_cols).reset_index(drop=True)\n\n\ndef _df_to_text(df: pd.DataFrame, max_rows: int = 20) -> str:\n    if df is None or len(df) == 0:\n        return \"No rows found.\\\\n\"\n    return df.head(max_rows).to_string(index=False) + \"\\\\n\"\n\n\ndef generate_branch_a_report(phase2_root: Path, out_path: Path):\n    summary = safe_read_csv(phase2_root / \"branch_a_summary.csv\")\n    ablation = safe_read_csv(phase2_root / \"branch_a_feature_ablation.csv\")\n    selected_files = sorted(phase2_root.glob(\"selected_features_*.csv\"))\n\n    txt = []\n    txt.append(\"# report_branch_a\\\\n\")\n    txt.append(\"## Files detected\\\\n\")\n    txt.append(f\"- branch_a_summary.csv: {(phase2_root / 'branch_a_summary.csv').exists()}\\\\n\")\n    txt.append(f\"- branch_a_feature_ablation.csv: {(phase2_root / 'branch_a_feature_ablation.csv').exists()}\\\\n\")\n    txt.append(f\"- selected feature files: {len(selected_files)}\\\\n\")\n\n    if len(summary):\n        txt.append(\"\\\\n## Best Branch A models by dataset\\\\n\")\n        best = _best_rows(summary, \"mean_auc\", [\"dataset\", \"seed\"])\n        txt.append(_df_to_text(best[[\"dataset\", \"seed\", \"model\", \"mean_auc\", \"std_auc\", \"max_auc\"]], max_rows=50))\n\n    if len(ablation):\n        txt.append(\"\\\\n## Best ablation stage by dataset\\\\n\")\n        best_ab = _best_rows(ablation, \"auc\", [\"dataset\"])\n        txt.append(_df_to_text(best_ab[[\"dataset\", \"stage\", \"n_features_stage\", \"auc\"]], max_rows=50))\n\n    out_path.write_text(\"\\\\n\".join(txt), encoding=\"utf-8\")\n\n\ndef generate_branch_b_report(phase3_root: Path, out_path: Path):\n    summary = safe_read_csv(phase3_root / \"branch_b_summary.csv\")\n    exact_files = sorted(phase3_root.glob(\"disease_enriched_sequences_*.csv\"))\n    cluster_files = sorted(phase3_root.glob(\"cluster_catalog_*.csv\"))\n\n    txt = []\n    txt.append(\"# report_branch_b\\\\n\")\n    txt.append(\"## Files detected\\\\n\")\n    txt.append(f\"- branch_b_summary.csv: {(phase3_root / 'branch_b_summary.csv').exists()}\\\\n\")\n    txt.append(f\"- exact catalog files: {len(exact_files)}\\\\n\")\n    txt.append(f\"- cluster catalog files: {len(cluster_files)}\\\\n\")\n\n    if len(summary):\n        txt.append(\"\\\\n## Best Branch B models by dataset\\\\n\")\n        best = _best_rows(summary, \"mean_auc\", [\"dataset\", \"seed\"])\n        txt.append(_df_to_text(best[[\"dataset\", \"seed\", \"model\", \"mean_auc\", \"std_auc\", \"max_auc\"]], max_rows=50))\n\n    if len(exact_files):\n        sample_exact = safe_read_csv(exact_files[0])\n        txt.append(\"\\\\n## Example exact disease-enriched catalog\\\\n\")\n        txt.append(_df_to_text(sample_exact.head(10), max_rows=10))\n\n    if len(cluster_files):\n        sample_cluster = safe_read_csv(cluster_files[0])\n        txt.append(\"\\\\n## Example approximate cluster catalog\\\\n\")\n        txt.append(_df_to_text(sample_cluster.head(10), max_rows=10))\n\n    out_path.write_text(\"\\\\n\".join(txt), encoding=\"utf-8\")\n\n\ndef generate_branch_c_report(phase4_root: Path, out_path: Path):\n    summary = safe_read_csv(phase4_root / \"branch_c_summary.csv\")\n    emb_parquet = phase4_root / \"embedding_repertoire_vectors.parquet\"\n    emb_csv = phase4_root / \"embedding_repertoire_vectors_fallback.csv\"\n    pca_plot = phase4_root / \"embedding_pca_plot.png\"\n\n    txt = []\n    txt.append(\"# report_branch_c\\\\n\")\n    txt.append(\"## Files detected\\\\n\")\n    txt.append(f\"- branch_c_summary.csv: {(phase4_root / 'branch_c_summary.csv').exists()}\\\\n\")\n    txt.append(f\"- embedding_repertoire_vectors.parquet: {emb_parquet.exists()}\\\\n\")\n    txt.append(f\"- embedding_repertoire_vectors_fallback.csv: {emb_csv.exists()}\\\\n\")\n    txt.append(f\"- embedding_pca_plot.png: {pca_plot.exists()}\\\\n\")\n\n    if len(summary):\n        txt.append(\"\\\\n## Best Branch C models by dataset\\\\n\")\n        best = _best_rows(summary, \"mean_auc\", [\"dataset\", \"seed\"])\n        txt.append(_df_to_text(best[[\"dataset\", \"seed\", \"model\", \"mean_auc\", \"std_auc\", \"max_auc\"]], max_rows=50))\n\n    emb_df = safe_read_parquet(emb_parquet)\n    if len(emb_df) == 0:\n        emb_df = safe_read_csv(emb_csv)\n\n    if len(emb_df):\n        meta_cols = [c for c in [\"dataset\", \"embed_route_used\", \"embed_model\", \"embed_dim\"] if c in emb_df.columns]\n        meta_view = emb_df[meta_cols].drop_duplicates().head(20)\n        txt.append(\"\\\\n## Embedding metadata\\\\n\")\n        txt.append(_df_to_text(meta_view, max_rows=20))\n\n    out_path.write_text(\"\\\\n\".join(txt), encoding=\"utf-8\")\n\n\ndef generate_branch_d_report(phase5_root: Path, out_path: Path):\n    summary_files = sorted(phase5_root.glob(\"branch_d_summary_*.csv\"))\n    summary = concat_csvs(summary_files)\n    seq_files = sorted(phase5_root.glob(\"mil_sequence_scores_*.parquet\"))\n    seq_fallback = sorted(phase5_root.glob(\"mil_sequence_scores_*_fallback.csv\"))\n    attn_files = sorted(phase5_root.glob(\"mil_attention_maps_*.npz\"))\n\n    txt = []\n    txt.append(\"# report_branch_d\\\\n\")\n    txt.append(\"## Files detected\\\\n\")\n    txt.append(f\"- branch_d_oof.csv: {(phase5_root / 'branch_d_oof.csv').exists()}\\\\n\")\n    txt.append(f\"- branch_d summary files: {len(summary_files)}\\\\n\")\n    txt.append(f\"- mil sequence score files: {len(seq_files) + len(seq_fallback)}\\\\n\")\n    txt.append(f\"- attention map files: {len(attn_files)}\\\\n\")\n\n    if len(summary):\n        txt.append(\"\\\\n## Best Branch D folds / seeds\\\\n\")\n        best = _best_rows(summary[summary[\"fold\"] >= 0], \"auc\", [\"dataset\", \"seed\"])\n        txt.append(_df_to_text(best[[\"dataset\", \"seed\", \"fold\", \"model\", \"auc\", \"best_epoch\"]], max_rows=50))\n\n    seq_df = concat_parquet_or_csv(seq_files, seq_fallback)\n    if len(seq_df):\n        txt.append(\"\\\\n## Example ranked sequence attribution rows\\\\n\")\n        keep_cols = [c for c in [\"ID\", \"dataset\", \"rank\", \"sequence\", \"attention_weight\", \"attr_prob\", \"final_seq_score\", \"weak_target\"] if c in seq_df.columns]\n        txt.append(_df_to_text(seq_df[keep_cols].head(15), max_rows=15))\n\n    out_path.write_text(\"\\\\n\".join(txt), encoding=\"utf-8\")\n\n\ndef generate_fusion_report(phase7_root: Path, phase8_root: Path, out_path: Path):\n    fusion_summary = safe_read_csv(phase7_root / \"fusion_summary.csv\")\n    fusion_weights = safe_read_csv(phase7_root / \"fusion_weights.csv\")\n    bench_real = safe_read_csv(phase8_root / \"benchmark_real.csv\")\n    bench_sim = safe_read_csv(phase8_root / \"benchmark_simairr.csv\")\n    bench_ligo = safe_read_csv(phase8_root / \"benchmark_ligo.csv\")\n\n    txt = []\n    txt.append(\"# report_fusion\\\\n\")\n    txt.append(\"## Files detected\\\\n\")\n    txt.append(f\"- fusion_oof.csv: {(phase7_root / 'fusion_oof.csv').exists()}\\\\n\")\n    txt.append(f\"- fusion_weights.csv: {(phase7_root / 'fusion_weights.csv').exists()}\\\\n\")\n    txt.append(f\"- fusion_summary.csv: {(phase7_root / 'fusion_summary.csv').exists()}\\\\n\")\n    txt.append(f\"- benchmark_real.csv: {(phase8_root / 'benchmark_real.csv').exists()}\\\\n\")\n    txt.append(f\"- benchmark_simairr.csv: {(phase8_root / 'benchmark_simairr.csv').exists()}\\\\n\")\n    txt.append(f\"- benchmark_ligo.csv: {(phase8_root / 'benchmark_ligo.csv').exists()}\\\\n\")\n\n    if len(fusion_summary):\n        txt.append(\"\\\\n## Best fusion method by dataset\\\\n\")\n        best = _best_rows(fusion_summary, \"auc\", [\"dataset\"])\n        txt.append(_df_to_text(best[[\"dataset\", \"method\", \"auc\", \"n_samples\"]], max_rows=50))\n\n    if len(fusion_weights):\n        txt.append(\"\\\\n## Fusion weights\\\\n\")\n        txt.append(_df_to_text(fusion_weights.head(50), max_rows=50))\n\n    if len(bench_real):\n        txt.append(\"\\\\n## Real benchmark excerpt\\\\n\")\n        txt.append(_df_to_text(bench_real.head(30), max_rows=30))\n\n    if len(bench_sim):\n        txt.append(\"\\\\n## simAIRR-style benchmark excerpt\\\\n\")\n        txt.append(_df_to_text(bench_sim, max_rows=30))\n\n    if len(bench_ligo):\n        txt.append(\"\\\\n## LIgO-style benchmark excerpt\\\\n\")\n        txt.append(_df_to_text(bench_ligo, max_rows=30))\n\n    out_path.write_text(\"\\\\n\".join(txt), encoding=\"utf-8\")\n\n\ndef generate_benchmark_report(phase8_root: Path, out_path: Path):\n    bench_real = safe_read_csv(phase8_root / \"benchmark_real.csv\")\n    bench_sim = safe_read_csv(phase8_root / \"benchmark_simairr.csv\")\n    bench_ligo = safe_read_csv(phase8_root / \"benchmark_ligo.csv\")\n\n    txt = []\n    txt.append(\"# report_benchmarks\\\\n\")\n\n    txt.append(\"## benchmark_real.csv\\\\n\")\n    txt.append(_df_to_text(bench_real.head(40), max_rows=40))\n\n    txt.append(\"\\\\n## benchmark_simairr.csv\\\\n\")\n    txt.append(_df_to_text(bench_sim, max_rows=40))\n\n    txt.append(\"\\\\n## benchmark_ligo.csv\\\\n\")\n    txt.append(_df_to_text(bench_ligo, max_rows=40))\n\n    out_path.write_text(\"\\\\n\".join(txt), encoding=\"utf-8\")\n\n\ndef generate_old_code_modification_map(out_path: Path):\n    txt = []\n    txt.append(\"# exact_old_code_modification_map\\\\n\")\n\n    txt.append(\"## Config split\\\\n\")\n    txt.append(\"- Old: one constant-heavy `Config` class.\\\\n\")\n    txt.append(\"- New: separate static paths and runtime YAML config.\\\\n\")\n    txt.append(\"- Add runtime flags:\\\\n\")\n    txt.append(\"  - `USE_BRANCH_A = True`\\\\n\")\n    txt.append(\"  - `USE_BRANCH_B = True`\\\\n\")\n    txt.append(\"  - `USE_BRANCH_C = True` or `False` depending on checkpoint availability\\\\n\")\n    txt.append(\"  - `USE_BRANCH_D = True`\\\\n\")\n    txt.append(\"  - `SPLIT_MODE = \\\\\"group_stratified\\\\\"`\\\\n\")\n    txt.append(\"  - `N_SEEDS = 3`\\\\n\")\n\n    txt.append(\"\\\\n## read_repertoire() changes\\\\n\")\n    txt.append(\"- Keep function, but add optional output-support fields: `duplicate_count`, `normalized_templates`, `seq_len`.\\\\n\")\n\n    txt.append(\"\\\\n## FeatureExtractor rename and split\\\\n\")\n    txt.append(\"- Rename to `BranchAFeatureExtractor`.\\\\n\")\n    txt.append(\"- Refactor into methods:\\\\n\")\n    txt.append(\"  - `extract_basic_stats()`\\\\n\")\n    txt.append(\"  - `extract_physchem()`\\\\n\")\n    txt.append(\"  - `extract_kmers()`\\\\n\")\n    txt.append(\"  - `extract_vj_usage()`\\\\n\")\n    txt.append(\"  - `extract_public_clone_features()`\\\\n\")\n    txt.append(\"  - `extract_overlap_features()`\\\\n\")\n\n    txt.append(\"\\\\n## mine_public_clones() scope fix\\\\n\")\n    txt.append(\"- Old: full-dataset public clone mining before CV.\\\\n\")\n    txt.append(\"- New: fold-local mining only.\\\\n\")\n    txt.append(\"- Required signature:\\\\n\")\n    txt.append(\"  `def mine_public_clones_from_fold(train_meta_fold, dataset_path, ...):`\\\\n\")\n\n    txt.append(\"\\\\n## MultiModelTrainer rename and restriction\\\\n\")\n    txt.append(\"- Rename to `BranchATrainer`.\\\\n\")\n    txt.append(\"- Restrict responsibility to Branch A only.\\\\n\")\n    txt.append(\"- Add:\\\\n\")\n    txt.append(\"  - group-aware fold loop\\\\n\")\n    txt.append(\"  - fold-local feature selection\\\\n\")\n    txt.append(\"  - multi-seed training\\\\n\")\n    txt.append(\"  - ablation mode\\\\n\")\n\n    txt.append(\"\\\\n## main() rewrite\\\\n\")\n    txt.append(\"- Old flow: load data -> extract features -> train one trainer -> predict\\\\n\")\n    txt.append(\"- New orchestrated flow:\\\\n\")\n    txt.append(\"```python\\\\n\")\n    txt.append(\"def main():\\\\n\")\n    txt.append(\"    phase1_build_metadata()\\\\n\")\n    txt.append(\"    phase1_make_splits()\\\\n\")\n    txt.append(\"    phase1_run_drift()\\\\n\\\\n\")\n    txt.append(\"    if USE_BRANCH_A:\\\\n\")\n    txt.append(\"        run_branch_a()\\\\n\\\\n\")\n    txt.append(\"    if USE_BRANCH_B:\\\\n\")\n    txt.append(\"        run_branch_b()\\\\n\\\\n\")\n    txt.append(\"    if USE_BRANCH_C:\\\\n\")\n    txt.append(\"        run_branch_c()\\\\n\\\\n\")\n    txt.append(\"    if USE_BRANCH_D:\\\\n\")\n    txt.append(\"        run_branch_d()\\\\n\\\\n\")\n    txt.append(\"    run_fusion()\\\\n\")\n    txt.append(\"    run_benchmarks()\\\\n\")\n    txt.append(\"    make_submission()\\\\n\")\n    txt.append(\"```\\\\n\")\n\n    out_path.write_text(\"\\\\n\".join(txt), encoding=\"utf-8\")\n\"\"\")\n\nrun_phase9_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nimport sys\nfrom pathlib import Path\n\nimport yaml\n\nPROJECT_ROOT = Path(__file__).resolve().parent\nif str(PROJECT_ROOT) not in sys.path:\n    sys.path.insert(0, str(PROJECT_ROOT))\n\nfrom src.repro import (\n    build_manifest,\n    generate_benchmark_report,\n    generate_branch_a_report,\n    generate_branch_b_report,\n    generate_branch_c_report,\n    generate_branch_d_report,\n    generate_fusion_report,\n    generate_old_code_modification_map,\n    snapshot_configs,\n    write_environment_file,\n    write_run_pipeline_skeleton,\n    write_runtime_flags_yaml,\n)\nfrom src.utils import ensure_dir\n\n\ndef main():\n    cfg = yaml.safe_load((PROJECT_ROOT / \"configs\" / \"phase9.yaml\").read_text())\n\n    paths = cfg[\"paths\"]\n    phase9_root = ensure_dir(paths[\"output_root\"])\n    reports_dir = ensure_dir(phase9_root / \"reports\")\n    config_snapshot_dir = ensure_dir(phase9_root / \"config_snapshot\")\n\n    phase1_root = Path(paths[\"phase1_root\"])\n    phase2_root = Path(paths[\"phase2_root\"])\n    phase3_root = Path(paths[\"phase3_root\"])\n    phase4_root = Path(paths[\"phase4_root\"])\n    phase5_root = Path(paths[\"phase5_root\"])\n    phase6_root = Path(paths[\"phase6_root\"])\n    phase7_root = Path(paths[\"phase7_root\"])\n    phase8_root = Path(paths[\"phase8_root\"])\n\n    print(\"=\" * 80)\n    print(\"PHASE 9: Reproducibility / packaging\")\n    print(\"=\" * 80)\n\n    # Snapshot configs\n    snapshot_configs(PROJECT_ROOT / \"configs\", config_snapshot_dir)\n\n    # Environment\n    write_environment_file(phase9_root / \"environment.txt\")\n\n    # Runtime flags\n    write_runtime_flags_yaml(cfg[\"runtime_flags\"], phase9_root / \"runtime_flags.yaml\")\n\n    # Reports\n    generate_branch_a_report(phase2_root, reports_dir / \"report_branch_a.md\")\n    generate_branch_b_report(phase3_root, reports_dir / \"report_branch_b.md\")\n    generate_branch_c_report(phase4_root, reports_dir / \"report_branch_c.md\")\n    generate_branch_d_report(phase5_root, reports_dir / \"report_branch_d.md\")\n    generate_fusion_report(phase7_root, phase8_root, reports_dir / \"report_fusion.md\")\n    generate_benchmark_report(phase8_root, reports_dir / \"report_benchmarks.md\")\n\n    # Exact old-code modification map\n    generate_old_code_modification_map(phase9_root / \"exact_old_code_modification_map.md\")\n\n    # Orchestrator skeleton\n    write_run_pipeline_skeleton(PROJECT_ROOT)\n\n    # Manifest শেষের দিকে বানাও যাতে সব generated files ঢুকে যায়\n    build_manifest(PROJECT_ROOT, phase9_root / \"project_file_manifest.csv\")\n\n    print(\"Saved reports:\")\n    for p in sorted(reports_dir.glob(\"*.md\")):\n        print(\"-\", p)\n\n    print(\"\\\\nSaved packaging files:\")\n    print(\"-\", phase9_root / \"environment.txt\")\n    print(\"-\", phase9_root / \"runtime_flags.yaml\")\n    print(\"-\", phase9_root / \"project_file_manifest.csv\")\n    print(\"-\", phase9_root / \"exact_old_code_modification_map.md\")\n    print(\"-\", PROJECT_ROOT / \"run_pipeline.py\")\n    print(\"=\" * 80)\n\n\nif __name__ == \"__main__\":\n    main()\n\"\"\")\n\n(PROJECT_ROOT / \"src\" / \"repro.py\").write_text(repro_py, encoding=\"utf-8\")\n(PROJECT_ROOT / \"run_phase9.py\").write_text(run_phase9_py, encoding=\"utf-8\")\n\nprint(\"Written:\")\nprint(\"-\", PROJECT_ROOT / \"src\" / \"repro.py\")\nprint(\"-\", PROJECT_ROOT / \"run_phase9.py\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-24T11:32:59.060058Z","iopub.execute_input":"2026-07-24T11:32:59.060687Z","iopub.status.idle":"2026-07-24T11:32:59.078750Z","shell.execute_reply.started":"2026-07-24T11:32:59.060639Z","shell.execute_reply":"2026-07-24T11:32:59.078112Z"}},"outputs":[],"execution_count":null},{"id":"2ba91a15","cell_type":"code","source":"# Cell 3: Packaging imports verify\n\nfrom pathlib import Path\np = Path(\"/kaggle/working/project/src\")\nprint(\"src/ exists:\", p.exists())\nprint(\"contents:\", sorted(f.name for f in p.glob(\"*\")) if p.exists() else \"N/A\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-24T11:32:59.079618Z","iopub.execute_input":"2026-07-24T11:32:59.079886Z","iopub.status.idle":"2026-07-24T11:32:59.091974Z","shell.execute_reply.started":"2026-07-24T11:32:59.079854Z","shell.execute_reply":"2026-07-24T11:32:59.091194Z"}},"outputs":[],"execution_count":null},{"id":"8c8ede45","cell_type":"code","source":"# Cell 4: Run Phase 9 reproducibility / packaging\n\n!python /kaggle/working/project/run_phase9.py","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-24T11:32:59.093353Z","iopub.execute_input":"2026-07-24T11:32:59.093762Z","iopub.status.idle":"2026-07-24T11:32:59.787435Z","shell.execute_reply.started":"2026-07-24T11:32:59.093741Z","shell.execute_reply":"2026-07-24T11:32:59.786708Z"}},"outputs":[],"execution_count":null},{"id":"c6c0d5f6","cell_type":"code","source":"#Cell 5 — outputs inspect করো\nfrom pathlib import Path\nimport pandas as pd\n\nOUT_DIR = Path(\"/kaggle/working/project/artifacts/phase3_branch_b\")\n\nprint(\"OUT_DIR exists:\", OUT_DIR.exists())\nprint(\"Files:\")\nfound = sorted(OUT_DIR.glob(\"*\")) if OUT_DIR.exists() else []\nfor p in found:\n    print(\"-\", p.name)\nif not found:\n    print(\"(none — Branch B training/inference did not write any outputs here.)\")\n\ndef show(name, path):\n    print(f\"\\n{name}\")\n    if path.exists():\n        display(pd.read_csv(path).head())\n    else:\n        print(f\"  -> missing: {path}\")\n\nshow(\"branch_b_oof.csv\", OUT_DIR / \"branch_b_oof.csv\")\n\nsummary_path = OUT_DIR / \"branch_b_summary.csv\"\nif summary_path.exists():\n    print(\"\\nbranch_b_summary.csv\")\n    display(pd.read_csv(summary_path))\nelse:\n    print(\"\\nbranch_b_summary.csv -> missing\")\n\nexact_files = sorted(OUT_DIR.glob(\"disease_enriched_sequences_*.csv\"))\nif exact_files:\n    print(f\"\\n{exact_files[0].name}\")\n    display(pd.read_csv(exact_files[0]).head())\nelse:\n    print(\"\\ndisease_enriched_sequences_*.csv -> none found\")\n\ncluster_files = sorted(OUT_DIR.glob(\"cluster_catalog_*.csv\"))\nif cluster_files:\n    print(f\"\\n{cluster_files[0].name}\")\n    display(pd.read_csv(cluster_files[0]).head())\nelse:\n    print(\"\\ncluster_catalog_*.csv -> none found\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-24T11:32:59.789292Z","iopub.execute_input":"2026-07-24T11:32:59.789595Z","iopub.status.idle":"2026-07-24T11:32:59.798777Z","shell.execute_reply.started":"2026-07-24T11:32:59.789568Z","shell.execute_reply":"2026-07-24T11:32:59.798061Z"}},"outputs":[],"execution_count":null}]}