{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"4ec6eee2","cell_type":"markdown","source":"# Phase 3 — Branch B: Sequence-cluster / Public-signal Model\n\nAdds CDR3 exact/approximate matching, edit-distance/minhash clustering, disease-enriched cluster discovery, and repertoire-level cluster burden features. Lyme-paper public-signature logic + Mal-ID CDR3 clustering + CompAIRR-style overlap acceleration. Outputs branch_b_oof.csv.\n\n---\n\n## Kaggle inputs to add before running this notebook\n\nAdd the following as Kaggle inputs (via the \"Add Input\" button on the right\npanel of the notebook editor):\n- AIRR-ML competition dataset\n- Phase 1 notebook output\n\n## Workflow\n\n1. Run the **SETUP** cell — it creates `/kaggle/working/project/` and auto-merges\n   any previous-phase notebook outputs found under `/kaggle/input/`.\n2. Run each subsequent cell in order. The **Run** cell executes the phase's\n   training script; the **Inspect** cell prints a quick summary of the outputs.\n3. When the run completes, click **Save Version → Save & Run All (Commit)** so\n   the next phase can pick this phase's outputs up via \"Add Input\".\n","metadata":{}},{"id":"e7c6b74e","cell_type":"code","source":"# ============================================================\n# SETUP — Initialize project + merge previous-phase inputs\n# ============================================================\n# This cell:\n#   1. Creates /kaggle/working/project/ fresh (idempotent re-runs).\n#   2. Auto-detects previous-phase notebook outputs under /kaggle/input/.\n#   3. Merges their project/ contents (src/, artifacts/, configs/) into\n#      /kaggle/working/project/ so this phase can build on them.\n#\n# Kaggle \"Add Input\" workflow:\n#   - Phase 1: add the AIRR-ML competition dataset only.\n#   - Phase N (N>=2): add the AIRR-ML competition dataset AND the previous\n#     phase notebook output(s). For Phase 9 add ALL of Phases 1..8.\n# ============================================================\n\nimport os\nimport shutil\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\n# Reset working project dir (idempotent re-runs)\nif PROJECT_ROOT.exists():\n    shutil.rmtree(PROJECT_ROOT)\nPROJECT_ROOT.mkdir(parents=True, exist_ok=True)\n(PROJECT_ROOT / \"src\").mkdir(parents=True, exist_ok=True)\n(PROJECT_ROOT / \"src\" / \"__init__.py\").write_text(\"\", encoding=\"utf-8\")\n(PROJECT_ROOT / \"configs\").mkdir(parents=True, exist_ok=True)\n(PROJECT_ROOT / \"artifacts\").mkdir(parents=True, exist_ok=True)\n\n# Auto-detect and merge previous-phase inputs.\n# Each previous-phase notebook output should contain a top-level `project/`\n# directory (created by Phase 1 and propagated through every later phase).\nprev_inputs_found = []\ninput_root = Path(\"/kaggle/input\")\nif input_root.exists():\n    for entry in sorted(input_root.iterdir()):\n        if not entry.is_dir():\n            continue\n        prev_project = entry / \"project\"\n        if not prev_project.is_dir():\n            continue\n        prev_inputs_found.append(entry.name)\n        for sub in [\"src\", \"artifacts\", \"configs\"]:\n            src_dir = prev_project / sub\n            if not src_dir.exists():\n                continue\n            for path in src_dir.rglob(\"*\"):\n                if path.is_file():\n                    rel = path.relative_to(src_dir)\n                    target = PROJECT_ROOT / sub / rel\n                    target.parent.mkdir(parents=True, exist_ok=True)\n                    shutil.copy2(path, target)\n\nprint(\"PROJECT_ROOT :\", PROJECT_ROOT)\nif prev_inputs_found:\n    print(\"Previous-phase inputs merged:\")\n    for name in prev_inputs_found:\n        print(\"  -\", name)\nelse:\n    print(\"Previous-phase inputs: (none — running fresh)\")\nprint(\"\\nExisting artifacts:\")\nart_dir = PROJECT_ROOT / \"artifacts\"\nif art_dir.exists():\n    for p in sorted(art_dir.glob(\"*\")):\n        print(\"  -\", p.name)\nelse:\n    print(\"  (none)\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-24T10:55:04.205907Z","iopub.execute_input":"2026-07-24T10:55:04.206184Z","iopub.status.idle":"2026-07-24T10:55:04.226672Z","shell.execute_reply.started":"2026-07-24T10:55:04.206152Z","shell.execute_reply":"2026-07-24T10:55:04.225931Z"}},"outputs":[],"execution_count":null},{"id":"039acb7d","cell_type":"code","source":"#Cell 1 — branch_b.yaml write করো\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nbranch_b_yaml = \"\"\"\nenabled: true\n\npaths:\n  output_root: /kaggle/working/project/artifacts/phase3_branch_b\n\ndictionary:\n  max_sequences_per_file: 30000\n  max_repertoires_per_class: 150\n  min_freq: 0.10\n  enrichment: 3.0\n  top_exact: 3000\n  top_clusters: 5000\n\ntraining:\n  seeds: [42, 52, 62]\n\"\"\"\n\n(PROJECT_ROOT / \"configs\" / \"branch_b.yaml\").write_text(branch_b_yaml.strip() + \"\\n\", encoding=\"utf-8\")\nprint(\"Written:\", PROJECT_ROOT / \"configs\" / \"branch_b.yaml\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-24T10:55:04.228195Z","iopub.execute_input":"2026-07-24T10:55:04.228460Z","iopub.status.idle":"2026-07-24T10:55:04.240988Z","shell.execute_reply.started":"2026-07-24T10:55:04.228427Z","shell.execute_reply":"2026-07-24T10:55:04.240216Z"}},"outputs":[],"execution_count":null},{"id":"6fe45c54","cell_type":"code","source":"#Cell 2 — src/branch_b_cluster.py, src/branch_b_public.py, train_branch_b.py write করো\nfrom pathlib import Path\nfrom textwrap import dedent\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nbranch_b_cluster_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nimport itertools\nfrom typing import List, Set\n\nimport numpy as np\n\nfrom .utils import stable_hash\n\n\ndef kmer_set(seq: str, k: int = 3) -> List[str]:\n    if not seq or len(seq) < k:\n        return [seq] if seq else []\n    return list({seq[i:i+k] for i in range(len(seq) - k + 1)})\n\n\ndef minhash_cluster_key(seq: str, k: int = 3, num_hash: int = 6) -> str:\n    kms = kmer_set(seq, k=k)\n    if len(kms) == 0:\n        return f\"mh|L{len(seq)}|EMPTY\"\n    hs = sorted(stable_hash(km, mod=10_000_019) for km in kms)\n    hs = hs[:num_hash]\n    return f\"mh|L{len(seq)}|\" + \"_\".join(map(str, hs))\n\n\ndef deletion_proxy_keys(seq: str, max_del1: int = 4, max_del2: int = 2) -> Set[str]:\n    keys = set()\n    n = len(seq)\n    if n <= 2:\n        return keys\n\n    # 1-deletion proxies\n    idxs = np.linspace(0, n - 1, min(max_del1, n), dtype=int).tolist()\n    for i in sorted(set(idxs)):\n        s = seq[:i] + seq[i+1:]\n        keys.add(f\"d1|L{len(s)}|{stable_hash(s, mod=10_000_019)}\")\n\n    # 2-deletion proxies\n    if n >= 5:\n        cand = sorted(set(idxs))\n        pairs = list(itertools.combinations(cand, 2))[:max_del2]\n        for i, j in pairs:\n            if i == j:\n                continue\n            keep = [ch for idx, ch in enumerate(seq) if idx not in {i, j}]\n            s = \"\".join(keep)\n            keys.add(f\"d2|L{len(s)}|{stable_hash(s, mod=10_000_019)}\")\n\n    return keys\n\n\ndef signature_key(seq: str) -> str:\n    if not seq:\n        return \"sig|EMPTY\"\n    p2 = seq[:2] if len(seq) >= 2 else seq\n    s2 = seq[-2:] if len(seq) >= 2 else seq\n    p3 = seq[:3] if len(seq) >= 3 else seq\n    s3 = seq[-3:] if len(seq) >= 3 else seq\n    return f\"sig|L{len(seq)}|P2{p2}|S2{s2}|P3{p3}|S3{s3}\"\n\n\ndef approx_cluster_keys(seq: str) -> Set[str]:\n    keys = set()\n    if not seq:\n        return keys\n    keys.add(signature_key(seq))\n    keys.add(minhash_cluster_key(seq, k=3, num_hash=6))\n    keys.update(deletion_proxy_keys(seq, max_del1=4, max_del2=2))\n    return keys\n\"\"\")\n\nbranch_b_public_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nfrom collections import Counter\nfrom pathlib import Path\nfrom typing import Dict, Tuple\n\nimport numpy as np\nimport pandas as pd\nfrom joblib import Parallel, delayed\n\nfrom .branch_a_features import read_repertoire, counter_from_df\nfrom .branch_b_cluster import approx_cluster_keys\nfrom .utils import ensure_dir\n\n\ndef _cache_one(row_dict: dict, ds_path: Path, max_sequences_per_file: int, random_state: int):\n    rep_id = row_dict[\"repertoire_id\"]\n    fn = row_dict[\"filename\"]\n    df = read_repertoire(ds_path / fn, max_seqs=max_sequences_per_file, random_state=random_state)\n    seq_counter, _ = counter_from_df(df)\n    total_weight = float(sum(seq_counter.values()))\n    return rep_id, {\n        \"seq_counter\": seq_counter,\n        \"total_weight\": total_weight,\n    }\n\n\ndef build_dataset_cache(meta_ds: pd.DataFrame, ds_path: Path, max_sequences_per_file: int = 30000, random_state: int = 42, n_jobs: int = 4):\n    rows = meta_ds[[\"repertoire_id\", \"filename\"]].drop_duplicates().to_dict(orient=\"records\")\n    cached = Parallel(n_jobs=n_jobs, backend=\"loky\")(\n        delayed(_cache_one)(r, ds_path, max_sequences_per_file, random_state)\n        for r in rows\n    )\n    return dict(cached)\n\n\ndef _maybe_cap_rows(df: pd.DataFrame, max_repertoires_per_class: int | None, random_state: int = 42):\n    if max_repertoires_per_class is None:\n        return df.copy()\n    if len(df) <= max_repertoires_per_class:\n        return df.copy()\n    return df.sample(n=max_repertoires_per_class, random_state=random_state).copy()\n\n\ndef build_fold_signal_catalog(\n    fold_train_meta: pd.DataFrame,\n    cache: Dict[str, Dict],\n    min_freq: float = 0.10,\n    enrichment: float = 3.0,\n    top_exact: int = 3000,\n    top_clusters: int = 5000,\n    max_repertoires_per_class: int | None = 150,\n    random_state: int = 42,\n):\n    meta = fold_train_meta.copy()\n    meta[\"label_positive\"] = pd.to_numeric(meta[\"label_positive\"], errors=\"coerce\").fillna(0).astype(int)\n\n    pos_rows = _maybe_cap_rows(meta[meta[\"label_positive\"] == 1], max_repertoires_per_class, random_state)\n    neg_rows = _maybe_cap_rows(meta[meta[\"label_positive\"] == 0], max_repertoires_per_class, random_state)\n\n    pos_exact = Counter()\n    neg_exact = Counter()\n\n    pos_exact_weight = Counter()\n    neg_exact_weight = Counter()\n\n    pos_cluster = Counter()\n    neg_cluster = Counter()\n\n    def update_from_rows(rows, exact_counter, exact_weight_counter, cluster_counter):\n        for _, row in rows.iterrows():\n            rep_id = row[\"repertoire_id\"]\n            if rep_id not in cache:\n                continue\n            seq_counter = cache[rep_id][\"seq_counter\"]\n            seqs = list(seq_counter.keys())\n\n            exact_counter.update(seqs)\n            exact_weight_counter.update(seq_counter)\n\n            rep_cluster_keys = set()\n            for seq in seqs:\n                rep_cluster_keys.update(approx_cluster_keys(seq))\n            cluster_counter.update(rep_cluster_keys)\n\n    update_from_rows(pos_rows, pos_exact, pos_exact_weight, pos_cluster)\n    update_from_rows(neg_rows, neg_exact, neg_exact_weight, neg_cluster)\n\n    n_pos = max(1, len(pos_rows))\n    n_neg = max(1, len(neg_rows))\n\n    exact_rows = []\n    for seq, count in pos_exact.items():\n        pf = count / n_pos\n        nf = neg_exact.get(seq, 0) / n_neg\n        if pf >= min_freq and pf > nf * enrichment:\n            score = float(np.log((pf + 1e-6) / (nf + 1e-6)))\n            exact_rows.append({\n                \"key\": seq,\n                \"score\": score,\n                \"pos_freq\": pf,\n                \"neg_freq\": nf,\n                \"pos_weight_sum\": float(pos_exact_weight.get(seq, 0.0)),\n                \"neg_weight_sum\": float(neg_exact_weight.get(seq, 0.0)),\n                \"kind\": \"exact_sequence\",\n            })\n\n    cluster_rows = []\n    for key, count in pos_cluster.items():\n        pf = count / n_pos\n        nf = neg_cluster.get(key, 0) / n_neg\n        if pf >= min_freq and pf > nf * enrichment:\n            score = float(np.log((pf + 1e-6) / (nf + 1e-6)))\n            cluster_rows.append({\n                \"key\": key,\n                \"score\": score,\n                \"pos_freq\": pf,\n                \"neg_freq\": nf,\n                \"kind\": \"approx_cluster\",\n            })\n\n    exact_df = pd.DataFrame(exact_rows).sort_values(\"score\", ascending=False).head(top_exact) if len(exact_rows) else pd.DataFrame(columns=[\"key\", \"score\", \"pos_freq\", \"neg_freq\", \"pos_weight_sum\", \"neg_weight_sum\", \"kind\"])\n    cluster_df = pd.DataFrame(cluster_rows).sort_values(\"score\", ascending=False).head(top_clusters) if len(cluster_rows) else pd.DataFrame(columns=[\"key\", \"score\", \"pos_freq\", \"neg_freq\", \"kind\"])\n\n    exact_catalog = {row[\"key\"]: row for row in exact_df.to_dict(orient=\"records\")}\n    cluster_catalog = {row[\"key\"]: row for row in cluster_df.to_dict(orient=\"records\")}\n\n    bundle = {\n        \"exact_catalog\": exact_catalog,\n        \"cluster_catalog\": cluster_catalog,\n        \"n_pos_repertoires\": n_pos,\n        \"n_neg_repertoires\": n_neg,\n    }\n    return bundle, exact_df, cluster_df\n\n\ndef extract_branch_b_features_from_counter(seq_counter: Counter, bundle: dict) -> dict:\n    features = {\n        \"public_seq_count\": 0.0,\n        \"public_seq_weighted_score\": 0.0,\n        \"public_seq_hit_weight\": 0.0,\n        \"cluster_hit_count\": 0.0,\n        \"cluster_hit_weight\": 0.0,\n        \"cluster_enrichment_sum\": 0.0,\n        \"max_enriched_cluster_score\": 0.0,\n        \"exact_or_cluster_hit_count\": 0.0,\n        \"cluster_density\": 0.0,\n        \"public_cluster_combo_score\": 0.0,\n    }\n\n    if seq_counter is None or len(seq_counter) == 0:\n        return features\n\n    exact_catalog = bundle.get(\"exact_catalog\", {})\n    cluster_catalog = bundle.get(\"cluster_catalog\", {})\n\n    total_weight = float(sum(seq_counter.values()))\n    if total_weight <= 0:\n        return features\n\n    exact_hit_count = 0\n    exact_hit_weight = 0.0\n    exact_score_sum = 0.0\n\n    cluster_hit_count = 0\n    cluster_hit_weight = 0.0\n    cluster_score_sum = 0.0\n    max_cluster_score = 0.0\n\n    combined_hit_count = 0\n\n    for seq, cw in seq_counter.items():\n        exact_hit = False\n        cluster_hit = False\n\n        if seq in exact_catalog:\n            sc = float(exact_catalog[seq][\"score\"])\n            exact_hit = True\n            exact_hit_count += 1\n            exact_hit_weight += cw\n            exact_score_sum += sc * np.log1p(cw)\n\n        seq_best_cluster = 0.0\n        for key in approx_cluster_keys(seq):\n            if key in cluster_catalog:\n                cluster_hit = True\n                seq_best_cluster = max(seq_best_cluster, float(cluster_catalog[key][\"score\"]))\n\n        if cluster_hit:\n            cluster_hit_count += 1\n            cluster_hit_weight += cw\n            cluster_score_sum += seq_best_cluster * np.log1p(cw)\n            max_cluster_score = max(max_cluster_score, seq_best_cluster)\n\n        if exact_hit or cluster_hit:\n            combined_hit_count += 1\n\n    n_unique = max(1, len(seq_counter))\n    features[\"public_seq_count\"] = float(exact_hit_count)\n    features[\"public_seq_weighted_score\"] = float(exact_score_sum)\n    features[\"public_seq_hit_weight\"] = float(exact_hit_weight / total_weight)\n\n    features[\"cluster_hit_count\"] = float(cluster_hit_count)\n    features[\"cluster_hit_weight\"] = float(cluster_hit_weight / total_weight)\n    features[\"cluster_enrichment_sum\"] = float(cluster_score_sum)\n    features[\"max_enriched_cluster_score\"] = float(max_cluster_score)\n\n    features[\"exact_or_cluster_hit_count\"] = float(combined_hit_count)\n    features[\"cluster_density\"] = float(cluster_hit_count / n_unique)\n    features[\"public_cluster_combo_score\"] = float(exact_score_sum + cluster_score_sum)\n    return features\n\n\ndef build_feature_table_from_meta(meta_df: pd.DataFrame, cache: Dict[str, Dict], bundle: dict):\n    rows = []\n    for _, row in meta_df.iterrows():\n        rep_id = row[\"repertoire_id\"]\n        seq_counter = cache.get(rep_id, {}).get(\"seq_counter\", Counter())\n        feats = extract_branch_b_features_from_counter(seq_counter, bundle)\n        rows.append({\n            \"ID\": rep_id,\n            \"dataset\": row[\"dataset_name\"],\n            \"label_positive\": int(row[\"label_positive\"]) if pd.notna(row[\"label_positive\"]) else pd.NA,\n            **feats,\n        })\n    return pd.DataFrame(rows)\n\n\ndef save_catalogs(exact_df: pd.DataFrame, cluster_df: pd.DataFrame, out_dir: Path, ds_name: str):\n    out_dir = ensure_dir(out_dir)\n    exact_df.to_csv(out_dir / f\"disease_enriched_sequences_{ds_name}.csv\", index=False)\n    cluster_df.to_csv(out_dir / f\"cluster_catalog_{ds_name}.csv\", index=False)\n\"\"\")\n\ntrain_branch_b_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nimport sys\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport yaml\n\nPROJECT_ROOT = Path(__file__).resolve().parent\nif str(PROJECT_ROOT) not in sys.path:\n    sys.path.insert(0, str(PROJECT_ROOT))\n\nfrom src.branch_a_models import (\n    _train_cat,\n    _train_lgb,\n    _train_lr,\n    _train_xgb,\n    check_gpu,\n    safe_auc,\n)\nfrom src.branch_b_public import (\n    build_dataset_cache,\n    build_feature_table_from_meta,\n    build_fold_signal_catalog,\n)\nfrom src.utils import ensure_dir, seed_everything, dataset_id_from_name\n\n\nSCALE_POS_WEIGHT = {\n    1: 1.0, 2: 1.0, 3: 1.0, 4: 1.0, 5: 1.0, 6: 1.0, 7: 5.04, 8: 2.05\n}\n\n\ndef main():\n    cfg_base = yaml.safe_load((PROJECT_ROOT / \"configs\" / \"base.yaml\").read_text())\n    cfg_branch = yaml.safe_load((PROJECT_ROOT / \"configs\" / \"branch_b.yaml\").read_text())\n\n    runtime = cfg_base[\"runtime\"]\n    phase1_paths = cfg_base[\"paths\"]\n    out_dir = ensure_dir(cfg_branch[\"paths\"][\"output_root\"])\n\n    seed_everything(runtime[\"random_state\"])\n    gpu_ok = check_gpu()\n\n    canonical = pd.read_csv(Path(phase1_paths[\"output_root\"]) / \"canonical_metadata.csv\")\n    all_folds = pd.read_csv(Path(phase1_paths[\"output_root\"]) / \"all_folds.csv\")\n\n    train_root = Path(phase1_paths[\"train_root\"])\n    seeds = list(cfg_branch[\"training\"][\"seeds\"])\n\n    train_meta_all = canonical[canonical[\"source\"] == \"train\"].copy()\n    train_meta_all = train_meta_all[pd.notna(train_meta_all[\"label_positive\"])].copy()\n\n    dataset_names = sorted(train_meta_all[\"dataset_name\"].unique().tolist())\n\n    all_oof = []\n    all_summary = []\n\n    print(\"=\" * 80)\n    print(\"PHASE 3: Branch B - sequence cluster / public signal\")\n    print(\"=\" * 80)\n    print(\"Datasets:\", dataset_names)\n    print(\"GPU available:\", gpu_ok)\n\n    for ds_name in dataset_names:\n        ds_id = dataset_id_from_name(ds_name)\n        ds_scale = SCALE_POS_WEIGHT.get(ds_id, 1.0)\n        ds_path = train_root / ds_name\n\n        print(f\"\\\\n{'-' * 80}\")\n        print(f\"Branch B training: {ds_name} (id={ds_id})\")\n        print(f\"{'-' * 80}\")\n\n        meta_ds = train_meta_all[train_meta_all[\"dataset_name\"] == ds_name].copy()\n        fold_ds = all_folds[all_folds[\"dataset_name\"] == ds_name].copy()\n\n        meta_ds = meta_ds.merge(\n            fold_ds[[\"dataset_name\", \"repertoire_id\", \"fold\", \"splitter\", \"n_splits_used\"]],\n            on=[\"dataset_name\", \"repertoire_id\"],\n            how=\"inner\",\n        ).reset_index(drop=True)\n\n        cache = build_dataset_cache(\n            meta_ds=meta_ds,\n            ds_path=ds_path,\n            max_sequences_per_file=cfg_branch[\"dictionary\"][\"max_sequences_per_file\"],\n            random_state=runtime[\"random_state\"],\n            n_jobs=runtime[\"n_jobs\"],\n        )\n\n        seed_frames = []\n        summary_rows = []\n        all_exact_rows = []\n        all_cluster_rows = []\n\n        unique_folds = sorted(meta_ds[\"fold\"].dropna().unique().tolist())\n        model_names = [\"LR\", \"XGB\", \"LGB\", \"CAT\"]\n\n        for seed in seeds:\n            pred_store = {\n                \"ID\": meta_ds[\"repertoire_id\"].astype(str).tolist(),\n                \"dataset\": meta_ds[\"dataset_name\"].astype(str).tolist(),\n                \"label_positive\": meta_ds[\"label_positive\"].astype(int).tolist(),\n                \"fold\": meta_ds[\"fold\"].tolist(),\n                \"seed\": [seed] * len(meta_ds),\n            }\n            for m in model_names:\n                pred_store[m] = [np.nan] * len(meta_ds)\n\n            for fold in unique_folds:\n                tr_meta = meta_ds[meta_ds[\"fold\"] != fold].copy().reset_index(drop=True)\n                va_meta = meta_ds[meta_ds[\"fold\"] == fold].copy().reset_index(drop=True)\n                va_global_idx = meta_ds.index[meta_ds[\"fold\"] == fold].to_numpy()\n\n                bundle, exact_df, cluster_df = build_fold_signal_catalog(\n                    fold_train_meta=tr_meta,\n                    cache=cache,\n                    min_freq=cfg_branch[\"dictionary\"][\"min_freq\"],\n                    enrichment=cfg_branch[\"dictionary\"][\"enrichment\"],\n                    top_exact=cfg_branch[\"dictionary\"][\"top_exact\"],\n                    top_clusters=cfg_branch[\"dictionary\"][\"top_clusters\"],\n                    max_repertoires_per_class=cfg_branch[\"dictionary\"][\"max_repertoires_per_class\"],\n                    random_state=seed,\n                )\n\n                if len(exact_df):\n                    exact_df = exact_df.copy()\n                    exact_df.insert(0, \"dataset\", ds_name)\n                    exact_df.insert(1, \"fold\", fold)\n                    all_exact_rows.append(exact_df)\n\n                if len(cluster_df):\n                    cluster_df = cluster_df.copy()\n                    cluster_df.insert(0, \"dataset\", ds_name)\n                    cluster_df.insert(1, \"fold\", fold)\n                    all_cluster_rows.append(cluster_df)\n\n                X_tr_df = build_feature_table_from_meta(tr_meta, cache, bundle)\n                X_va_df = build_feature_table_from_meta(va_meta, cache, bundle)\n\n                feature_cols = [c for c in X_tr_df.columns if c not in [\"ID\", \"dataset\", \"label_positive\"]]\n                X_tr = X_tr_df[feature_cols].fillna(0.0).astype(np.float32).values\n                X_va = X_va_df[feature_cols].fillna(0.0).astype(np.float32).values\n                y_tr = X_tr_df[\"label_positive\"].astype(int).values\n                y_va = X_va_df[\"label_positive\"].astype(int).values\n\n                _, pred_lr = _train_lr(X_tr, y_tr, X_va)\n                for gidx, p in zip(va_global_idx, pred_lr):\n                    pred_store[\"LR\"][gidx] = float(p)\n                summary_rows.append({\"dataset\": ds_name, \"seed\": seed, \"fold\": fold, \"model\": \"LR\", \"auc\": safe_auc(y_va, pred_lr)})\n\n                model, pred_xgb = _train_xgb(X_tr, y_tr, X_va, y_va, ds_scale, seed, gpu_ok)\n                if pred_xgb is not None:\n                    for gidx, p in zip(va_global_idx, pred_xgb):\n                        pred_store[\"XGB\"][gidx] = float(p)\n                    summary_rows.append({\"dataset\": ds_name, \"seed\": seed, \"fold\": fold, \"model\": \"XGB\", \"auc\": safe_auc(y_va, pred_xgb)})\n\n                model, pred_lgb = _train_lgb(X_tr, y_tr, X_va, y_va, ds_scale, seed, gpu_ok)\n                if pred_lgb is not None:\n                    for gidx, p in zip(va_global_idx, pred_lgb):\n                        pred_store[\"LGB\"][gidx] = float(p)\n                    summary_rows.append({\"dataset\": ds_name, \"seed\": seed, \"fold\": fold, \"model\": \"LGB\", \"auc\": safe_auc(y_va, pred_lgb)})\n\n                model, pred_cat = _train_cat(X_tr, y_tr, X_va, y_va, seed, gpu_ok)\n                if pred_cat is not None:\n                    for gidx, p in zip(va_global_idx, pred_cat):\n                        pred_store[\"CAT\"][gidx] = float(p)\n                    summary_rows.append({\"dataset\": ds_name, \"seed\": seed, \"fold\": fold, \"model\": \"CAT\", \"auc\": safe_auc(y_va, pred_cat)})\n\n            pred_df = pd.DataFrame(pred_store)\n            valid_model_cols = [c for c in model_names if c in pred_df.columns and pred_df[c].notna().any()]\n            pred_df[\"AVG\"] = pred_df[valid_model_cols].mean(axis=1)\n            seed_frames.append(pred_df)\n\n            for m in valid_model_cols + [\"AVG\"]:\n                auc_full = safe_auc(pred_df[\"label_positive\"].values, pred_df[m].values)\n                summary_rows.append({\"dataset\": ds_name, \"seed\": seed, \"fold\": -1, \"model\": m, \"auc\": auc_full})\n\n        ds_oof = pd.concat(seed_frames, ignore_index=True)\n        ds_oof.to_csv(out_dir / f\"branch_b_oof_{ds_name}.csv\", index=False)\n        all_oof.append(ds_oof)\n\n        summary_df = pd.DataFrame(summary_rows)\n        if len(summary_df):\n            summary_df.to_csv(out_dir / f\"branch_b_summary_{ds_name}.csv\", index=False)\n        all_summary.append(summary_df)\n\n        exact_all = pd.concat(all_exact_rows, ignore_index=True) if len(all_exact_rows) else pd.DataFrame(columns=[\"dataset\", \"fold\", \"key\", \"score\", \"pos_freq\", \"neg_freq\", \"pos_weight_sum\", \"neg_weight_sum\", \"kind\"])\n        cluster_all = pd.concat(all_cluster_rows, ignore_index=True) if len(all_cluster_rows) else pd.DataFrame(columns=[\"dataset\", \"fold\", \"key\", \"score\", \"pos_freq\", \"neg_freq\", \"kind\"])\n\n        exact_all.to_csv(out_dir / f\"disease_enriched_sequences_{ds_name}.csv\", index=False)\n        cluster_all.to_csv(out_dir / f\"cluster_catalog_{ds_name}.csv\", index=False)\n\n        print(\"\\\\nOOF preview:\")\n        print(ds_oof.head().to_string(index=False))\n\n        if len(summary_df):\n            overall = (\n                summary_df[summary_df[\"fold\"] >= 0]\n                .groupby([\"dataset\", \"seed\", \"model\"], as_index=False)[\"auc\"]\n                .agg(mean_auc=\"mean\", std_auc=\"std\", max_auc=\"max\")\n                .sort_values([\"dataset\", \"seed\", \"mean_auc\"], ascending=[True, True, False])\n            )\n            print(\"\\\\nSummary:\")\n            print(overall.to_string(index=False))\n\n        print(f\"Saved exact catalog -> {out_dir / f'disease_enriched_sequences_{ds_name}.csv'}\")\n        print(f\"Saved cluster catalog -> {out_dir / f'cluster_catalog_{ds_name}.csv'}\")\n\n    final_oof = pd.concat(all_oof, ignore_index=True) if len(all_oof) else pd.DataFrame()\n    final_oof.to_csv(out_dir / \"branch_b_oof.csv\", index=False)\n\n    final_summary = pd.concat(all_summary, ignore_index=True) if len(all_summary) else pd.DataFrame()\n    if len(final_summary):\n        agg = (\n            final_summary[final_summary[\"fold\"] >= 0]\n            .groupby([\"dataset\", \"seed\", \"model\"], as_index=False)[\"auc\"]\n            .agg(mean_auc=\"mean\", std_auc=\"std\", max_auc=\"max\")\n            .sort_values([\"dataset\", \"seed\", \"mean_auc\"], ascending=[True, True, False])\n        )\n        agg.to_csv(out_dir / \"branch_b_summary.csv\", index=False)\n    else:\n        pd.DataFrame().to_csv(out_dir / \"branch_b_summary.csv\", index=False)\n\n    print(\"\\\\n\" + \"=\" * 80)\n    print(\"PHASE 3 DONE\")\n    print(\"Saved:\")\n    print(out_dir / \"branch_b_oof.csv\")\n    print(out_dir / \"branch_b_summary.csv\")\n    print(\"=\" * 80)\n\n\nif __name__ == \"__main__\":\n    main()\n\"\"\")\n\n(PROJECT_ROOT / \"src\" / \"branch_b_cluster.py\").write_text(branch_b_cluster_py, encoding=\"utf-8\")\n(PROJECT_ROOT / \"src\" / \"branch_b_public.py\").write_text(branch_b_public_py, encoding=\"utf-8\")\n(PROJECT_ROOT / \"train_branch_b.py\").write_text(train_branch_b_py, encoding=\"utf-8\")\n\nprint(\"Written:\")\nprint(\"-\", PROJECT_ROOT / \"src\" / \"branch_b_cluster.py\")\nprint(\"-\", PROJECT_ROOT / \"src\" / \"branch_b_public.py\")\nprint(\"-\", PROJECT_ROOT / \"train_branch_b.py\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-24T10:55:04.318469Z","iopub.execute_input":"2026-07-24T10:55:04.318913Z","iopub.status.idle":"2026-07-24T10:55:04.346362Z","shell.execute_reply.started":"2026-07-24T10:55:04.318890Z","shell.execute_reply":"2026-07-24T10:55:04.345544Z"}},"outputs":[],"execution_count":null},{"id":"0a91163a-59c3-4a54-b053-1db1576f4195","cell_type":"code","source":"#Cell 3 — imports verify করো\nfrom pathlib import Path\np = Path(\"/kaggle/working/project/src\")\nprint(\"src/ exists:\", p.exists())\nprint(\"contents:\", sorted(f.name for f in p.glob(\"*\")) if p.exists() else \"N/A\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-24T11:02:34.163623Z","iopub.execute_input":"2026-07-24T11:02:34.164155Z","iopub.status.idle":"2026-07-24T11:02:34.171351Z","shell.execute_reply.started":"2026-07-24T11:02:34.164108Z","shell.execute_reply":"2026-07-24T11:02:34.170508Z"}},"outputs":[],"execution_count":null},{"id":"47dccc15","cell_type":"code","source":"#Cell 4 — Phase 3 run করো\n!python /kaggle/working/project/train_branch_b.py","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-24T11:02:34.373493Z","iopub.execute_input":"2026-07-24T11:02:34.374019Z","iopub.status.idle":"2026-07-24T11:02:35.015119Z","shell.execute_reply.started":"2026-07-24T11:02:34.373996Z","shell.execute_reply":"2026-07-24T11:02:35.014135Z"}},"outputs":[],"execution_count":null},{"id":"a6531979","cell_type":"code","source":"#Cell 5 — outputs inspect করো\nfrom pathlib import Path\nimport pandas as pd\n\nOUT_DIR = Path(\"/kaggle/working/project/artifacts/phase3_branch_b\")\n\nprint(\"OUT_DIR exists:\", OUT_DIR.exists())\nprint(\"Files:\")\nfound = sorted(OUT_DIR.glob(\"*\")) if OUT_DIR.exists() else []\nfor p in found:\n    print(\"-\", p.name)\nif not found:\n    print(\"(none — Branch B training/inference did not write any outputs here.)\")\n\ndef show(name, path):\n    print(f\"\\n{name}\")\n    if path.exists():\n        display(pd.read_csv(path).head())\n    else:\n        print(f\"  -> missing: {path}\")\n\nshow(\"branch_b_oof.csv\", OUT_DIR / \"branch_b_oof.csv\")\n\nsummary_path = OUT_DIR / \"branch_b_summary.csv\"\nif summary_path.exists():\n    print(\"\\nbranch_b_summary.csv\")\n    display(pd.read_csv(summary_path))\nelse:\n    print(\"\\nbranch_b_summary.csv -> missing\")\n\nexact_files = sorted(OUT_DIR.glob(\"disease_enriched_sequences_*.csv\"))\nif exact_files:\n    print(f\"\\n{exact_files[0].name}\")\n    display(pd.read_csv(exact_files[0]).head())\nelse:\n    print(\"\\ndisease_enriched_sequences_*.csv -> none found\")\n\ncluster_files = sorted(OUT_DIR.glob(\"cluster_catalog_*.csv\"))\nif cluster_files:\n    print(f\"\\n{cluster_files[0].name}\")\n    display(pd.read_csv(cluster_files[0]).head())\nelse:\n    print(\"\\ncluster_catalog_*.csv -> none found\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-24T11:03:16.663568Z","iopub.execute_input":"2026-07-24T11:03:16.664261Z","iopub.status.idle":"2026-07-24T11:03:16.673483Z","shell.execute_reply.started":"2026-07-24T11:03:16.664226Z","shell.execute_reply":"2026-07-24T11:03:16.672664Z"}},"outputs":[],"execution_count":null}]}