{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":106680,"databundleVersionId":13374319}],"dockerImageVersionId":31286,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-19T20:51:55.424647Z","iopub.execute_input":"2026-03-19T20:51:55.424876Z","iopub.status.idle":"2026-03-19T20:51:59.623200Z","shell.execute_reply.started":"2026-03-19T20:51:55.424854Z","shell.execute_reply":"2026-03-19T20:51:59.622331Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nAIRR-ML-25 Challenge - ADVANCED MULTI-MODEL GPU PIPELINE\n========================================================\nWhat is added vs your original version:\n- Stronger feature engineering\n- Multiple models: XGB + LGB + CatBoost + LogisticRegression\n- OOF CV tracking for every model\n- Result tables (CSV) + comparison figures (PNG)\n- Non-negative stacked ensemble\n- EDA summary export\n- Selected feature export\n- Fixed submission generation from sample_submissions.csv\n\nOutputs:\n- /kaggle/working/artifacts/eda_summary.csv\n- /kaggle/working/artifacts/cv_fold_results.csv\n- /kaggle/working/artifacts/cv_summary.csv\n- /kaggle/working/artifacts/cv_pivot_mean_auc.csv\n- /kaggle/working/artifacts/model_compare_.png\n- /kaggle/working/artifacts/top_features_.csv\n- /kaggle/working/best_submission.csv\n\nNotes:\n- This version focuses on a reliable tabular ensemble that often beats a single XGB.\n- Protein LM / pretrained sequence embeddings (ESM2 / ProtBERT) can help, but in Kaggle offline/GPU\n environments they are often brittle and expensive. Keep this pipeline as the strong baseline first.\n\"\"\"\n\nimport os\nimport gc\nimport re\nimport math\nimport time\nimport hashlib\nimport warnings\nimport subprocess\nfrom pathlib import Path\nfrom collections import Counter, defaultdict\nfrom typing import Dict, Optional, List, Tuple\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom joblib import Parallel, delayed\n\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler\n\nimport xgboost as xgb\nimport lightgbm as lgb\n\ntry:\n from catboost import CatBoostClassifier\n HAS_CATBOOST = True\nexcept Exception:\n HAS_CATBOOST = False\n\nwarnings.filterwarnings(\"ignore\")\n\n\n# ============================================================\n# CONFIG\n# ============================================================\nclass Config:\n DATA_ROOT = Path(\"/kaggle/input/competitions/adaptive-immune-profiling-challenge-2025/\")\n TRAIN_DIR = DATA_ROOT / \"train_datasets\" / \"train_datasets\"\n TEST_DIR = DATA_ROOT / \"test_datasets\" / \"test_datasets\"\n SAMPLE_SUBMISSION = DATA_ROOT / \"sample_submissions.csv\"\n\n OUT_DIR = Path(\"/kaggle/working/artifacts\")\n OUT_DIR.mkdir(parents=True, exist_ok=True)\n\n RANDOM_STATE = 42\n N_JOBS = min(8, os.cpu_count() or 4)\n\n # Sequence processing\n K_LIST = [3, 4]\n MAX_SEQUENCES_PER_FILE = 60000\n TOP_KMERS_PER_FILE = {3: 400, 4: 600}\n TOP_POS_MOTIFS = 25\n\n # Public clone mining\n PUB_MAX_FILES = 20\n PUB_MIN_FREQ = 0.18\n PUB_ENRICH = 6.0\n PUB_TOP_N = {7: 5000, 8: 3000, \"default\": 2500}\n\n # CV / training\n N_SPLITS = 5\n EARLY_STOP = 80\n TOP_FEATURES = 650\n\n SCALE_POS_WEIGHT = {\n 1: 1.0, 2: 1.0, 3: 1.0, 4: 1.0, 5: 1.0, 6: 1.0, 7: 5.04, 8: 2.05\n }\n\n\n# ============================================================\n# AMINO ACID PROPERTIES\n# ============================================================\nAA_PROPERTIES = {\n \"A\": {\"hydro\": 1.8, \"vol\": 88.6, \"charge\": 0.0, \"polar\": 0},\n \"R\": {\"hydro\": -4.5, \"vol\": 173.4, \"charge\": 1.0, \"polar\": 1},\n \"N\": {\"hydro\": -3.5, \"vol\": 114.1, \"charge\": 0.0, \"polar\": 1},\n \"D\": {\"hydro\": -3.5, \"vol\": 111.1, \"charge\": -1.0, \"polar\": 1},\n \"C\": {\"hydro\": 2.5, \"vol\": 108.5, \"charge\": 0.0, \"polar\": 0},\n \"Q\": {\"hydro\": -3.5, \"vol\": 143.8, \"charge\": 0.0, \"polar\": 1},\n \"E\": {\"hydro\": -3.5, \"vol\": 138.4, \"charge\": -1.0, \"polar\": 1},\n \"G\": {\"hydro\": -0.4, \"vol\": 60.1, \"charge\": 0.0, \"polar\": 0},\n \"H\": {\"hydro\": -3.2, \"vol\": 153.2, \"charge\": 0.5, \"polar\": 1},\n \"I\": {\"hydro\": 4.5, \"vol\": 166.7, \"charge\": 0.0, \"polar\": 0},\n \"L\": {\"hydro\": 3.8, \"vol\": 166.7, \"charge\": 0.0, \"polar\": 0},\n \"K\": {\"hydro\": -3.9, \"vol\": 168.6, \"charge\": 1.0, \"polar\": 1},\n \"M\": {\"hydro\": 1.9, \"vol\": 162.9, \"charge\": 0.0, \"polar\": 0},\n \"F\": {\"hydro\": 2.8, \"vol\": 189.9, \"charge\": 0.0, \"polar\": 0},\n \"P\": {\"hydro\": -1.6, \"vol\": 112.7, \"charge\": 0.0, \"polar\": 0},\n \"S\": {\"hydro\": -0.8, \"vol\": 89.0, \"charge\": 0.0, \"polar\": 1},\n \"T\": {\"hydro\": -0.7, \"vol\": 116.1, \"charge\": 0.0, \"polar\": 1},\n \"W\": {\"hydro\": -0.9, \"vol\": 227.8, \"charge\": 0.0, \"polar\": 0},\n \"Y\": {\"hydro\": -1.3, \"vol\": 193.6, \"charge\": 0.0, \"polar\": 1},\n \"V\": {\"hydro\": 4.2, \"vol\": 140.0, \"charge\": 0.0, \"polar\": 0},\n}\n\nAA_LIST = sorted(list(AA_PROPERTIES.keys()))\nAROMATIC = set([\"F\", \"W\", \"Y\"])\nSMALL = set([\"A\", \"G\", \"S\", \"T\", \"P\"])\nHYDROPHOBIC = set([\"A\", \"V\", \"I\", \"L\", \"M\", \"F\", \"W\", \"Y\", \"C\"])\nPOLAR = set([aa for aa, p in AA_PROPERTIES.items() if p[\"polar\"] == 1])\n\n\n# ============================================================\n# UTILS\n# ============================================================\ndef seed_everything(seed: int = 42):\n np.random.seed(seed)\n os.environ[\"PYTHONHASHSEED\"] = str(seed)\n\n\ndef stable_hash(x: str, mod: int = 1000) -> float:\n s = str(x).encode(\"utf-8\")\n h = hashlib.md5(s).hexdigest()[:8]\n return (int(h, 16) % mod) / mod\n\n\ndef dataset_id_from_name(name: str) -> int:\n m = re.search(r\"(\\d+)\", str(name))\n return int(m.group(1)) if m else 1\n\n\ndef check_gpu() -> bool:\n try:\n r = subprocess.run([\"nvidia-smi\"], capture_output=True, text=True, timeout=5)\n return r.returncode == 0\n except Exception:\n return False\n\n\ndef get_gpu_memory() -> str:\n try:\n result = subprocess.run(\n [\"nvidia-smi\", \"--query-gpu=memory.used,memory.total\", \"--format=csv,noheader,nounits\"],\n capture_output=True, text=True, timeout=5,\n )\n if result.returncode == 0 and result.stdout.strip():\n used, total = map(int, result.stdout.strip().split(\",\"))\n return f\"{used}/{total} MB\"\n except Exception:\n pass\n return \"N/A\"\n\n\ndef safe_auc(y_true, y_prob) -> float:\n return float(roc_auc_score(y_true, y_prob))\n\n\ndef weighted_quantile(values, weights, q):\n values = np.asarray(values)\n weights = np.asarray(weights)\n if len(values) == 0:\n return np.nan\n sorter = np.argsort(values)\n values = values[sorter]\n weights = weights[sorter]\n cdf = np.cumsum(weights) / np.sum(weights)\n return float(np.interp(q, cdf, values))\n\n\n# ============================================================\n# DATA IO\n# ============================================================\ndef read_repertoire(tsv_path: Path, max_seqs: Optional[int] = None) -> pd.DataFrame:\n cols = [\"junction_aa\", \"v_call\", \"j_call\", \"templates\"]\n try:\n header = pd.read_csv(tsv_path, sep=\"\\t\", nrows=0)\n usecols = [c for c in cols if c in header.columns]\n df = pd.read_csv(tsv_path, sep=\"\\t\", usecols=usecols)\n except Exception:\n return pd.DataFrame(columns=cols)\n\n for c in cols:\n if c not in df.columns:\n df[c] = \"\" if c != \"templates\" else 1.0\n\n df[\"junction_aa\"] = df[\"junction_aa\"].fillna(\"\").astype(str)\n df[\"templates\"] = pd.to_numeric(df[\"templates\"], errors=\"coerce\").fillna(1.0).clip(lower=1.0)\n\n if max_seqs and len(df) > max_seqs:\n weights = df[\"templates\"].values.astype(float)\n weights = weights / weights.sum()\n idx = np.random.choice(len(df), size=max_seqs, replace=False, p=weights)\n df = df.iloc[idx].reset_index(drop=True)\n\n return df\n\n\n# ============================================================\n# FEATURE ENGINEERING\n# ============================================================\nclass FeatureExtractor:\n def __init__(self, k_list=None):\n self.k_list = k_list or [3, 4]\n\n @staticmethod\n def gene_family(gene_call: str) -> str:\n if not isinstance(gene_call, str) or not gene_call:\n return \"UNK\"\n return gene_call.split(\"*\")[0].split(\"-\")[0].upper() or \"UNK\"\n\n def extract_all(\n self,\n df: pd.DataFrame,\n pub_dict: Optional[Dict] = None,\n meta_row: Optional[pd.Series] = None,\n ds_id: int = 1,\n ) -> Dict[str, float]:\n features: Dict[str, float] = {}\n\n if df is None or len(df) == 0:\n return features\n\n seqs_raw = df[\"junction_aa\"].fillna(\"\").astype(str).tolist()\n tmpls_raw = pd.to_numeric(df[\"templates\"], errors=\"coerce\").fillna(1.0).clip(lower=1.0).tolist()\n\n seqs, weights = [], []\n for s, w in zip(seqs_raw, tmpls_raw):\n if s and isinstance(s, str):\n seqs.append(s)\n weights.append(float(w))\n\n if len(seqs) == 0:\n return features\n\n weights = np.asarray(weights, dtype=np.float32)\n total_weight = float(weights.sum())\n\n # -------------------------\n # Basic counts / EDA-like\n # -------------------------\n unique_clone_counter = Counter()\n for s, w in zip(seqs, weights):\n unique_clone_counter[s] += float(w)\n\n lens = np.array([len(s) for s in seqs], dtype=np.float32)\n features[\"seq_count\"] = float(len(seqs))\n features[\"unique_seq_count\"] = float(len(unique_clone_counter))\n features[\"unique_ratio\"] = float(len(unique_clone_counter) / max(1, len(seqs)))\n features[\"total_templates\"] = float(total_weight)\n\n if len(lens) > 0:\n features[\"len_mean\"] = float(np.average(lens, weights=weights))\n features[\"len_std\"] = float(np.sqrt(np.average((lens - np.average(lens, weights=weights)) ** 2, weights=weights)))\n features[\"len_min\"] = float(np.min(lens))\n features[\"len_max\"] = float(np.max(lens))\n features[\"len_median\"] = weighted_quantile(lens, weights, 0.50)\n features[\"len_q10\"] = weighted_quantile(lens, weights, 0.10)\n features[\"len_q90\"] = weighted_quantile(lens, weights, 0.90)\n\n clone_weights = np.array(list(unique_clone_counter.values()), dtype=np.float32)\n clone_probs = clone_weights / clone_weights.sum()\n features[\"clone_top1_frac\"] = float(clone_probs.max())\n features[\"clone_top10_frac\"] = float(np.sort(clone_probs)[::-1][:10].sum())\n features[\"clone_shannon\"] = float(-(clone_probs * np.log(clone_probs + 1e-12)).sum())\n features[\"clone_simpson\"] = float(1.0 - np.sum(clone_probs ** 2))\n\n # -------------------------\n # AA composition + physicochemical\n # -------------------------\n aa_counter = Counter()\n total_aa_weight = 0.0\n\n hydro_vals, vol_vals, charge_vals = [], [], []\n hydro_w = []\n\n aromatic_num = 0.0\n small_num = 0.0\n hydrophobic_num = 0.0\n polar_num = 0.0\n cysteine_num = 0.0\n glycine_num = 0.0\n proline_num = 0.0\n\n for seq, w in zip(seqs, weights):\n valid_props = []\n for aa in seq:\n if aa in AA_PROPERTIES:\n aa_counter[aa] += w\n total_aa_weight += w\n valid_props.append(AA_PROPERTIES[aa])\n\n if aa in AROMATIC:\n aromatic_num += w\n if aa in SMALL:\n small_num += w\n if aa in HYDROPHOBIC:\n hydrophobic_num += w\n if aa in POLAR:\n polar_num += w\n if aa == \"C\":\n cysteine_num += w\n if aa == \"G\":\n glycine_num += w\n if aa == \"P\":\n proline_num += w\n\n if len(valid_props) > 0:\n hydro_vals.append(np.mean([p[\"hydro\"] for p in valid_props]))\n vol_vals.append(np.mean([p[\"vol\"] for p in valid_props]))\n charge_vals.append(np.mean([p[\"charge\"] for p in valid_props]))\n hydro_w.append(w)\n\n if total_aa_weight > 0:\n for aa in AA_LIST:\n features[f\"aa_frac_{aa}\"] = float(aa_counter.get(aa, 0.0) / total_aa_weight)\n features[\"aa_aromatic_frac\"] = float(aromatic_num / total_aa_weight)\n features[\"aa_small_frac\"] = float(small_num / total_aa_weight)\n features[\"aa_hydrophobic_frac\"] = float(hydrophobic_num / total_aa_weight)\n features[\"aa_polar_frac\"] = float(polar_num / total_aa_weight)\n features[\"aa_cysteine_frac\"] = float(cysteine_num / total_aa_weight)\n features[\"aa_glycine_frac\"] = float(glycine_num / total_aa_weight)\n features[\"aa_proline_frac\"] = float(proline_num / total_aa_weight)\n\n if len(hydro_vals) > 0:\n hydro_w = np.asarray(hydro_w, dtype=np.float32)\n features[\"phys_hydro_mean\"] = float(np.average(np.array(hydro_vals), weights=hydro_w))\n features[\"phys_vol_mean\"] = float(np.average(np.array(vol_vals), weights=hydro_w))\n features[\"phys_charge_mean\"] = float(np.average(np.array(charge_vals), weights=hydro_w))\n features[\"phys_charge_abs_mean\"] = float(np.average(np.abs(np.array(charge_vals)), weights=hydro_w))\n\n # -------------------------\n # K-mers\n # -------------------------\n for k in self.k_list:\n km_counter = Counter()\n total_k = 0.0\n\n for seq, w in zip(seqs, weights):\n if len(seq) < k:\n continue\n for i in range(len(seq) - k + 1):\n km = seq[i:i + k]\n if all(ch in AA_PROPERTIES for ch in km):\n km_counter[km] += w\n total_k += w\n\n if total_k > 0:\n keep_n = Config.TOP_KMERS_PER_FILE.get(k, None)\n items = km_counter.most_common(keep_n) if keep_n is not None else km_counter.items()\n for km, cnt in items:\n features[f\"kmer_{k}_{km}\"] = float(cnt / total_k)\n\n # -------------------------\n # Positional motifs\n # -------------------------\n pos_k = 3\n start_counter = Counter()\n end_counter = Counter()\n prefix1 = Counter()\n suffix1 = Counter()\n total_pos = 0.0\n\n for seq, w in zip(seqs, weights):\n if len(seq) >= pos_k:\n s0 = seq[:pos_k]\n s1 = seq[-pos_k:]\n if all(ch in AA_PROPERTIES for ch in s0):\n start_counter[s0] += w\n if all(ch in AA_PROPERTIES for ch in s1):\n end_counter[s1] += w\n total_pos += w\n\n if len(seq) >= 1:\n a0, a1 = seq[0], seq[-1]\n if a0 in AA_PROPERTIES:\n prefix1[a0] += w\n if a1 in AA_PROPERTIES:\n suffix1[a1] += w\n\n if total_pos > 0:\n for motif, cnt in start_counter.most_common(Config.TOP_POS_MOTIFS):\n features[f\"pos_start3_{motif}\"] = float(cnt / total_pos)\n for motif, cnt in end_counter.most_common(Config.TOP_POS_MOTIFS):\n features[f\"pos_end3_{motif}\"] = float(cnt / total_pos)\n for aa, cnt in prefix1.items():\n features[f\"pos_first_{aa}\"] = float(cnt / total_pos)\n for aa, cnt in suffix1.items():\n features[f\"pos_last_{aa}\"] = float(cnt / total_pos)\n\n # -------------------------\n # V/J family usage\n # -------------------------\n if \"v_call\" in df.columns:\n vf_counter = Counter()\n for fam, w in zip(df[\"v_call\"].fillna(\"\").astype(str).apply(self.gene_family), weights):\n vf_counter[fam] += float(w)\n for fam, cnt in vf_counter.most_common(25):\n features[f\"v_fam_{fam}\"] = float(cnt / total_weight)\n\n if \"j_call\" in df.columns:\n jf_counter = Counter()\n for fam, w in zip(df[\"j_call\"].fillna(\"\").astype(str).apply(self.gene_family), weights):\n jf_counter[fam] += float(w)\n for fam, cnt in jf_counter.most_common(15):\n features[f\"j_fam_{fam}\"] = float(cnt / total_weight)\n\n # -------------------------\n # Template-related stats\n # -------------------------\n tmpl = weights\n features[\"tmpl_mean\"] = float(np.mean(tmpl))\n features[\"tmpl_std\"] = float(np.std(tmpl))\n features[\"tmpl_median\"] = float(np.median(tmpl))\n features[\"tmpl_q90\"] = float(np.quantile(tmpl, 0.90))\n features[\"tmpl_top1_over_sum\"] = float(np.max(tmpl) / max(1.0, np.sum(tmpl)))\n features[\"tmpl_log_sum\"] = float(np.log1p(np.sum(tmpl)))\n p_t = tmpl / np.sum(tmpl)\n features[\"tmpl_entropy\"] = float(-(p_t * np.log(p_t + 1e-12)).sum())\n\n # -------------------------\n # Public clone features\n # -------------------------\n if pub_dict:\n hit_scores = []\n hit_weight = 0.0\n hit_count = 0\n\n for seq, cw in unique_clone_counter.items():\n if seq in pub_dict:\n sc = float(pub_dict[seq][\"score\"])\n hit_scores.append(sc * math.log1p(cw))\n hit_weight += cw\n hit_count += 1\n\n features[\"pub_hits\"] = float(hit_count)\n features[\"pub_hit_weight\"] = float(hit_weight / max(1.0, total_weight))\n features[\"pub_score_sum\"] = float(np.sum(hit_scores)) if len(hit_scores) else 0.0\n features[\"pub_score_max\"] = float(np.max(hit_scores)) if len(hit_scores) else 0.0\n\n # -------------------------\n # Metadata\n # -------------------------\n if meta_row is not None:\n for col in meta_row.index:\n if col in {\"filename\", \"repertoire_id\", \"label_positive\"}:\n continue\n\n val = meta_row[col]\n coln = str(col).strip().lower()\n\n if pd.isna(val):\n continue\n\n if isinstance(val, (int, float, np.number)):\n features[f\"meta_{coln}\"] = float(val)\n else:\n sval = str(val).strip()\n if not sval:\n continue\n\n # keep common low-cardinality metadata features\n if coln in {\"sex\", \"race\"}:\n key = re.sub(r\"[^a-z0-9]+\", \"_\", sval.lower())[:25]\n features[f\"meta_{coln}_{key}\"] = 1.0\n elif \"hla\" in coln:\n features[f\"meta_{coln}_present\"] = 1.0\n elif \"run\" in coln or \"batch\" in coln or \"center\" in coln:\n features[f\"meta_{coln}_hash\"] = stable_hash(sval)\n else:\n # generic fallback for a few small categorical fields\n if len(sval) <= 30:\n key = re.sub(r\"[^a-z0-9]+\", \"_\", sval.lower())[:25]\n features[f\"meta_{coln}_{key}\"] = 1.0\n\n # dataset-aware guardrails\n if ds_id == 7 and meta_row is not None and \"race\" in meta_row.index:\n features[\"meta_ds7_race_missing\"] = float(pd.isna(meta_row[\"race\"]))\n if ds_id == 8 and meta_row is not None:\n for hla in [\"A\", \"B\", \"C\", \"DRB1\"]:\n if hla in meta_row.index:\n features[f\"meta_hla_{hla}_present\"] = 1.0 if pd.notna(meta_row[hla]) else 0.0\n\n return features\n\n\n# ============================================================\n# PUBLIC CLONE MINING\n# ============================================================\ndef mine_public_clones(\n dataset_path: Path,\n max_files: int = 20,\n min_freq: float = 0.18,\n enrichment: float = 6.0,\n top_n: int = 2000,\n) -> Dict[str, Dict]:\n meta = pd.read_csv(dataset_path / \"metadata.csv\")\n y = pd.to_numeric(meta[\"label_positive\"], errors=\"coerce\").fillna(0).astype(int)\n\n pos_files = meta.loc[y == 1, \"filename\"].tolist()[:max_files]\n neg_files = meta.loc[y == 0, \"filename\"].tolist()[:max_files]\n\n if len(pos_files) == 0:\n return {}\n\n def get_unique_seq_counts(files):\n c = Counter()\n for fn in files:\n try:\n df = pd.read_csv(dataset_path / fn, sep=\"\\t\", usecols=[\"junction_aa\"])\n c.update(df[\"junction_aa\"].dropna().astype(str).unique().tolist())\n except Exception:\n pass\n return c\n\n pos_c = get_unique_seq_counts(pos_files)\n neg_c = get_unique_seq_counts(neg_files)\n\n n_pos = max(1, len(pos_files))\n n_neg = max(1, len(neg_files))\n\n scored = []\n for seq, count in pos_c.items():\n pf = count / n_pos\n nf = neg_c.get(seq, 0) / n_neg\n if pf >= min_freq and pf > nf * enrichment:\n score = float(np.log((pf + 1e-6) / (nf + 1e-6)))\n scored.append({\"seq\": seq, \"score\": score})\n\n scored.sort(key=lambda x: -x[\"score\"])\n return {x[\"seq\"]: x for x in scored[:top_n]}\n\n\n# ============================================================\n# PARALLEL FEATURE BUILDERS\n# ============================================================\ndef process_train_file(row, ds_path: Path, ds_id: int, pub_dict: Dict, extractor: FeatureExtractor):\n try:\n df = read_repertoire(ds_path / row[\"filename\"], Config.MAX_SEQUENCES_PER_FILE)\n feats = extractor.extract_all(df, pub_dict=pub_dict, meta_row=row, ds_id=ds_id)\n return {\n **feats,\n \"ID\": row.get(\"repertoire_id\", Path(row[\"filename\"]).stem),\n \"label_positive\": int(row[\"label_positive\"]),\n \"dataset\": ds_path.name,\n }\n except Exception:\n return None\n\n\ndef process_test_file(tsv_path: Path, ds_path: Path, ds_id: int, pub_dict: Dict, extractor: FeatureExtractor):\n try:\n df = read_repertoire(tsv_path, Config.MAX_SEQUENCES_PER_FILE)\n feats = extractor.extract_all(df, pub_dict=pub_dict, meta_row=None, ds_id=ds_id)\n return {\n **feats,\n \"ID\": tsv_path.stem,\n \"dataset\": ds_path.name,\n }\n except Exception:\n return None\n\n\n# ============================================================\n# FEATURE SELECTION\n# ============================================================\ndef select_features_xgb(X_df: pd.DataFrame, y: np.ndarray, top_features: int, gpu_ok: bool) -> Tuple[List[str], pd.DataFrame]:\n X_df = X_df.fillna(0.0)\n all_cols = X_df.columns.tolist()\n\n protected_prefixes = (\n \"len_\", \"clone_\", \"tmpl_\", \"phys_\", \"aa_\", \"meta_\", \"pub_\", \"v_fam_\", \"j_fam_\",\n \"seq_count\", \"unique_\", \"total_templates\", \"pos_first_\", \"pos_last_\"\n )\n protected = [c for c in all_cols if c.startswith(protected_prefixes)]\n\n dtrain = xgb.DMatrix(X_df, label=y, feature_names=all_cols)\n params = {\n \"objective\": \"binary:logistic\",\n \"eval_metric\": \"auc\",\n \"max_depth\": 4,\n \"learning_rate\": 0.08,\n \"subsample\": 0.9,\n \"colsample_bytree\": 0.8,\n \"tree_method\": \"hist\",\n \"device\": \"cuda\" if gpu_ok else \"cpu\",\n \"seed\": Config.RANDOM_STATE,\n \"verbosity\": 0,\n }\n bst = xgb.train(params, dtrain, num_boost_round=80, verbose_eval=False)\n\n scores = bst.get_score(importance_type=\"gain\")\n imp = pd.DataFrame({\n \"feature\": list(scores.keys()),\n \"gain\": list(scores.values()),\n }).sort_values(\"gain\", ascending=False)\n\n selected = list(dict.fromkeys(protected))\n for f in imp[\"feature\"].tolist():\n if f not in selected:\n selected.append(f)\n if len(selected) >= min(top_features, len(all_cols)):\n break\n\n if len(selected) < min(top_features, len(all_cols)):\n remaining = [c for c in all_cols if c not in selected]\n selected.extend(remaining[:min(top_features, len(all_cols)) - len(selected)])\n\n selected = [c for c in selected if c in X_df.columns]\n return selected, imp\n\n\n# ============================================================\n# MODEL TRAINING HELPERS\n# ============================================================\ndef train_xgb_fold(X_tr, y_tr, X_va, y_va, ds_id, gpu_ok):\n params = {\n \"objective\": \"binary:logistic\",\n \"eval_metric\": \"auc\",\n \"max_depth\": 6,\n \"learning_rate\": 0.03,\n \"subsample\": 0.85,\n \"colsample_bytree\": 0.85,\n \"min_child_weight\": 10,\n \"reg_alpha\": 0.20,\n \"reg_lambda\": 1.50,\n \"scale_pos_weight\": Config.SCALE_POS_WEIGHT.get(ds_id, 1.0),\n \"tree_method\": \"hist\",\n \"device\": \"cuda\" if gpu_ok else \"cpu\",\n \"seed\": Config.RANDOM_STATE,\n \"verbosity\": 0,\n }\n\n dtr = xgb.DMatrix(X_tr, label=y_tr)\n dva = xgb.DMatrix(X_va, label=y_va)\n\n bst = xgb.train(\n params,\n dtr,\n num_boost_round=1500,\n evals=[(dva, \"valid\")],\n early_stopping_rounds=Config.EARLY_STOP,\n verbose_eval=False,\n )\n pred = bst.predict(dva)\n best_iter = bst.best_iteration if bst.best_iteration is not None else 300\n return bst, pred, int(best_iter) + 30, params\n\n\ndef train_lgb_fold(X_tr, y_tr, X_va, y_va, ds_id, gpu_ok):\n params = {\n \"objective\": \"binary\",\n \"metric\": \"auc\",\n \"learning_rate\": 0.025,\n \"num_leaves\": 63,\n \"feature_fraction\": 0.85,\n \"bagging_fraction\": 0.85,\n \"bagging_freq\": 1,\n \"min_data_in_leaf\": 20,\n \"lambda_l1\": 0.10,\n \"lambda_l2\": 1.00,\n \"scale_pos_weight\": Config.SCALE_POS_WEIGHT.get(ds_id, 1.0),\n \"device_type\": \"gpu\" if gpu_ok else \"cpu\",\n \"seed\": Config.RANDOM_STATE,\n \"verbosity\": -1,\n }\n\n dtr = lgb.Dataset(X_tr, label=y_tr)\n dva = lgb.Dataset(X_va, label=y_va, reference=dtr)\n\n try:\n bst = lgb.train(\n params,\n dtr,\n num_boost_round=1500,\n valid_sets=[dva],\n callbacks=[lgb.early_stopping(Config.EARLY_STOP, verbose=False)],\n )\n except Exception:\n params[\"device_type\"] = \"cpu\"\n bst = lgb.train(\n params,\n dtr,\n num_boost_round=1500,\n valid_sets=[dva],\n callbacks=[lgb.early_stopping(Config.EARLY_STOP, verbose=False)],\n )\n\n pred = bst.predict(X_va)\n best_iter = bst.best_iteration if bst.best_iteration is not None else 400\n return bst, pred, int(best_iter) + 30, params\n\n\ndef train_cat_fold(X_tr, y_tr, X_va, y_va, gpu_ok):\n if not HAS_CATBOOST:\n return None, None, None, None\n\n params = dict(\n loss_function=\"Logloss\",\n eval_metric=\"AUC\",\n iterations=1800,\n depth=6,\n learning_rate=0.03,\n l2_leaf_reg=5.0,\n random_seed=Config.RANDOM_STATE,\n verbose=False,\n allow_writing_files=False,\n task_type=\"GPU\" if gpu_ok else \"CPU\",\n auto_class_weights=\"Balanced\",\n )\n\n try:\n model = CatBoostClassifier(**params)\n model.fit(X_tr, y_tr, eval_set=(X_va, y_va), use_best_model=True, verbose=False)\n except Exception:\n params[\"task_type\"] = \"CPU\"\n model = CatBoostClassifier(**params)\n model.fit(X_tr, y_tr, eval_set=(X_va, y_va), use_best_model=True, verbose=False)\n\n pred = model.predict_proba(X_va)[:, 1]\n best_iter = model.get_best_iteration()\n best_iter = 500 if best_iter is None or best_iter <= 0 else int(best_iter) + 30\n return model, pred, best_iter, params\n\n\ndef train_lr_fold(X_tr, y_tr, X_va, y_va):\n pipe = Pipeline([\n (\"scaler\", StandardScaler()),\n (\"lr\", LogisticRegression(\n C=0.8,\n max_iter=3000,\n solver=\"liblinear\",\n class_weight=\"balanced\",\n random_state=Config.RANDOM_STATE,\n )),\n ])\n pipe.fit(X_tr, y_tr)\n pred = pipe.predict_proba(X_va)[:, 1]\n return pipe, pred, None, None\n\n\n# ============================================================\n# MULTI-MODEL TRAINER\n# ============================================================\nclass MultiModelTrainer:\n def __init__(self, gpu_ok: bool):\n self.gpu_ok = gpu_ok\n self.models = {}\n self.feature_cols = []\n self.stack_weights = {}\n self.best_model = None\n self.feature_importance = None\n self.model_rounds = {}\n\n def fit(self, df: pd.DataFrame, ds_id: int, ds_name: str):\n y = df[\"label_positive\"].values.astype(np.int32)\n X_df = df.drop(columns=[\"ID\", \"dataset\", \"label_positive\"], errors=\"ignore\").fillna(0.0)\n\n self.feature_cols, imp = select_features_xgb(\n X_df, y, top_features=Config.TOP_FEATURES, gpu_ok=self.gpu_ok\n )\n self.feature_importance = imp\n\n X = X_df[self.feature_cols].fillna(0.0).astype(np.float32).values\n\n pos = int((y == 1).sum())\n neg = int((y == 0).sum())\n min_class = max(2, min(pos, neg))\n n_splits = min(Config.N_SPLITS, min_class)\n\n skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=Config.RANDOM_STATE)\n\n available_models = [\"xgb\", \"lgb\"]\n if HAS_CATBOOST:\n available_models.append(\"cat\")\n available_models.append(\"lr\")\n\n oof = {m: np.zeros(len(y), dtype=np.float32) for m in available_models}\n best_rounds = defaultdict(list)\n fold_records = []\n\n print(f\" Selected features: {len(self.feature_cols)}\")\n\n for fold, (tr_idx, va_idx) in enumerate(skf.split(X, y), 1):\n X_tr, X_va = X[tr_idx], X[va_idx]\n y_tr, y_va = y[tr_idx], y[va_idx]\n\n # XGB\n bst, pred, rounds, params = train_xgb_fold(X_tr, y_tr, X_va, y_va, ds_id, self.gpu_ok)\n oof[\"xgb\"][va_idx] = pred\n auc = safe_auc(y_va, pred)\n best_rounds[\"xgb\"].append(rounds)\n fold_records.append({\"dataset\": ds_name, \"fold\": fold, \"model\": \"XGB\", \"auc\": auc})\n\n # LGB\n bst, pred, rounds, params = train_lgb_fold(X_tr, y_tr, X_va, y_va, ds_id, self.gpu_ok)\n oof[\"lgb\"][va_idx] = pred\n auc = safe_auc(y_va, pred)\n best_rounds[\"lgb\"].append(rounds)\n fold_records.append({\"dataset\": ds_name, \"fold\": fold, \"model\": \"LGB\", \"auc\": auc})\n\n # CAT\n if HAS_CATBOOST:\n model, pred, rounds, params = train_cat_fold(X_tr, y_tr, X_va, y_va, self.gpu_ok)\n oof[\"cat\"][va_idx] = pred\n auc = safe_auc(y_va, pred)\n best_rounds[\"cat\"].append(rounds)\n fold_records.append({\"dataset\": ds_name, \"fold\": fold, \"model\": \"CAT\", \"auc\": auc})\n\n # LR\n model, pred, _, _ = train_lr_fold(X_tr, y_tr, X_va, y_va)\n oof[\"lr\"][va_idx] = pred\n auc = safe_auc(y_va, pred)\n fold_records.append({\"dataset\": ds_name, \"fold\": fold, \"model\": \"LR\", \"auc\": auc})\n\n # Stack weights from OOF\n oof_cols = []\n model_names_for_stack = []\n for m in available_models:\n if m in oof:\n oof_cols.append(oof[m])\n model_names_for_stack.append(m)\n\n oof_mat = np.column_stack(oof_cols)\n meta = LogisticRegression(max_iter=2000, random_state=Config.RANDOM_STATE)\n meta.fit(oof_mat, y)\n\n coef = np.clip(meta.coef_[0], 0, None)\n if coef.sum() <= 0:\n coef = np.ones_like(coef) / len(coef)\n else:\n coef = coef / coef.sum()\n\n self.stack_weights = {m: float(w) for m, w in zip(model_names_for_stack, coef)}\n stack_pred = np.zeros(len(y), dtype=np.float32)\n for i, m in enumerate(model_names_for_stack):\n stack_pred += coef[i] * oof[m]\n\n stack_auc = safe_auc(y, stack_pred)\n\n # Summary table\n fold_df = pd.DataFrame(fold_records)\n summary = (\n fold_df.groupby([\"dataset\", \"model\"], as_index=False)[\"auc\"]\n .agg(mean_auc=\"mean\", std_auc=\"std\", max_auc=\"max\")\n .sort_values([\"dataset\", \"mean_auc\"], ascending=[True, False])\n )\n summary = pd.concat(\n [\n summary,\n pd.DataFrame([{\n \"dataset\": ds_name,\n \"model\": \"STACK\",\n \"mean_auc\": stack_auc,\n \"std_auc\": np.nan,\n \"max_auc\": stack_auc,\n }]),\n ],\n ignore_index=True,\n ).sort_values([\"dataset\", \"mean_auc\"], ascending=[True, False])\n\n self.best_model = summary.loc[summary[\"dataset\"] == ds_name].sort_values(\"mean_auc\", ascending=False).iloc[0][\"model\"]\n\n print(\"\\n CV Summary\")\n print(summary.loc[summary[\"dataset\"] == ds_name, [\"model\", \"mean_auc\", \"std_auc\", \"max_auc\"]].to_string(index=False))\n print(f\" Stack weights: {self.stack_weights}\")\n print(f\" Best by OOF/CV: {self.best_model}\")\n\n # Fit full-data models\n X_full = X\n y_full = y\n\n # XGB full\n xgb_rounds = int(np.mean(best_rounds[\"xgb\"])) if len(best_rounds[\"xgb\"]) else 400\n xgb_params = {\n \"objective\": \"binary:logistic\",\n \"eval_metric\": \"auc\",\n \"max_depth\": 6,\n \"learning_rate\": 0.03,\n \"subsample\": 0.85,\n \"colsample_bytree\": 0.85,\n \"min_child_weight\": 10,\n \"reg_alpha\": 0.20,\n \"reg_lambda\": 1.50,\n \"scale_pos_weight\": Config.SCALE_POS_WEIGHT.get(ds_id, 1.0),\n \"tree_method\": \"hist\",\n \"device\": \"cuda\" if self.gpu_ok else \"cpu\",\n \"seed\": Config.RANDOM_STATE,\n \"verbosity\": 0,\n }\n self.models[\"xgb\"] = xgb.train(\n xgb_params, xgb.DMatrix(X_full, label=y_full), num_boost_round=max(120, xgb_rounds)\n )\n self.model_rounds[\"xgb\"] = max(120, xgb_rounds)\n\n # LGB full\n lgb_rounds = int(np.mean(best_rounds[\"lgb\"])) if len(best_rounds[\"lgb\"]) else 500\n lgb_params = {\n \"objective\": \"binary\",\n \"metric\": \"auc\",\n \"learning_rate\": 0.025,\n \"num_leaves\": 63,\n \"feature_fraction\": 0.85,\n \"bagging_fraction\": 0.85,\n \"bagging_freq\": 1,\n \"min_data_in_leaf\": 20,\n \"lambda_l1\": 0.10,\n \"lambda_l2\": 1.00,\n \"scale_pos_weight\": Config.SCALE_POS_WEIGHT.get(ds_id, 1.0),\n \"device_type\": \"gpu\" if self.gpu_ok else \"cpu\",\n \"seed\": Config.RANDOM_STATE,\n \"verbosity\": -1,\n }\n try:\n self.models[\"lgb\"] = lgb.train(\n lgb_params, lgb.Dataset(X_full, label=y_full), num_boost_round=max(150, lgb_rounds)\n )\n except Exception:\n lgb_params[\"device_type\"] = \"cpu\"\n self.models[\"lgb\"] = lgb.train(\n lgb_params, lgb.Dataset(X_full, label=y_full), num_boost_round=max(150, lgb_rounds)\n )\n self.model_rounds[\"lgb\"] = max(150, lgb_rounds)\n\n # CAT full\n if HAS_CATBOOST and \"cat\" in available_models:\n cat_rounds = int(np.mean(best_rounds[\"cat\"])) if len(best_rounds[\"cat\"]) else 700\n cat_params = dict(\n loss_function=\"Logloss\",\n eval_metric=\"AUC\",\n iterations=max(250, cat_rounds),\n depth=6,\n learning_rate=0.03,\n l2_leaf_reg=5.0,\n random_seed=Config.RANDOM_STATE,\n verbose=False,\n allow_writing_files=False,\n task_type=\"GPU\" if self.gpu_ok else \"CPU\",\n auto_class_weights=\"Balanced\",\n )\n try:\n self.models[\"cat\"] = CatBoostClassifier(**cat_params)\n self.models[\"cat\"].fit(X_full, y_full, verbose=False)\n except Exception:\n cat_params[\"task_type\"] = \"CPU\"\n self.models[\"cat\"] = CatBoostClassifier(**cat_params)\n self.models[\"cat\"].fit(X_full, y_full, verbose=False)\n self.model_rounds[\"cat\"] = max(250, cat_rounds)\n\n # LR full\n self.models[\"lr\"] = Pipeline([\n (\"scaler\", StandardScaler()),\n (\"lr\", LogisticRegression(\n C=0.8,\n max_iter=3000,\n solver=\"liblinear\",\n class_weight=\"balanced\",\n random_state=Config.RANDOM_STATE,\n )),\n ])\n self.models[\"lr\"].fit(X_full, y_full)\n\n return {\n \"fold_df\": fold_df,\n \"summary_df\": summary,\n \"selected_features\": self.feature_cols,\n \"stack_weights\": self.stack_weights,\n \"best_model\": self.best_model,\n }\n\n def predict(self, X_df: pd.DataFrame) -> Tuple[np.ndarray, pd.DataFrame]:\n X = X_df.reindex(columns=self.feature_cols, fill_value=0.0).fillna(0.0).astype(np.float32).values\n\n per_model = {}\n if \"xgb\" in self.models:\n per_model[\"xgb\"] = self.models[\"xgb\"].predict(xgb.DMatrix(X))\n if \"lgb\" in self.models:\n per_model[\"lgb\"] = self.models[\"lgb\"].predict(X)\n if \"cat\" in self.models:\n per_model[\"cat\"] = self.models[\"cat\"].predict_proba(X)[:, 1]\n if \"lr\" in self.models:\n per_model[\"lr\"] = self.models[\"lr\"].predict_proba(X)[:, 1]\n\n final = np.zeros(len(X), dtype=np.float32)\n for m, p in per_model.items():\n w = self.stack_weights.get(m, 0.0)\n final += w * p\n\n per_model_df = pd.DataFrame(per_model)\n per_model_df[\"stack\"] = final\n return final, per_model_df\n\n\n# ============================================================\n# REPORTING / PLOTS\n# ============================================================\ndef save_eda_row(df: pd.DataFrame, ds_name: str) -> Dict:\n y = df[\"label_positive\"].astype(int).values\n X = df.drop(columns=[\"ID\", \"dataset\", \"label_positive\"], errors=\"ignore\")\n nonzero = (X.fillna(0.0) != 0).sum(axis=1).mean() if len(X) else 0\n return {\n \"dataset\": ds_name,\n \"n_samples\": int(len(df)),\n \"n_pos\": int((y == 1).sum()),\n \"n_neg\": int((y == 0).sum()),\n \"pos_rate\": float((y == 1).mean()),\n \"n_features_before_select\": int(X.shape[1]),\n \"avg_nonzero_features\": float(nonzero),\n }\n\n\ndef save_model_plot(summary_df: pd.DataFrame, ds_name: str):\n sdf = summary_df[summary_df[\"dataset\"] == ds_name].copy()\n sdf = sdf.sort_values(\"mean_auc\", ascending=False)\n\n plt.figure(figsize=(8, 4.5))\n yerr = sdf[\"std_auc\"].fillna(0).values\n plt.bar(sdf[\"model\"].astype(str), sdf[\"mean_auc\"].values, yerr=yerr)\n plt.ylabel(\"AUC\")\n plt.xlabel(\"Model\")\n plt.title(f\"Model Comparison - {ds_name}\")\n plt.ylim(max(0.0, sdf[\"mean_auc\"].min() - 0.05), min(1.0, sdf[\"mean_auc\"].max() + 0.05))\n plt.tight_layout()\n plt.savefig(Config.OUT_DIR / f\"model_compare_{ds_name}.png\", dpi=180, bbox_inches=\"tight\")\n plt.close()\n\n\ndef save_global_plots(eda_df: pd.DataFrame, cv_summary_df: pd.DataFrame):\n if len(eda_df):\n plt.figure(figsize=(8, 4.5))\n plt.bar(eda_df[\"dataset\"].astype(str), eda_df[\"pos_rate\"].values)\n plt.ylabel(\"Positive Rate\")\n plt.xlabel(\"Dataset\")\n plt.title(\"Class Balance by Dataset\")\n plt.xticks(rotation=45, ha=\"right\")\n plt.tight_layout()\n plt.savefig(Config.OUT_DIR / \"eda_pos_rate_by_dataset.png\", dpi=180, bbox_inches=\"tight\")\n plt.close()\n\n if len(cv_summary_df):\n g = cv_summary_df.groupby(\"model\", as_index=False)[\"mean_auc\"].mean().sort_values(\"mean_auc\", ascending=False)\n plt.figure(figsize=(8, 4.5))\n plt.bar(g[\"model\"].astype(str), g[\"mean_auc\"].values)\n plt.ylabel(\"Mean AUC Across Datasets\")\n plt.xlabel(\"Model\")\n plt.title(\"Global Model Comparison\")\n plt.tight_layout()\n plt.savefig(Config.OUT_DIR / \"global_model_comparison.png\", dpi=180, bbox_inches=\"tight\")\n plt.close()\n\n\n# ============================================================\n# SUBMISSION\n# ============================================================\ndef create_submission_from_final(\n final_pred_df: pd.DataFrame,\n sample_path: str | Path,\n output_path: str = \"/kaggle/working/best_submission.csv\",\n) -> pd.DataFrame:\n sample = pd.read_csv(sample_path)\n\n pred_map = (\n final_pred_df.drop_duplicates(subset=[\"dataset\", \"ID\"])\n .set_index([\"dataset\", \"ID\"])[\"label_positive_probability\"]\n )\n\n test_mask = sample[\"dataset\"].astype(str).str.startswith(\"test_dataset_\")\n idx = pd.MultiIndex.from_frame(sample.loc[test_mask, [\"dataset\", \"ID\"]])\n\n new_vals = pred_map.reindex(idx).to_numpy()\n old_vals = sample.loc[test_mask, \"label_positive_probability\"].to_numpy()\n\n sample.loc[test_mask, \"label_positive_probability\"] = np.where(\n pd.isna(new_vals), old_vals, new_vals\n )\n sample.to_csv(output_path, index=False)\n return sample\n\n\n# ============================================================\n# MAIN\n# ============================================================\ndef main():\n seed_everything(Config.RANDOM_STATE)\n t0 = time.time()\n\n print(\"=\" * 80)\n print(\"AIRR-ML-25 Advanced Multi-Model GPU Pipeline\")\n print(\"=\" * 80)\n\n gpu_ok = check_gpu()\n print(f\"GPU detected: {gpu_ok} | GPU mem: {get_gpu_memory()} | CatBoost: {HAS_CATBOOST}\")\n\n train_sets = sorted([d.name for d in Config.TRAIN_DIR.glob(\"train_dataset_*\")])\n test_sets = sorted([d.name for d in Config.TEST_DIR.glob(\"test_dataset_*\")])\n\n extractor = FeatureExtractor(Config.K_LIST)\n\n bundles = {}\n all_eda_rows = []\n all_fold_dfs = []\n all_summary_dfs = []\n\n # ------------------------------------------------------\n # TRAIN\n # ------------------------------------------------------\n for ds_name in train_sets:\n ds_id = dataset_id_from_name(ds_name)\n ds_path = Config.TRAIN_DIR / ds_name\n\n print(f\"\\n{'-' * 80}\")\n print(f\"Training {ds_name} (id={ds_id})\")\n print(f\"{'-' * 80}\")\n\n pub_dict = mine_public_clones(\n ds_path,\n max_files=Config.PUB_MAX_FILES,\n min_freq=Config.PUB_MIN_FREQ,\n enrichment=Config.PUB_ENRICH,\n top_n=Config.PUB_TOP_N.get(ds_id, Config.PUB_TOP_N[\"default\"]),\n )\n print(f\" Public clones mined: {len(pub_dict)}\")\n\n meta = pd.read_csv(ds_path / \"metadata.csv\")\n meta[\"label_positive\"] = pd.to_numeric(meta[\"label_positive\"], errors=\"coerce\").fillna(0).astype(int)\n\n print(\" Extracting features...\")\n res = Parallel(n_jobs=Config.N_JOBS, backend=\"loky\")(\n delayed(process_train_file)(row, ds_path, ds_id, pub_dict, extractor)\n for _, row in tqdm(meta.iterrows(), total=len(meta), leave=False)\n )\n train_df = pd.DataFrame([r for r in res if r is not None]).fillna(0.0)\n\n eda_row = save_eda_row(train_df, ds_name)\n all_eda_rows.append(eda_row)\n\n trainer = MultiModelTrainer(gpu_ok=gpu_ok)\n report = trainer.fit(train_df, ds_id=ds_id, ds_name=ds_name)\n\n # Save selected features\n pd.DataFrame({\"feature\": report[\"selected_features\"]}).to_csv(\n Config.OUT_DIR / f\"selected_features_{ds_name}.csv\", index=False\n )\n\n # Save top importances\n if trainer.feature_importance is not None and len(trainer.feature_importance):\n trainer.feature_importance.head(100).to_csv(\n Config.OUT_DIR / f\"top_features_{ds_name}.csv\", index=False\n )\n\n # Save tables / plots\n save_model_plot(report[\"summary_df\"], ds_name)\n\n all_fold_dfs.append(report[\"fold_df\"])\n all_summary_dfs.append(report[\"summary_df\"])\n\n bundles[ds_name] = {\n \"trainer\": trainer,\n \"pub_dict\": pub_dict,\n \"feature_cols\": report[\"selected_features\"],\n \"stack_weights\": report[\"stack_weights\"],\n \"best_model\": report[\"best_model\"],\n }\n\n print(f\" Stored bundle for {ds_name} | best model = {report['best_model']}\")\n\n del meta, res, train_df\n gc.collect()\n\n # ------------------------------------------------------\n # SAVE GLOBAL REPORTS\n # ------------------------------------------------------\n eda_df = pd.DataFrame(all_eda_rows)\n eda_df.to_csv(Config.OUT_DIR / \"eda_summary.csv\", index=False)\n\n cv_fold_df = pd.concat(all_fold_dfs, ignore_index=True) if len(all_fold_dfs) else pd.DataFrame()\n cv_summary_df = pd.concat(all_summary_dfs, ignore_index=True) if len(all_summary_dfs) else pd.DataFrame()\n\n if len(cv_fold_df):\n cv_fold_df.to_csv(Config.OUT_DIR / \"cv_fold_results.csv\", index=False)\n if len(cv_summary_df):\n cv_summary_df.to_csv(Config.OUT_DIR / \"cv_summary.csv\", index=False)\n\n pivot = cv_summary_df.pivot_table(index=\"dataset\", columns=\"model\", values=\"mean_auc\", aggfunc=\"mean\")\n pivot.to_csv(Config.OUT_DIR / \"cv_pivot_mean_auc.csv\")\n\n save_global_plots(eda_df, cv_summary_df)\n\n # ------------------------------------------------------\n # PREDICT TEST\n # ------------------------------------------------------\n print(f\"\\n{'=' * 80}\")\n print(\"PREDICTING TEST SETS\")\n print(f\"{'=' * 80}\")\n\n all_preds = []\n default_train = train_sets[0] if len(train_sets) else None\n\n for test_name in test_sets:\n ds_id = dataset_id_from_name(test_name)\n train_key = f\"train_dataset_{ds_id}\"\n if train_key not in bundles:\n train_key = default_train\n\n if train_key is None:\n raise RuntimeError(\"No training bundle found.\")\n\n print(f\"\\n {test_name} -> using {train_key}\")\n test_path = Config.TEST_DIR / test_name\n files = sorted(test_path.glob(\"*.tsv\"))\n\n bundle = bundles[train_key]\n trainer = bundle[\"trainer\"]\n pub_dict = bundle[\"pub_dict\"]\n\n res = Parallel(n_jobs=Config.N_JOBS, backend=\"loky\")(\n delayed(process_test_file)(fp, test_path, ds_id, pub_dict, extractor)\n for fp in tqdm(files, total=len(files), leave=False)\n )\n test_df = pd.DataFrame([r for r in res if r is not None]).fillna(0.0)\n\n feature_df = test_df.drop(columns=[\"ID\", \"dataset\"], errors=\"ignore\")\n pred, per_model_df = trainer.predict(feature_df)\n\n out = test_df[[\"ID\", \"dataset\"]].copy()\n out[\"label_positive_probability\"] = pred.astype(float)\n all_preds.append(out)\n\n per_model_save = test_df[[\"ID\", \"dataset\"]].copy()\n per_model_save = pd.concat([per_model_save, per_model_df.reset_index(drop=True)], axis=1)\n per_model_save.to_csv(Config.OUT_DIR / f\"test_model_preds_{test_name}.csv\", index=False)\n\n del res, test_df, feature_df, per_model_df\n gc.collect()\n\n # ------------------------------------------------------\n # SUBMISSION\n # ------------------------------------------------------\n final_pred_df = pd.concat(all_preds, ignore_index=True) if len(all_preds) else pd.DataFrame(\n columns=[\"ID\", \"dataset\", \"label_positive_probability\"]\n )\n\n create_submission_from_final(\n final_pred_df=final_pred_df,\n sample_path=Config.SAMPLE_SUBMISSION,\n output_path=\"/kaggle/working/best_submission.csv\",\n )\n\n print(f\"\\nDone. Submission saved -> /kaggle/working/best_submission.csv\")\n print(f\"Artifacts saved -> {Config.OUT_DIR}\")\n print(f\"Total runtime: {(time.time() - t0) / 60:.2f} min\")\n\n\nif __name__ == \"__main__\":\n main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-19T21:02:19.613891Z","iopub.execute_input":"2026-03-19T21:02:19.614255Z","iopub.status.idle":"2026-03-19T23:13:25.496720Z","shell.execute_reply.started":"2026-03-19T21:02:19.614229Z","shell.execute_reply":"2026-03-19T23:13:25.495201Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 1 — project structure create করো\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\ndirs = [\n \"configs\",\n \"src\",\n \"artifacts/phase1\",\n]\n\nfor d in dirs:\n (PROJECT_ROOT / d).mkdir(parents=True, exist_ok=True)\n\n# package init\n(PROJECT_ROOT / \"src\" / \"__init__.py\").write_text(\"\", encoding=\"utf-8\")\n\n# placeholder files for later phases\nplaceholder_files = [\n \"src/branch_a_features.py\",\n \"src/branch_a_models.py\",\n \"src/branch_b_public.py\",\n \"src/branch_b_cluster.py\",\n \"src/branch_c_embeddings.py\",\n \"src/branch_d_mil.py\",\n \"src/attribution.py\",\n \"src/fusion.py\",\n \"src/evaluation.py\",\n \"src/simulation.py\",\n \"train_branch_a.py\",\n \"train_branch_b.py\",\n \"train_branch_c.py\",\n \"train_branch_d.py\",\n \"train_fusion.py\",\n \"make_submission.py\",\n]\n\nfor rel in placeholder_files:\n path = PROJECT_ROOT / rel\n path.parent.mkdir(parents=True, exist_ok=True)\n if not path.exists():\n path.write_text(\"# placeholder for later phases\\n\", encoding=\"utf-8\")\n\nprint(\"Project root:\", PROJECT_ROOT)\nprint(\"Structure created.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 2 — config files write করো\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nbase_yaml = \"\"\"\nproject_root: /kaggle/working/project\n\npaths:\n data_root: /kaggle/input/competitions/adaptive-immune-profiling-challenge-2025\n train_root: /kaggle/input/competitions/adaptive-immune-profiling-challenge-2025/train_datasets/train_datasets\n test_root: /kaggle/input/competitions/adaptive-immune-profiling-challenge-2025/test_datasets/test_datasets\n sample_submission: /kaggle/input/competitions/adaptive-immune-profiling-challenge-2025/sample_submissions.csv\n output_root: /kaggle/working/project/artifacts/phase1\n\nruntime:\n random_state: 42\n n_jobs: 4\n\nphase1:\n n_splits: 5\n max_rows_profile: 20000\n\"\"\"\n\nbranch_a_yaml = \"enabled: false\\n\"\nbranch_b_yaml = \"enabled: false\\n\"\nbranch_c_yaml = \"enabled: false\\n\"\nbranch_d_yaml = \"enabled: false\\n\"\nfusion_yaml = \"enabled: false\\n\"\n\n(PROJECT_ROOT / \"configs\" / \"base.yaml\").write_text(base_yaml.strip() + \"\\n\", encoding=\"utf-8\")\n(PROJECT_ROOT / \"configs\" / \"branch_a.yaml\").write_text(branch_a_yaml, encoding=\"utf-8\")\n(PROJECT_ROOT / \"configs\" / \"branch_b.yaml\").write_text(branch_b_yaml, encoding=\"utf-8\")\n(PROJECT_ROOT / \"configs\" / \"branch_c.yaml\").write_text(branch_c_yaml, encoding=\"utf-8\")\n(PROJECT_ROOT / \"configs\" / \"branch_d.yaml\").write_text(branch_d_yaml, encoding=\"utf-8\")\n(PROJECT_ROOT / \"configs\" / \"fusion.yaml\").write_text(fusion_yaml, encoding=\"utf-8\")\n\nprint(\"Config files written.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 3 — Phase 1 module files write করো\nfrom pathlib import Path\nfrom textwrap import dedent\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nfiles = {}\n\nfiles[\"src/utils.py\"] = dedent(\"\"\"\nfrom __future__ import annotations\n\nimport os\nimport re\nimport random\nimport hashlib\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\n\n\ndef seed_everything(seed: int = 42):\n random.seed(seed)\n np.random.seed(seed)\n os.environ[\"PYTHONHASHSEED\"] = str(seed)\n\n\ndef ensure_dir(path):\n path = Path(path)\n path.mkdir(parents=True, exist_ok=True)\n return path\n\n\ndef normalize_colname(name: str) -> str:\n return re.sub(r\"[^a-z0-9]+\", \"_\", str(name).strip().lower()).strip(\"_\")\n\n\ndef dataset_id_from_name(name: str) -> int:\n s = str(name)\n m = re.search(r\"(?:train|test)_dataset_(\\\\d+)\", s)\n if m:\n return int(m.group(1))\n nums = re.findall(r\"\\\\d+\", s)\n return int(nums[0]) if nums else -1\n\n\ndef stable_hash(x, mod: int = 1000003) -> int:\n if pd.isna(x):\n x = \"NA\"\n h = hashlib.md5(str(x).encode(\"utf-8\")).hexdigest()\n return int(h[:12], 16) % mod\n\n\ndef candidate_lookup(columns):\n return {normalize_colname(c): c for c in columns}\n\n\ndef first_existing(columns, candidates):\n lookup = candidate_lookup(columns)\n for cand in candidates:\n key = normalize_colname(cand)\n if key in lookup:\n return lookup[key]\n return None\n\n\ndef clean_string_series(series, default_value=pd.NA):\n if series is None:\n return None\n s = series.astype(\"string\").str.strip()\n s = s.replace({\"\": pd.NA, \"nan\": pd.NA, \"None\": pd.NA, \"NA\": pd.NA})\n return s.fillna(default_value)\n\n\ndef coerce_binary_label(series):\n if series is None:\n return None\n if pd.api.types.is_numeric_dtype(series):\n return pd.to_numeric(series, errors=\"coerce\")\n\n mapping = {\n \"true\": 1,\n \"false\": 0,\n \"positive\": 1,\n \"negative\": 0,\n \"yes\": 1,\n \"no\": 0,\n \"1\": 1,\n \"0\": 0,\n }\n s = series.astype(\"string\").str.strip().str.lower()\n return s.map(mapping)\n\n\ndef safe_numeric_series(series):\n if series is None:\n return None\n return pd.to_numeric(series, errors=\"coerce\")\n\"\"\")\n\nfiles[\"src/data_io.py\"] = dedent(\"\"\"\nfrom __future__ import annotations\n\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\n\nfrom .utils import dataset_id_from_name\n\nAA_SET = set(list(\"ARNDCQEGHILKMFPSTWYV\"))\n\n\ndef list_repertoire_files(ds_path: Path):\n if not ds_path.exists():\n return []\n return sorted([p for p in ds_path.glob(\"*.tsv\") if p.is_file()])\n\n\ndef read_metadata_raw(ds_path: Path) -> pd.DataFrame:\n meta_path = ds_path / \"metadata.csv\"\n if meta_path.exists():\n return pd.read_csv(meta_path)\n files = list_repertoire_files(ds_path)\n return pd.DataFrame({\"filename\": [f.name for f in files]})\n\n\ndef quick_profile_repertoire(tsv_path: Path, max_rows: int = 20000) -> dict:\n default = {\n \"n_rows_loaded\": 0.0,\n \"unique_seq_count\": 0.0,\n \"unique_ratio\": 0.0,\n \"seq_len_mean\": 0.0,\n \"seq_len_std\": 0.0,\n \"seq_len_q10\": 0.0,\n \"seq_len_q90\": 0.0,\n \"templates_sum\": 0.0,\n \"templates_mean\": 0.0,\n \"templates_std\": 0.0,\n \"top1_template_frac\": 0.0,\n \"v_unique_count\": 0.0,\n \"j_unique_count\": 0.0,\n \"valid_aa_ratio\": 0.0,\n \"file_size_bytes\": float(tsv_path.stat().st_size) if tsv_path.exists() else 0.0,\n }\n\n if not tsv_path.exists():\n return default\n\n cols = [\"junction_aa\", \"templates\", \"v_call\", \"j_call\"]\n try:\n header = pd.read_csv(tsv_path, sep=\"\\\\t\", nrows=0)\n usecols = [c for c in cols if c in header.columns]\n if len(usecols) == 0:\n return default\n df = pd.read_csv(tsv_path, sep=\"\\\\t\", usecols=usecols, nrows=max_rows)\n except Exception:\n return default\n\n if \"junction_aa\" not in df.columns:\n return default\n\n df[\"junction_aa\"] = df[\"junction_aa\"].fillna(\"\").astype(str)\n df = df[df[\"junction_aa\"] != \"\"].copy()\n\n if len(df) == 0:\n default[\"n_rows_loaded\"] = 0.0\n return default\n\n if \"templates\" not in df.columns:\n df[\"templates\"] = 1.0\n df[\"templates\"] = pd.to_numeric(df[\"templates\"], errors=\"coerce\").fillna(1.0).clip(lower=0.0)\n\n seqs = df[\"junction_aa\"]\n lens = seqs.str.len().astype(float)\n templates = df[\"templates\"].astype(float)\n\n total_chars = float(lens.sum())\n valid_chars = 0.0\n for s in seqs.tolist():\n valid_chars += sum(ch in AA_SET for ch in s)\n\n out = dict(default)\n out[\"n_rows_loaded\"] = float(len(df))\n out[\"unique_seq_count\"] = float(seqs.nunique())\n out[\"unique_ratio\"] = float(seqs.nunique() / max(1, len(df)))\n out[\"seq_len_mean\"] = float(lens.mean())\n out[\"seq_len_std\"] = float(lens.std(ddof=0))\n out[\"seq_len_q10\"] = float(lens.quantile(0.10))\n out[\"seq_len_q90\"] = float(lens.quantile(0.90))\n out[\"templates_sum\"] = float(templates.sum())\n out[\"templates_mean\"] = float(templates.mean())\n out[\"templates_std\"] = float(templates.std(ddof=0))\n out[\"top1_template_frac\"] = float(templates.max() / max(1e-9, templates.sum()))\n out[\"v_unique_count\"] = float(df[\"v_call\"].fillna(\"\").astype(str).nunique()) if \"v_call\" in df.columns else 0.0\n out[\"j_unique_count\"] = float(df[\"j_call\"].fillna(\"\").astype(str).nunique()) if \"j_call\" in df.columns else 0.0\n out[\"valid_aa_ratio\"] = float(valid_chars / max(1.0, total_chars))\n out[\"file_size_bytes\"] = float(tsv_path.stat().st_size)\n\n return out\n\n\ndef discover_dataset_dirs(train_root: Path, test_root: Path):\n train_dirs = sorted([p for p in train_root.glob(\"train_dataset_*\") if p.is_dir()])\n test_dirs = sorted([p for p in test_root.glob(\"test_dataset_*\") if p.is_dir()])\n return train_dirs, test_dirs\n\"\"\")\n\nfiles[\"src/metadata.py\"] = dedent(\"\"\"\nfrom __future__ import annotations\n\nfrom pathlib import Path\n\nimport pandas as pd\n\nfrom .data_io import discover_dataset_dirs, list_repertoire_files, read_metadata_raw\nfrom .utils import (\n clean_string_series,\n coerce_binary_label,\n dataset_id_from_name,\n first_existing,\n safe_numeric_series,\n stable_hash,\n)\n\n\nclass MetadataBuilder:\n def __init__(self):\n self.required_columns = [\n \"dataset_name\",\n \"dataset_id\",\n \"source\",\n \"repertoire_id\",\n \"filename\",\n \"absolute_path\",\n \"label_positive\",\n \"subject_id\",\n \"sample_id\",\n \"sequencing_run_id\",\n \"batch_id\",\n \"center_id\",\n \"locus\",\n \"chain\",\n \"receptor_type\",\n \"sex\",\n \"race\",\n \"age\",\n \"hla_A\",\n \"hla_B\",\n \"hla_C\",\n \"hla_DRB1\",\n \"split_group_key\",\n \"processing_hash\",\n ]\n\n def _pick_series(self, df: pd.DataFrame, candidates, default=pd.NA):\n col = first_existing(df.columns, candidates)\n if col is None:\n return pd.Series([default] * len(df), index=df.index)\n return df[col]\n\n def build_dataset_table(self, ds_path: Path, source: str | None = None) -> pd.DataFrame:\n source = source or (\"train\" if \"train_datasets\" in str(ds_path) else \"test\")\n\n raw = read_metadata_raw(ds_path).copy()\n files = list_repertoire_files(ds_path)\n\n file_df = pd.DataFrame({\"filename\": [f.name for f in files]})\n file_df[\"absolute_path\"] = [str(f) for f in files]\n\n if len(raw) == 0:\n raw = file_df.copy()\n\n filename_col = first_existing(raw.columns, [\"filename\", \"file_name\", \"file\"])\n if filename_col is None:\n raw[\"filename\"] = [Path(x).name for x in raw.index.astype(str)]\n else:\n raw[\"filename\"] = raw[filename_col].astype(str).map(lambda x: Path(x).name)\n\n if len(file_df) > 0:\n meta = file_df.merge(raw, on=\"filename\", how=\"left\")\n else:\n meta = raw.copy()\n meta[\"absolute_path\"] = meta[\"filename\"].astype(str).map(lambda x: str(ds_path / Path(x).name))\n\n meta[\"dataset_name\"] = ds_path.name\n meta[\"dataset_id\"] = dataset_id_from_name(ds_path.name)\n meta[\"source\"] = source\n\n repertoire_series = self._pick_series(\n meta,\n [\"repertoire_id\", \"sample_id\", \"specimen_id\", \"id\", \"sample\"],\n default=pd.NA,\n )\n repertoire_series = repertoire_series.astype(\"string\")\n meta[\"repertoire_id\"] = repertoire_series.fillna(\n meta[\"filename\"].astype(str).map(lambda x: Path(x).stem)\n )\n\n label_series = self._pick_series(meta, [\"label_positive\", \"label\", \"y\"], default=pd.NA)\n label_series = coerce_binary_label(label_series)\n if source == \"train\":\n meta[\"label_positive\"] = label_series\n else:\n meta[\"label_positive\"] = pd.NA\n\n subject_id = self._pick_series(\n meta,\n [\"subject_id\", \"subject\", \"participant_id\", \"donor_id\", \"patient_id\", \"individual_id\"],\n default=pd.NA,\n )\n sample_id = self._pick_series(\n meta,\n [\"sample_id\", \"sample\", \"specimen_id\"],\n default=pd.NA,\n )\n sequencing_run_id = self._pick_series(\n meta,\n [\"sequencing_run_id\", \"run_id\", \"run\", \"sequencing_run\", \"seq_run_id\"],\n default=pd.NA,\n )\n batch_id = self._pick_series(\n meta,\n [\"batch_id\", \"batch\", \"library_batch\", \"processing_batch\"],\n default=pd.NA,\n )\n center_id = self._pick_series(\n meta,\n [\"center_id\", \"center\", \"site\", \"lab\", \"institution\"],\n default=pd.NA,\n )\n locus = self._pick_series(meta, [\"locus\", \"gene_locus\"], default=\"UNK\")\n chain = self._pick_series(meta, [\"chain\", \"chain_type\"], default=\"UNK\")\n receptor_type = self._pick_series(meta, [\"receptor_type\", \"receptor\", \"airr_type\"], default=\"UNK\")\n sex = self._pick_series(meta, [\"sex\", \"gender\"], default=pd.NA)\n race = self._pick_series(meta, [\"race\", \"ethnicity\"], default=pd.NA)\n age = self._pick_series(meta, [\"age\", \"age_years\"], default=pd.NA)\n\n hla_A = self._pick_series(meta, [\"hla_a\", \"hla-a\", \"a\"], default=pd.NA)\n hla_B = self._pick_series(meta, [\"hla_b\", \"hla-b\", \"b\"], default=pd.NA)\n hla_C = self._pick_series(meta, [\"hla_c\", \"hla-c\", \"c\"], default=pd.NA)\n hla_DRB1 = self._pick_series(meta, [\"hla_drb1\", \"hla-drb1\", \"drb1\"], default=pd.NA)\n\n meta[\"subject_id\"] = clean_string_series(subject_id).fillna(meta[\"repertoire_id\"])\n meta[\"sample_id\"] = clean_string_series(sample_id).fillna(meta[\"repertoire_id\"])\n meta[\"sequencing_run_id\"] = clean_string_series(sequencing_run_id)\n meta[\"batch_id\"] = clean_string_series(batch_id)\n meta[\"center_id\"] = clean_string_series(center_id)\n meta[\"locus\"] = clean_string_series(locus, default_value=\"UNK\").fillna(\"UNK\")\n meta[\"chain\"] = clean_string_series(chain, default_value=\"UNK\").fillna(\"UNK\")\n meta[\"receptor_type\"] = clean_string_series(receptor_type, default_value=\"UNK\").fillna(\"UNK\")\n meta[\"sex\"] = clean_string_series(sex)\n meta[\"race\"] = clean_string_series(race)\n meta[\"age\"] = safe_numeric_series(age)\n meta[\"hla_A\"] = clean_string_series(hla_A)\n meta[\"hla_B\"] = clean_string_series(hla_B)\n meta[\"hla_C\"] = clean_string_series(hla_C)\n meta[\"hla_DRB1\"] = clean_string_series(hla_DRB1)\n\n meta[\"split_group_key\"] = meta[\"sequencing_run_id\"].fillna(meta[\"subject_id\"]).astype(\"string\")\n\n meta[\"processing_hash\"] = meta.apply(\n lambda row: stable_hash(\n f\"{row['sequencing_run_id']}|{row['batch_id']}|{row['center_id']}|{row['dataset_name']}\"\n ),\n axis=1,\n )\n\n out = meta[self.required_columns].copy()\n out[\"label_positive\"] = pd.to_numeric(out[\"label_positive\"], errors=\"coerce\")\n return out\n\n def build_all(self, train_root: Path, test_root: Path) -> pd.DataFrame:\n train_dirs, test_dirs = discover_dataset_dirs(train_root, test_root)\n all_tables = []\n\n for ds in train_dirs:\n all_tables.append(self.build_dataset_table(ds, source=\"train\"))\n\n for ds in test_dirs:\n all_tables.append(self.build_dataset_table(ds, source=\"test\"))\n\n if len(all_tables) == 0:\n return pd.DataFrame(columns=self.required_columns)\n\n canonical = pd.concat(all_tables, ignore_index=True)\n canonical = canonical.sort_values([\"source\", \"dataset_name\", \"repertoire_id\"]).reset_index(drop=True)\n return canonical\n\"\"\")\n\nfiles[\"src/splits.py\"] = dedent(\"\"\"\nfrom __future__ import annotations\n\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\n\ntry:\n from sklearn.model_selection import StratifiedGroupKFold\n HAS_SGKF = True\nexcept Exception:\n HAS_SGKF = False\n\nfrom .utils import ensure_dir\n\n\ndef _assign_dataset_folds(df: pd.DataFrame, n_splits: int = 5, random_state: int = 42) -> pd.DataFrame:\n out = df.copy().reset_index(drop=True)\n out[\"fold\"] = -1\n out[\"splitter\"] = \"NA\"\n out[\"n_splits_used\"] = 1\n\n if len(out) == 0:\n return out\n\n y = pd.to_numeric(out[\"label_positive\"], errors=\"coerce\").fillna(0).astype(int).values\n class_counts = pd.Series(y).value_counts()\n\n if len(class_counts) < 2:\n out[\"fold\"] = 0\n out[\"splitter\"] = \"single_class\"\n out[\"n_splits_used\"] = 1\n return out\n\n min_class = int(class_counts.min())\n n_splits_used = min(n_splits, min_class)\n\n if n_splits_used < 2:\n out[\"fold\"] = 0\n out[\"splitter\"] = \"not_enough_samples\"\n out[\"n_splits_used\"] = 1\n return out\n\n groups = out[\"split_group_key\"].astype(\"string\").fillna(out[\"subject_id\"].astype(\"string\")).values\n use_group = HAS_SGKF and (pd.Series(groups).nunique() >= n_splits_used)\n\n if use_group:\n splitter = StratifiedGroupKFold(n_splits=n_splits_used, shuffle=True, random_state=random_state)\n split_iter = splitter.split(np.zeros(len(out)), y, groups=groups)\n splitter_name = \"StratifiedGroupKFold\"\n else:\n splitter = StratifiedKFold(n_splits=n_splits_used, shuffle=True, random_state=random_state)\n split_iter = splitter.split(np.zeros(len(out)), y)\n splitter_name = \"StratifiedKFold\"\n\n for fold, (_, va_idx) in enumerate(split_iter):\n out.loc[va_idx, \"fold\"] = fold\n\n out[\"splitter\"] = splitter_name\n out[\"n_splits_used\"] = n_splits_used\n return out\n\n\ndef build_folds_for_all(canonical_meta_df: pd.DataFrame, out_dir: Path, n_splits: int = 5, random_state: int = 42):\n out_dir = ensure_dir(out_dir)\n train_df = canonical_meta_df[canonical_meta_df[\"source\"] == \"train\"].copy()\n train_df = train_df[pd.notna(train_df[\"label_positive\"])].copy()\n\n all_fold_tables = []\n\n for ds_name, g in train_df.groupby(\"dataset_name\", sort=True):\n fold_df = _assign_dataset_folds(g, n_splits=n_splits, random_state=random_state)\n fold_path = out_dir / f\"folds_{ds_name}.csv\"\n fold_df.to_csv(fold_path, index=False)\n all_fold_tables.append(fold_df)\n\n if len(all_fold_tables) == 0:\n return pd.DataFrame()\n\n all_folds = pd.concat(all_fold_tables, ignore_index=True)\n all_folds.to_csv(out_dir / \"all_folds.csv\", index=False)\n return all_folds\n\"\"\")\n\nfiles[\"src/drift.py\"] = dedent(\"\"\"\nfrom __future__ import annotations\n\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nfrom joblib import Parallel, delayed\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold\n\ntry:\n from xgboost import XGBClassifier\n HAS_XGB = True\nexcept Exception:\n HAS_XGB = False\n\ntry:\n from lightgbm import LGBMClassifier\n HAS_LGB = True\nexcept Exception:\n HAS_LGB = False\n\nfrom .data_io import quick_profile_repertoire\nfrom .utils import ensure_dir\n\n\nclass DriftAnalyzer:\n def __init__(self, max_rows: int = 20000, n_jobs: int = 4, random_state: int = 42):\n self.max_rows = max_rows\n self.n_jobs = n_jobs\n self.random_state = random_state\n\n def _profile_one(self, row_dict: dict):\n feats = quick_profile_repertoire(Path(row_dict[\"absolute_path\"]), max_rows=self.max_rows)\n return {\n \"dataset_name\": row_dict[\"dataset_name\"],\n \"dataset_id\": row_dict[\"dataset_id\"],\n \"repertoire_id\": row_dict[\"repertoire_id\"],\n **feats,\n }\n\n def build_profile_table(self, meta_df: pd.DataFrame) -> pd.DataFrame:\n base = meta_df[\n [\"dataset_name\", \"dataset_id\", \"repertoire_id\", \"source\", \"absolute_path\"]\n ].drop_duplicates().reset_index(drop=True)\n\n rows = base.to_dict(orient=\"records\")\n prof = Parallel(n_jobs=self.n_jobs, backend=\"loky\")(\n delayed(self._profile_one)(r) for r in rows\n )\n prof_df = pd.DataFrame(prof)\n\n merged = base.merge(\n prof_df,\n on=[\"dataset_name\", \"dataset_id\", \"repertoire_id\"],\n how=\"left\",\n )\n return merged\n\n def _make_model(self):\n if HAS_XGB:\n return XGBClassifier(\n n_estimators=300,\n max_depth=4,\n learning_rate=0.05,\n subsample=0.9,\n colsample_bytree=0.8,\n objective=\"binary:logistic\",\n eval_metric=\"auc\",\n tree_method=\"hist\",\n random_state=self.random_state,\n n_jobs=1,\n )\n if HAS_LGB:\n return LGBMClassifier(\n n_estimators=300,\n max_depth=-1,\n learning_rate=0.05,\n num_leaves=31,\n subsample=0.9,\n colsample_bytree=0.8,\n random_state=self.random_state,\n )\n return LogisticRegression(max_iter=2000, random_state=self.random_state)\n\n def _extract_importance(self, model, feature_cols):\n if hasattr(model, \"feature_importances_\"):\n imp = np.asarray(model.feature_importances_, dtype=float)\n return pd.DataFrame({\"feature\": feature_cols, \"importance\": imp}).sort_values(\n \"importance\", ascending=False\n )\n if hasattr(model, \"coef_\"):\n coef = np.abs(np.asarray(model.coef_).ravel())\n return pd.DataFrame({\"feature\": feature_cols, \"importance\": coef}).sort_values(\n \"importance\", ascending=False\n )\n return pd.DataFrame({\"feature\": feature_cols, \"importance\": np.zeros(len(feature_cols))})\n\n def analyze_pair(self, train_meta: pd.DataFrame, test_meta: pd.DataFrame, out_dir: Path):\n pair_meta = pd.concat([train_meta, test_meta], ignore_index=True)\n profile = self.build_profile_table(pair_meta)\n profile[\"drift_target\"] = (profile[\"source\"] == \"test\").astype(int)\n\n feature_cols = [\n c for c in profile.columns\n if c not in {\"dataset_name\", \"dataset_id\", \"repertoire_id\", \"source\", \"absolute_path\", \"drift_target\"}\n ]\n X_df = profile[feature_cols].apply(pd.to_numeric, errors=\"coerce\").fillna(0.0)\n y = profile[\"drift_target\"].astype(int).values\n\n class_counts = pd.Series(y).value_counts()\n if len(class_counts) < 2:\n auc_mean = np.nan\n auc_std = np.nan\n model = self._make_model()\n model.fit(X_df, y)\n imp_df = self._extract_importance(model, feature_cols)\n return auc_mean, auc_std, imp_df, profile\n\n min_class = int(class_counts.min())\n n_splits = min(5, min_class)\n\n if n_splits < 2:\n auc_mean = np.nan\n auc_std = np.nan\n model = self._make_model()\n model.fit(X_df, y)\n imp_df = self._extract_importance(model, feature_cols)\n return auc_mean, auc_std, imp_df, profile\n\n skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=self.random_state)\n scores = []\n\n for tr_idx, va_idx in skf.split(X_df, y):\n model = self._make_model()\n model.fit(X_df.iloc[tr_idx], y[tr_idx])\n\n if hasattr(model, \"predict_proba\"):\n pred = model.predict_proba(X_df.iloc[va_idx])[:, 1]\n else:\n pred = model.decision_function(X_df.iloc[va_idx])\n\n auc = roc_auc_score(y[va_idx], pred)\n scores.append(float(auc))\n\n final_model = self._make_model()\n final_model.fit(X_df, y)\n imp_df = self._extract_importance(final_model, feature_cols)\n\n return float(np.mean(scores)), float(np.std(scores)), imp_df, profile\n\n def run(self, canonical_meta_df: pd.DataFrame, out_dir: Path):\n out_dir = ensure_dir(out_dir)\n\n train_df = canonical_meta_df[canonical_meta_df[\"source\"] == \"train\"].copy()\n test_df = canonical_meta_df[canonical_meta_df[\"source\"] == \"test\"].copy()\n\n common_ids = sorted(set(train_df[\"dataset_id\"].unique()) & set(test_df[\"dataset_id\"].unique()))\n summary_rows = []\n\n for ds_id in common_ids:\n tr = train_df[train_df[\"dataset_id\"] == ds_id].copy()\n te = test_df[test_df[\"dataset_id\"] == ds_id].copy()\n\n if len(tr) == 0 or len(te) == 0:\n continue\n\n auc_mean, auc_std, imp_df, profile_df = self.analyze_pair(tr, te, out_dir=out_dir)\n\n train_names = \",\".join(sorted(tr[\"dataset_name\"].astype(str).unique()))\n test_names = \",\".join(sorted(te[\"dataset_name\"].astype(str).unique()))\n\n imp_path = out_dir / f\"drift_feature_importance_dataset_{ds_id}.csv\"\n imp_df.to_csv(imp_path, index=False)\n\n profile_path = out_dir / f\"drift_profile_dataset_{ds_id}.csv\"\n profile_df.to_csv(profile_path, index=False)\n\n summary_rows.append(\n {\n \"dataset_id\": ds_id,\n \"train_dataset_names\": train_names,\n \"test_dataset_names\": test_names,\n \"n_train\": int(len(tr)),\n \"n_test\": int(len(te)),\n \"drift_auc_mean\": auc_mean,\n \"drift_auc_std\": auc_std,\n \"importance_file\": str(imp_path),\n }\n )\n\n summary_df = pd.DataFrame(summary_rows)\n summary_df.to_csv(out_dir / \"drift_auc_by_dataset.csv\", index=False)\n return summary_df\n\"\"\")\n\nfiles[\"run_phase1.py\"] = dedent(\"\"\"\nfrom __future__ import annotations\n\nimport sys\nfrom pathlib import Path\n\nimport yaml\n\nPROJECT_ROOT = Path(__file__).resolve().parent\nif str(PROJECT_ROOT) not in sys.path:\n sys.path.insert(0, str(PROJECT_ROOT))\n\nfrom src.utils import ensure_dir, seed_everything\nfrom src.metadata import MetadataBuilder\nfrom src.splits import build_folds_for_all\nfrom src.drift import DriftAnalyzer\n\n\ndef main():\n cfg = yaml.safe_load((PROJECT_ROOT / \"configs\" / \"base.yaml\").read_text())\n\n paths = cfg[\"paths\"]\n runtime = cfg[\"runtime\"]\n phase1 = cfg[\"phase1\"]\n\n train_root = Path(paths[\"train_root\"])\n test_root = Path(paths[\"test_root\"])\n out_root = ensure_dir(paths[\"output_root\"])\n\n seed_everything(runtime[\"random_state\"])\n\n print(\"=\" * 80)\n print(\"PHASE 1: Standards-first data layer\")\n print(\"=\" * 80)\n print(\"Train root:\", train_root)\n print(\"Test root :\", test_root)\n print(\"Output :\", out_root)\n\n builder = MetadataBuilder()\n canonical_meta = builder.build_all(train_root=train_root, test_root=test_root)\n canonical_path = out_root / \"canonical_metadata.csv\"\n canonical_meta.to_csv(canonical_path, index=False)\n print(f\"Saved canonical metadata -> {canonical_path}\")\n print(\"Canonical shape:\", canonical_meta.shape)\n\n folds_df = build_folds_for_all(\n canonical_meta_df=canonical_meta,\n out_dir=out_root,\n n_splits=phase1[\"n_splits\"],\n random_state=runtime[\"random_state\"],\n )\n print(\"Saved fold files.\")\n if len(folds_df) > 0:\n print(\"Fold table shape:\", folds_df.shape)\n\n analyzer = DriftAnalyzer(\n max_rows=phase1[\"max_rows_profile\"],\n n_jobs=runtime[\"n_jobs\"],\n random_state=runtime[\"random_state\"],\n )\n drift_df = analyzer.run(canonical_meta_df=canonical_meta, out_dir=out_root)\n print(\"Saved drift outputs.\")\n if len(drift_df) > 0:\n print(drift_df.to_string(index=False))\n\n print(\"=\" * 80)\n print(\"PHASE 1 DONE\")\n print(\"=\" * 80)\n\n\nif __name__ == \"__main__\":\n main()\n\"\"\")\n\nfor rel_path, content in files.items():\n path = PROJECT_ROOT / rel_path\n path.parent.mkdir(parents=True, exist_ok=True)\n path.write_text(content.strip() + \"\\\\n\", encoding=\"utf-8\")\n\nprint(\"Phase 1 module files written.\")\nprint(\"Files created:\")\nfor rel_path in sorted(files.keys()):\n print(\"-\", rel_path)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 4 — imports verify করো\nimport sys\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\nif str(PROJECT_ROOT) not in sys.path:\n sys.path.insert(0, str(PROJECT_ROOT))\n\nfrom src.metadata import MetadataBuilder\nfrom src.splits import build_folds_for_all\nfrom src.drift import DriftAnalyzer\n\nprint(\"Imports OK\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 5 — Phase 1 run করো\n!python /kaggle/working/project/run_phase1.py","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 6 — outputs inspect করো\nfrom pathlib import Path\nimport pandas as pd\n\nOUT_DIR = Path(\"/kaggle/working/project/artifacts/phase1\")\n\nprint(\"Output files:\")\nfor p in sorted(OUT_DIR.glob(\"*\")):\n print(\"-\", p.name)\n\nprint(\"\\ncanonical_metadata.csv\")\ndisplay(pd.read_csv(OUT_DIR / \"canonical_metadata.csv\").head())\n\nprint(\"\\ndrift_auc_by_dataset.csv\")\ndrift_path = OUT_DIR / \"drift_auc_by_dataset.csv\"\nif drift_path.exists():\n display(pd.read_csv(drift_path))\nelse:\n print(\"drift_auc_by_dataset.csv not found\")\n\nprint(\"\\nall_folds.csv\")\nall_folds_path = OUT_DIR / \"all_folds.csv\"\nif all_folds_path.exists():\n display(pd.read_csv(all_folds_path).head())\nelse:\n print(\"all_folds.csv not found\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#phase 2 :\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 1 — branch_a.yaml write করো\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nbranch_a_yaml = \"\"\"\nenabled: true\n\npaths:\n output_root: /kaggle/working/project/artifacts/phase2_branch_a\n\nfeatures:\n k_list: [3, 4]\n top_kmers_per_k:\n 3: 300\n 4: 450\n top_pos_motifs: 20\n max_sequences_per_file: 50000\n\npublic_clone:\n max_files: 20\n min_freq: 0.18\n enrichment: 6.0\n top_n_default: 2500\n\ntraining:\n seeds: [42, 52, 62]\n top_features: 650\n\"\"\"\n\n(PROJECT_ROOT / \"configs\" / \"branch_a.yaml\").write_text(branch_a_yaml.strip() + \"\\n\", encoding=\"utf-8\")\nprint(\"Written:\", PROJECT_ROOT / \"configs\" / \"branch_a.yaml\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 2 — src/branch_a_features.py, src/branch_a_models.py, train_branch_a.py write করো\nfrom pathlib import Path\nfrom textwrap import dedent\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nbranch_a_features_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nimport math\nfrom collections import Counter\nfrom pathlib import Path\nfrom typing import Dict, Optional, Tuple\n\nimport numpy as np\nimport pandas as pd\n\nAA_PROPERTIES = {\n \"A\": {\"hydro\": 1.8, \"vol\": 88.6, \"charge\": 0.0, \"polar\": 0},\n \"R\": {\"hydro\": -4.5, \"vol\": 173.4, \"charge\": 1.0, \"polar\": 1},\n \"N\": {\"hydro\": -3.5, \"vol\": 114.1, \"charge\": 0.0, \"polar\": 1},\n \"D\": {\"hydro\": -3.5, \"vol\": 111.1, \"charge\": -1.0, \"polar\": 1},\n \"C\": {\"hydro\": 2.5, \"vol\": 108.5, \"charge\": 0.0, \"polar\": 0},\n \"Q\": {\"hydro\": -3.5, \"vol\": 143.8, \"charge\": 0.0, \"polar\": 1},\n \"E\": {\"hydro\": -3.5, \"vol\": 138.4, \"charge\": -1.0, \"polar\": 1},\n \"G\": {\"hydro\": -0.4, \"vol\": 60.1, \"charge\": 0.0, \"polar\": 0},\n \"H\": {\"hydro\": -3.2, \"vol\": 153.2, \"charge\": 0.5, \"polar\": 1},\n \"I\": {\"hydro\": 4.5, \"vol\": 166.7, \"charge\": 0.0, \"polar\": 0},\n \"L\": {\"hydro\": 3.8, \"vol\": 166.7, \"charge\": 0.0, \"polar\": 0},\n \"K\": {\"hydro\": -3.9, \"vol\": 168.6, \"charge\": 1.0, \"polar\": 1},\n \"M\": {\"hydro\": 1.9, \"vol\": 162.9, \"charge\": 0.0, \"polar\": 0},\n \"F\": {\"hydro\": 2.8, \"vol\": 189.9, \"charge\": 0.0, \"polar\": 0},\n \"P\": {\"hydro\": -1.6, \"vol\": 112.7, \"charge\": 0.0, \"polar\": 0},\n \"S\": {\"hydro\": -0.8, \"vol\": 89.0, \"charge\": 0.0, \"polar\": 1},\n \"T\": {\"hydro\": -0.7, \"vol\": 116.1, \"charge\": 0.0, \"polar\": 1},\n \"W\": {\"hydro\": -0.9, \"vol\": 227.8, \"charge\": 0.0, \"polar\": 0},\n \"Y\": {\"hydro\": -1.3, \"vol\": 193.6, \"charge\": 0.0, \"polar\": 1},\n \"V\": {\"hydro\": 4.2, \"vol\": 140.0, \"charge\": 0.0, \"polar\": 0},\n}\nAA_LIST = sorted(list(AA_PROPERTIES.keys()))\nAROMATIC = set([\"F\", \"W\", \"Y\"])\nSMALL = set([\"A\", \"G\", \"S\", \"T\", \"P\"])\nHYDROPHOBIC = set([\"A\", \"V\", \"I\", \"L\", \"M\", \"F\", \"W\", \"Y\", \"C\"])\nPOLAR = set([aa for aa, p in AA_PROPERTIES.items() if p[\"polar\"] == 1])\n\n\ndef weighted_quantile(values, weights, q):\n values = np.asarray(values, dtype=float)\n weights = np.asarray(weights, dtype=float)\n if len(values) == 0:\n return np.nan\n sorter = np.argsort(values)\n values = values[sorter]\n weights = weights[sorter]\n total = np.sum(weights)\n if total <= 0:\n return float(np.quantile(values, q))\n cdf = np.cumsum(weights) / total\n return float(np.interp(q, cdf, values))\n\n\ndef read_repertoire(tsv_path: Path, max_seqs: Optional[int] = None, random_state: int = 42) -> pd.DataFrame:\n cols = [\"junction_aa\", \"v_call\", \"j_call\", \"templates\"]\n try:\n header = pd.read_csv(tsv_path, sep=\"\\\\t\", nrows=0)\n usecols = [c for c in cols if c in header.columns]\n df = pd.read_csv(tsv_path, sep=\"\\\\t\", usecols=usecols)\n except Exception:\n return pd.DataFrame(columns=cols)\n\n for c in cols:\n if c not in df.columns:\n df[c] = \"\" if c != \"templates\" else 1.0\n\n df[\"junction_aa\"] = df[\"junction_aa\"].fillna(\"\").astype(str)\n df[\"templates\"] = pd.to_numeric(df[\"templates\"], errors=\"coerce\").fillna(1.0).clip(lower=1.0)\n\n if max_seqs and len(df) > max_seqs:\n rng = np.random.RandomState(random_state)\n weights = df[\"templates\"].values.astype(float)\n s = weights.sum()\n if s <= 0:\n idx = rng.choice(len(df), size=max_seqs, replace=False)\n else:\n weights = weights / s\n idx = rng.choice(len(df), size=max_seqs, replace=False, p=weights)\n df = df.iloc[idx].reset_index(drop=True)\n\n return df\n\n\ndef sequence_signature(seq: str) -> str:\n if not seq:\n return \"UNK\"\n return f\"{len(seq)}|{seq[:3]}|{seq[-3:]}\"\n\n\ndef counter_from_df(df: pd.DataFrame) -> Tuple[Counter, Counter]:\n seq_counter = Counter()\n sig_counter = Counter()\n if df is None or len(df) == 0:\n return seq_counter, sig_counter\n\n seqs = df[\"junction_aa\"].fillna(\"\").astype(str).tolist()\n tmpls = pd.to_numeric(df[\"templates\"], errors=\"coerce\").fillna(1.0).clip(lower=1.0).tolist()\n\n for s, w in zip(seqs, tmpls):\n if not s:\n continue\n seq_counter[s] += float(w)\n sig_counter[sequence_signature(s)] += float(w)\n return seq_counter, sig_counter\n\n\ndef build_public_clone_bundle(\n fold_train_meta: pd.DataFrame,\n dataset_path: Path,\n max_files: int = 20,\n min_freq: float = 0.18,\n enrichment: float = 6.0,\n top_n: int = 2000,\n max_seqs_per_file: int = 60000,\n random_state: int = 42,\n) -> Dict[str, Dict]:\n meta = fold_train_meta.copy()\n y = pd.to_numeric(meta[\"label_positive\"], errors=\"coerce\").fillna(0).astype(int)\n\n pos_rows = meta.loc[y == 1].head(max_files)\n neg_rows = meta.loc[y == 0].head(max_files)\n\n def collect(rows):\n exact_counts = Counter()\n approx_counts = Counter()\n for _, row in rows.iterrows():\n fn = row[\"filename\"]\n df = read_repertoire(dataset_path / fn, max_seqs=max_seqs_per_file, random_state=random_state)\n seq_counter, sig_counter = counter_from_df(df)\n exact_counts.update(seq_counter.keys()) # unique presence per repertoire\n approx_counts.update(sig_counter.keys())\n return exact_counts, approx_counts\n\n pos_exact, pos_approx = collect(pos_rows)\n neg_exact, neg_approx = collect(neg_rows)\n\n n_pos = max(1, len(pos_rows))\n n_neg = max(1, len(neg_rows))\n\n exact_scored = []\n for seq, count in pos_exact.items():\n pf = count / n_pos\n nf = neg_exact.get(seq, 0) / n_neg\n if pf >= min_freq and pf > nf * enrichment:\n score = float(np.log((pf + 1e-6) / (nf + 1e-6)))\n exact_scored.append({\"key\": seq, \"score\": score})\n\n approx_scored = []\n for sig, count in pos_approx.items():\n pf = count / n_pos\n nf = neg_approx.get(sig, 0) / n_neg\n if pf >= min_freq and pf > nf * enrichment:\n score = float(np.log((pf + 1e-6) / (nf + 1e-6)))\n approx_scored.append({\"key\": sig, \"score\": score})\n\n exact_scored = sorted(exact_scored, key=lambda x: -x[\"score\"])[:top_n]\n approx_scored = sorted(approx_scored, key=lambda x: -x[\"score\"])[:top_n]\n\n return {\n \"exact\": {x[\"key\"]: x for x in exact_scored},\n \"approx\": {x[\"key\"]: x for x in approx_scored},\n }\n\n\nclass FeatureExtractor:\n def __init__(self, k_list=None, top_kmers_per_k=None, top_pos_motifs: int = 25):\n self.k_list = k_list or [3, 4]\n self.top_kmers_per_k = top_kmers_per_k or {3: 400, 4: 600}\n self.top_pos_motifs = top_pos_motifs\n\n @staticmethod\n def gene_family(gene_call: str) -> str:\n if not isinstance(gene_call, str) or not gene_call:\n return \"UNK\"\n return gene_call.split(\"*\")[0].split(\"-\")[0].upper() or \"UNK\"\n\n def _clone_summary(self, unique_clone_counter: Counter, total_weight: float, features: Dict[str, float]):\n clone_weights = np.array(list(unique_clone_counter.values()), dtype=np.float32)\n if len(clone_weights) == 0:\n return\n\n clone_probs = clone_weights / max(1e-9, clone_weights.sum())\n features[\"clone_top1_frac\"] = float(clone_probs.max())\n features[\"clone_top10_frac\"] = float(np.sort(clone_probs)[::-1][:10].sum())\n features[\"clone_shannon\"] = float(-(clone_probs * np.log(clone_probs + 1e-12)).sum())\n features[\"clone_simpson\"] = float(1.0 - np.sum(clone_probs ** 2))\n\n sorted_w = np.sort(clone_weights)\n n = len(sorted_w)\n gini = (2 * np.sum((np.arange(1, n + 1) * sorted_w)) / (n * np.sum(sorted_w))) - ((n + 1) / n)\n features[\"clone_gini\"] = float(gini)\n features[\"hill_q0\"] = float(len(clone_weights))\n features[\"hill_q1\"] = float(np.exp(features[\"clone_shannon\"]))\n features[\"hill_q2\"] = float(1.0 / max(1e-9, np.sum(clone_probs ** 2)))\n\n rare_thr = np.quantile(clone_weights, 0.10)\n features[\"rare_clone_burden\"] = float(clone_weights[clone_weights <= rare_thr].sum() / max(1e-9, total_weight))\n features[\"singleton_like_burden\"] = float(np.sum(clone_weights <= 1.0) / max(1, len(clone_weights)))\n features[\"clone_count_log\"] = float(np.log1p(len(clone_weights)))\n\n def _aa_physchem(self, seqs, weights, features):\n aa_counter = Counter()\n total_aa_weight = 0.0\n\n hydro_vals, vol_vals, charge_vals, hydro_w = [], [], [], []\n aromatic_num = small_num = hydrophobic_num = polar_num = cysteine_num = glycine_num = proline_num = 0.0\n\n for seq, w in zip(seqs, weights):\n valid_props = []\n for aa in seq:\n if aa in AA_PROPERTIES:\n aa_counter[aa] += w\n total_aa_weight += w\n valid_props.append(AA_PROPERTIES[aa])\n if aa in AROMATIC:\n aromatic_num += w\n if aa in SMALL:\n small_num += w\n if aa in HYDROPHOBIC:\n hydrophobic_num += w\n if aa in POLAR:\n polar_num += w\n if aa == \"C\":\n cysteine_num += w\n if aa == \"G\":\n glycine_num += w\n if aa == \"P\":\n proline_num += w\n\n if len(valid_props) > 0:\n hydro_vals.append(np.mean([p[\"hydro\"] for p in valid_props]))\n vol_vals.append(np.mean([p[\"vol\"] for p in valid_props]))\n charge_vals.append(np.mean([p[\"charge\"] for p in valid_props]))\n hydro_w.append(w)\n\n if total_aa_weight > 0:\n for aa in AA_LIST:\n features[f\"aa_frac_{aa}\"] = float(aa_counter.get(aa, 0.0) / total_aa_weight)\n features[\"aa_aromatic_frac\"] = float(aromatic_num / total_aa_weight)\n features[\"aa_small_frac\"] = float(small_num / total_aa_weight)\n features[\"aa_hydrophobic_frac\"] = float(hydrophobic_num / total_aa_weight)\n features[\"aa_polar_frac\"] = float(polar_num / total_aa_weight)\n features[\"aa_cysteine_frac\"] = float(cysteine_num / total_aa_weight)\n features[\"aa_glycine_frac\"] = float(glycine_num / total_aa_weight)\n features[\"aa_proline_frac\"] = float(proline_num / total_aa_weight)\n\n if len(hydro_vals) > 0:\n hydro_w = np.asarray(hydro_w, dtype=np.float32)\n features[\"phys_hydro_mean\"] = float(np.average(np.array(hydro_vals), weights=hydro_w))\n features[\"phys_vol_mean\"] = float(np.average(np.array(vol_vals), weights=hydro_w))\n features[\"phys_charge_mean\"] = float(np.average(np.array(charge_vals), weights=hydro_w))\n features[\"phys_charge_abs_mean\"] = float(np.average(np.abs(np.array(charge_vals)), weights=hydro_w))\n\n def _kmer_features(self, seqs, weights, features):\n for k in self.k_list:\n km_counter = Counter()\n total_k = 0.0\n for seq, w in zip(seqs, weights):\n if len(seq) < k:\n continue\n for i in range(len(seq) - k + 1):\n km = seq[i:i + k]\n if all(ch in AA_PROPERTIES for ch in km):\n km_counter[km] += w\n total_k += w\n if total_k > 0:\n keep_n = self.top_kmers_per_k.get(k, None)\n items = km_counter.most_common(keep_n) if keep_n is not None else km_counter.items()\n for km, cnt in items:\n features[f\"kmer_{k}_{km}\"] = float(cnt / total_k)\n\n def _positional_features(self, seqs, weights, features):\n pos_k = 3\n start_counter, end_counter, prefix1, suffix1 = Counter(), Counter(), Counter(), Counter()\n total_pos = 0.0\n length_buckets = {\n \"short\": [],\n \"medium\": [],\n \"long\": [],\n }\n\n for seq, w in zip(seqs, weights):\n L = len(seq)\n if L <= 13:\n length_buckets[\"short\"].append((seq, w))\n elif L <= 17:\n length_buckets[\"medium\"].append((seq, w))\n else:\n length_buckets[\"long\"].append((seq, w))\n\n if len(seq) >= pos_k:\n s0 = seq[:pos_k]\n s1 = seq[-pos_k:]\n if all(ch in AA_PROPERTIES for ch in s0):\n start_counter[s0] += w\n if all(ch in AA_PROPERTIES for ch in s1):\n end_counter[s1] += w\n total_pos += w\n\n if len(seq) >= 1:\n a0, a1 = seq[0], seq[-1]\n if a0 in AA_PROPERTIES:\n prefix1[a0] += w\n if a1 in AA_PROPERTIES:\n suffix1[a1] += w\n\n if total_pos > 0:\n for motif, cnt in start_counter.most_common(self.top_pos_motifs):\n features[f\"pos_start3_{motif}\"] = float(cnt / total_pos)\n for motif, cnt in end_counter.most_common(self.top_pos_motifs):\n features[f\"pos_end3_{motif}\"] = float(cnt / total_pos)\n for aa, cnt in prefix1.items():\n features[f\"pos_first_{aa}\"] = float(cnt / total_pos)\n for aa, cnt in suffix1.items():\n features[f\"pos_last_{aa}\"] = float(cnt / total_pos)\n\n total_w = float(np.sum(weights))\n if total_w > 0:\n for bucket, pairs in length_buckets.items():\n bw = float(sum(w for _, w in pairs))\n features[f\"lb_{bucket}_frac\"] = bw / total_w\n if bw > 0:\n bc_start = Counter()\n bc_end = Counter()\n for seq, w in pairs:\n if len(seq) >= 3:\n bc_start[seq[:3]] += w\n bc_end[seq[-3:]] += w\n for motif, cnt in bc_start.most_common(5):\n features[f\"lb_{bucket}_start3_{motif}\"] = float(cnt / bw)\n for motif, cnt in bc_end.most_common(5):\n features[f\"lb_{bucket}_end3_{motif}\"] = float(cnt / bw)\n\n def _gene_usage(self, df, weights, total_weight, features):\n if \"v_call\" in df.columns:\n vf_counter = Counter()\n for fam, w in zip(df[\"v_call\"].fillna(\"\").astype(str).apply(self.gene_family), weights):\n vf_counter[fam] += float(w)\n for fam, cnt in vf_counter.most_common(25):\n features[f\"v_fam_{fam}\"] = float(cnt / total_weight)\n\n if \"j_call\" in df.columns:\n jf_counter = Counter()\n for fam, w in zip(df[\"j_call\"].fillna(\"\").astype(str).apply(self.gene_family), weights):\n jf_counter[fam] += float(w)\n for fam, cnt in jf_counter.most_common(15):\n features[f\"j_fam_{fam}\"] = float(cnt / total_weight)\n\n if (\"v_call\" in df.columns) and (\"j_call\" in df.columns):\n vj_counter = Counter()\n v_series = df[\"v_call\"].fillna(\"\").astype(str).apply(self.gene_family)\n j_series = df[\"j_call\"].fillna(\"\").astype(str).apply(self.gene_family)\n for v, j, w in zip(v_series, j_series, weights):\n vj_counter[f\"{v}|{j}\"] += float(w)\n for pair, cnt in vj_counter.most_common(30):\n features[f\"vj_pair_{pair}\"] = float(cnt / total_weight)\n\n def _template_stats(self, weights, features):\n tmpl = np.asarray(weights, dtype=float)\n if len(tmpl) == 0:\n return\n features[\"tmpl_mean\"] = float(np.mean(tmpl))\n features[\"tmpl_std\"] = float(np.std(tmpl))\n features[\"tmpl_median\"] = float(np.median(tmpl))\n features[\"tmpl_q90\"] = float(np.quantile(tmpl, 0.90))\n features[\"tmpl_top1_over_sum\"] = float(np.max(tmpl) / max(1.0, np.sum(tmpl)))\n features[\"tmpl_log_sum\"] = float(np.log1p(np.sum(tmpl)))\n p_t = tmpl / np.sum(tmpl)\n features[\"tmpl_entropy\"] = float(-(p_t * np.log(p_t + 1e-12)).sum())\n\n def _public_features(self, unique_clone_counter: Counter, total_weight: float, pub_bundle, features):\n if not pub_bundle:\n return\n\n exact_dict = pub_bundle.get(\"exact\", {})\n approx_dict = pub_bundle.get(\"approx\", {})\n\n exact_scores = []\n approx_scores = []\n exact_hit_weight = 0.0\n approx_hit_weight = 0.0\n exact_hit_count = 0\n approx_hit_count = 0\n\n overlap_sum = 0.0\n overlap_count = 0\n\n for seq, cw in unique_clone_counter.items():\n if seq in exact_dict:\n sc = float(exact_dict[seq][\"score\"])\n exact_scores.append(sc * math.log1p(cw))\n exact_hit_weight += cw\n exact_hit_count += 1\n\n sig = sequence_signature(seq)\n if sig in approx_dict:\n sc = float(approx_dict[sig][\"score\"])\n approx_scores.append(sc * math.log1p(cw))\n approx_hit_weight += cw\n approx_hit_count += 1\n overlap_sum += cw\n overlap_count += 1\n\n features[\"pub_hits\"] = float(exact_hit_count)\n features[\"pub_hit_weight\"] = float(exact_hit_weight / max(1.0, total_weight))\n features[\"pub_score_sum\"] = float(np.sum(exact_scores)) if len(exact_scores) else 0.0\n features[\"pub_score_max\"] = float(np.max(exact_scores)) if len(exact_scores) else 0.0\n\n features[\"pub_approx_hits\"] = float(approx_hit_count)\n features[\"pub_approx_hit_weight\"] = float(approx_hit_weight / max(1.0, total_weight))\n features[\"pub_approx_score_sum\"] = float(np.sum(approx_scores)) if len(approx_scores) else 0.0\n features[\"pub_approx_score_max\"] = float(np.max(approx_scores)) if len(approx_scores) else 0.0\n\n features[\"overlap_burden\"] = float(overlap_sum / max(1.0, total_weight))\n features[\"clone_sharing_burden\"] = float(overlap_count / max(1, len(unique_clone_counter)))\n\n def _metadata_features(self, meta_row, ds_id, features):\n if meta_row is None:\n return\n for col in meta_row.index:\n if col in {\"filename\", \"repertoire_id\", \"label_positive\", \"dataset_name\", \"dataset_id\", \"absolute_path\"}:\n continue\n val = meta_row[col]\n coln = str(col).strip().lower()\n if pd.isna(val):\n continue\n if isinstance(val, (int, float, np.number)):\n features[f\"meta_{coln}\"] = float(val)\n else:\n sval = str(val).strip()\n if not sval:\n continue\n if coln in {\"sex\", \"race\", \"locus\", \"chain\", \"receptor_type\"}:\n key = \"\".join(ch if ch.isalnum() else \"_\" for ch in sval.lower())[:25]\n features[f\"meta_{coln}_{key}\"] = 1.0\n elif \"hla\" in coln:\n features[f\"meta_{coln}_present\"] = 1.0\n\n if ds_id == 7 and meta_row is not None and \"race\" in meta_row.index:\n features[\"meta_ds7_race_missing\"] = float(pd.isna(meta_row[\"race\"]))\n if ds_id == 8 and meta_row is not None:\n for hla in [\"hla_A\", \"hla_B\", \"hla_C\", \"hla_DRB1\", \"A\", \"B\", \"C\", \"DRB1\"]:\n if hla in meta_row.index:\n features[f\"meta_{hla}_present\"] = 1.0 if pd.notna(meta_row[hla]) else 0.0\n\n def extract_all(\n self,\n df: pd.DataFrame,\n pub_bundle: Optional[Dict] = None,\n meta_row: Optional[pd.Series] = None,\n ds_id: int = 1,\n ) -> Dict[str, float]:\n features: Dict[str, float] = {}\n if df is None or len(df) == 0:\n return features\n\n seq_counter, _ = counter_from_df(df)\n if len(seq_counter) == 0:\n return features\n\n seqs = list(seq_counter.keys())\n weights = np.array(list(seq_counter.values()), dtype=np.float32)\n total_weight = float(weights.sum())\n\n features[\"seq_count\"] = float(len(df))\n features[\"unique_seq_count\"] = float(len(seq_counter))\n features[\"unique_ratio\"] = float(len(seq_counter) / max(1, len(df)))\n features[\"total_templates\"] = float(total_weight)\n\n lens = np.array([len(s) for s in seqs], dtype=np.float32)\n if len(lens) > 0:\n features[\"len_mean\"] = float(np.average(lens, weights=weights))\n features[\"len_std\"] = float(np.sqrt(np.average((lens - np.average(lens, weights=weights)) ** 2, weights=weights)))\n features[\"len_min\"] = float(np.min(lens))\n features[\"len_max\"] = float(np.max(lens))\n features[\"len_median\"] = weighted_quantile(lens, weights, 0.50)\n features[\"len_q10\"] = weighted_quantile(lens, weights, 0.10)\n features[\"len_q90\"] = weighted_quantile(lens, weights, 0.90)\n\n self._clone_summary(seq_counter, total_weight, features)\n self._aa_physchem(seqs, weights, features)\n self._kmer_features(seqs, weights, features)\n self._positional_features(seqs, weights, features)\n\n uniq_df = (\n df.sort_values(\"templates\", ascending=False)\n .drop_duplicates(\"junction_aa\")\n .copy()\n )\n uniq_df = uniq_df[uniq_df[\"junction_aa\"].isin(seqs)].copy()\n uniq_df = uniq_df.set_index(\"junction_aa\").reindex(seqs).reset_index()\n\n self._gene_usage(uniq_df, weights, total_weight, features)\n self._template_stats(weights, features)\n self._public_features(seq_counter, total_weight, pub_bundle, features)\n self._metadata_features(meta_row, ds_id, features)\n return features\n\"\"\")\n\nbranch_a_models_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nfrom pathlib import Path\nfrom typing import Dict, List, Tuple\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler\n\ntry:\n import xgboost as xgb\n HAS_XGB = True\nexcept Exception:\n HAS_XGB = False\n\ntry:\n import lightgbm as lgb\n HAS_LGB = True\nexcept Exception:\n HAS_LGB = False\n\ntry:\n from catboost import CatBoostClassifier\n HAS_CATBOOST = True\nexcept Exception:\n HAS_CATBOOST = False\n\n\ndef safe_auc(y_true, y_prob) -> float:\n y_true = np.asarray(y_true)\n y_prob = np.asarray(y_prob)\n if len(np.unique(y_true)) < 2:\n return np.nan\n return float(roc_auc_score(y_true, y_prob))\n\n\ndef check_gpu() -> bool:\n try:\n import subprocess\n r = subprocess.run([\"nvidia-smi\"], capture_output=True, text=True, timeout=5)\n return r.returncode == 0\n except Exception:\n return False\n\n\ndef select_features_xgb(X_train: pd.DataFrame, y_train: np.ndarray, top_features: int = 650, gpu_ok: bool = False) -> Tuple[List[str], pd.DataFrame]:\n X_train = X_train.fillna(0.0)\n all_cols = X_train.columns.tolist()\n\n protected_prefixes = (\n \"len_\", \"clone_\", \"tmpl_\", \"phys_\", \"aa_\", \"meta_\", \"pub_\", \"v_fam_\", \"j_fam_\", \"vj_pair_\",\n \"seq_count\", \"unique_\", \"total_templates\", \"pos_first_\", \"pos_last_\", \"lb_\", \"hill_\", \"overlap_\",\n \"clone_sharing_\", \"rare_clone_\"\n )\n protected = [c for c in all_cols if c.startswith(protected_prefixes)]\n\n if not HAS_XGB:\n selected = list(dict.fromkeys(protected + all_cols[:max(0, top_features - len(protected))]))\n selected = selected[:min(top_features, len(all_cols))]\n imp = pd.DataFrame({\"feature\": selected, \"gain\": np.linspace(len(selected), 1, len(selected))})\n return selected, imp\n\n dtrain = xgb.DMatrix(X_train, label=y_train, feature_names=all_cols)\n params = {\n \"objective\": \"binary:logistic\",\n \"eval_metric\": \"auc\",\n \"max_depth\": 4,\n \"learning_rate\": 0.08,\n \"subsample\": 0.9,\n \"colsample_bytree\": 0.8,\n \"tree_method\": \"hist\",\n \"device\": \"cuda\" if gpu_ok else \"cpu\",\n \"seed\": 42,\n \"verbosity\": 0,\n }\n\n try:\n bst = xgb.train(params, dtrain, num_boost_round=80, verbose_eval=False)\n except Exception:\n params[\"device\"] = \"cpu\"\n bst = xgb.train(params, dtrain, num_boost_round=80, verbose_eval=False)\n\n scores = bst.get_score(importance_type=\"gain\")\n if len(scores) == 0:\n selected = list(dict.fromkeys(protected + all_cols[:max(0, top_features - len(protected))]))\n selected = selected[:min(top_features, len(all_cols))]\n imp = pd.DataFrame({\"feature\": selected, \"gain\": np.linspace(len(selected), 1, len(selected))})\n return selected, imp\n\n imp = pd.DataFrame({\"feature\": list(scores.keys()), \"gain\": list(scores.values())}).sort_values(\"gain\", ascending=False)\n\n selected = list(dict.fromkeys(protected))\n for f in imp[\"feature\"].tolist():\n if f not in selected:\n selected.append(f)\n if len(selected) >= min(top_features, len(all_cols)):\n break\n\n if len(selected) < min(top_features, len(all_cols)):\n remaining = [c for c in all_cols if c not in selected]\n selected.extend(remaining[:min(top_features, len(all_cols)) - len(selected)])\n\n return selected, imp\n\n\ndef _train_lr(X_tr, y_tr, X_va):\n pipe = Pipeline([\n (\"scaler\", StandardScaler()),\n (\"lr\", LogisticRegression(\n C=0.8,\n max_iter=3000,\n solver=\"liblinear\",\n class_weight=\"balanced\",\n random_state=42,\n )),\n ])\n pipe.fit(X_tr, y_tr)\n pred = pipe.predict_proba(X_va)[:, 1]\n return pipe, pred\n\n\ndef _train_xgb(X_tr, y_tr, X_va, y_va, ds_scale, seed, gpu_ok):\n if not HAS_XGB:\n return None, None\n dtr = xgb.DMatrix(X_tr, label=y_tr)\n dva = xgb.DMatrix(X_va, label=y_va)\n\n params = {\n \"objective\": \"binary:logistic\",\n \"eval_metric\": \"auc\",\n \"max_depth\": 6,\n \"learning_rate\": 0.03,\n \"subsample\": 0.85,\n \"colsample_bytree\": 0.85,\n \"min_child_weight\": 10,\n \"reg_alpha\": 0.20,\n \"reg_lambda\": 1.50,\n \"scale_pos_weight\": ds_scale,\n \"tree_method\": \"hist\",\n \"device\": \"cuda\" if gpu_ok else \"cpu\",\n \"seed\": seed,\n \"verbosity\": 0,\n }\n try:\n bst = xgb.train(\n params,\n dtr,\n num_boost_round=1200,\n evals=[(dva, \"valid\")],\n early_stopping_rounds=60,\n verbose_eval=False,\n )\n except Exception:\n params[\"device\"] = \"cpu\"\n bst = xgb.train(\n params,\n dtr,\n num_boost_round=1200,\n evals=[(dva, \"valid\")],\n early_stopping_rounds=60,\n verbose_eval=False,\n )\n pred = bst.predict(dva)\n return bst, pred\n\n\ndef _train_lgb(X_tr, y_tr, X_va, y_va, ds_scale, seed, gpu_ok):\n if not HAS_LGB:\n return None, None\n dtr = lgb.Dataset(X_tr, label=y_tr)\n dva = lgb.Dataset(X_va, label=y_va, reference=dtr)\n params = {\n \"objective\": \"binary\",\n \"metric\": \"auc\",\n \"learning_rate\": 0.025,\n \"num_leaves\": 63,\n \"feature_fraction\": 0.85,\n \"bagging_fraction\": 0.85,\n \"bagging_freq\": 1,\n \"min_data_in_leaf\": 20,\n \"lambda_l1\": 0.10,\n \"lambda_l2\": 1.00,\n \"scale_pos_weight\": ds_scale,\n \"device_type\": \"gpu\" if gpu_ok else \"cpu\",\n \"seed\": seed,\n \"verbosity\": -1,\n }\n try:\n bst = lgb.train(\n params,\n dtr,\n num_boost_round=1200,\n valid_sets=[dva],\n callbacks=[lgb.early_stopping(60, verbose=False)],\n )\n except Exception:\n params[\"device_type\"] = \"cpu\"\n bst = lgb.train(\n params,\n dtr,\n num_boost_round=1200,\n valid_sets=[dva],\n callbacks=[lgb.early_stopping(60, verbose=False)],\n )\n pred = bst.predict(X_va)\n return bst, pred\n\n\ndef _train_cat(X_tr, y_tr, X_va, y_va, seed, gpu_ok):\n if not HAS_CATBOOST:\n return None, None\n params = dict(\n loss_function=\"Logloss\",\n eval_metric=\"AUC\",\n iterations=1500,\n depth=6,\n learning_rate=0.03,\n l2_leaf_reg=5.0,\n random_seed=seed,\n verbose=False,\n allow_writing_files=False,\n task_type=\"GPU\" if gpu_ok else \"CPU\",\n auto_class_weights=\"Balanced\",\n )\n try:\n model = CatBoostClassifier(**params)\n model.fit(X_tr, y_tr, eval_set=(X_va, y_va), use_best_model=True, verbose=False)\n except Exception:\n params[\"task_type\"] = \"CPU\"\n model = CatBoostClassifier(**params)\n model.fit(X_tr, y_tr, eval_set=(X_va, y_va), use_best_model=True, verbose=False)\n pred = model.predict_proba(X_va)[:, 1]\n return model, pred\n\n\ndef get_feature_groups(columns: List[str]) -> Dict[str, List[str]]:\n cols = list(columns)\n\n def pick(prefixes):\n return [c for c in cols if c.startswith(prefixes)]\n\n groups = {\n \"core\": [c for c in cols if c.startswith((\"seq_count\", \"unique_\", \"total_templates\", \"len_\", \"clone_\", \"tmpl_\", \"hill_\", \"meta_\"))],\n \"physchem\": pick((\"aa_\", \"phys_\")),\n \"motif\": pick((\"kmer_\", \"pos_\", \"lb_\")),\n \"gene\": pick((\"v_fam_\", \"j_fam_\", \"vj_pair_\")),\n \"public\": pick((\"pub_\", \"overlap_\", \"clone_sharing_\", \"rare_clone_\")),\n }\n groups[\"full\"] = cols\n return groups\n\n\ndef run_ablation_xgb(X: pd.DataFrame, y: np.ndarray, folds_df: pd.DataFrame, gpu_ok: bool, top_features: int = 650):\n groups = get_feature_groups(list(X.columns))\n stages = [\n (\"core\", groups[\"core\"]),\n (\"core+physchem\", list(dict.fromkeys(groups[\"core\"] + groups[\"physchem\"]))),\n (\"core+physchem+motif\", list(dict.fromkeys(groups[\"core\"] + groups[\"physchem\"] + groups[\"motif\"]))),\n (\"core+physchem+motif+gene\", list(dict.fromkeys(groups[\"core\"] + groups[\"physchem\"] + groups[\"motif\"] + groups[\"gene\"]))),\n (\"core+physchem+motif+gene+public\", list(dict.fromkeys(groups[\"core\"] + groups[\"physchem\"] + groups[\"motif\"] + groups[\"gene\"] + groups[\"public\"]))),\n (\"full\", groups[\"full\"]),\n ]\n\n results = []\n use_xgb = HAS_XGB\n\n for stage_name, stage_cols in stages:\n if len(stage_cols) == 0:\n continue\n\n oof = np.zeros(len(X), dtype=float)\n valid_mask = np.zeros(len(X), dtype=bool)\n\n for fold in sorted(folds_df[\"fold\"].dropna().unique()):\n tr_idx = folds_df.index[folds_df[\"fold\"] != fold].to_numpy()\n va_idx = folds_df.index[folds_df[\"fold\"] == fold].to_numpy()\n\n X_tr = X.iloc[tr_idx][stage_cols].fillna(0.0)\n X_va = X.iloc[va_idx][stage_cols].fillna(0.0)\n y_tr = y[tr_idx]\n y_va = y[va_idx]\n\n selected, _ = select_features_xgb(X_tr, y_tr, top_features=min(top_features, len(stage_cols)), gpu_ok=gpu_ok)\n X_tr2 = X_tr[selected].astype(np.float32).values\n X_va2 = X_va[selected].astype(np.float32).values\n\n if use_xgb:\n _, pred = _train_xgb(X_tr2, y_tr, X_va2, y_va, 1.0, 42, gpu_ok)\n if pred is None:\n _, pred = _train_lr(X_tr2, y_tr, X_va2)\n else:\n _, pred = _train_lr(X_tr2, y_tr, X_va2)\n\n oof[va_idx] = pred\n valid_mask[va_idx] = True\n\n auc = safe_auc(y[valid_mask], oof[valid_mask])\n results.append({\n \"stage\": stage_name,\n \"n_features_stage\": len(stage_cols),\n \"auc\": auc,\n })\n\n return pd.DataFrame(results)\n\n\nclass BranchAMultiSeedRunner:\n def __init__(self, seeds, scale_pos_weight_map=None, gpu_ok=False, top_features=650):\n self.seeds = list(seeds)\n self.scale_pos_weight_map = scale_pos_weight_map or {}\n self.gpu_ok = gpu_ok\n self.top_features = top_features\n\n def run_dataset(\n self,\n feature_df: pd.DataFrame,\n fold_df: pd.DataFrame,\n ds_name: str,\n ds_id: int,\n out_dir: Path,\n ):\n out_dir.mkdir(parents=True, exist_ok=True)\n\n df = feature_df.copy().reset_index(drop=True)\n folds = fold_df.copy().reset_index(drop=True)\n\n y = df[\"label_positive\"].astype(int).values\n key_cols = [\"ID\", \"dataset\", \"label_positive\"]\n X_all = df.drop(columns=key_cols, errors=\"ignore\").fillna(0.0)\n\n selected_records = []\n oof_frames = []\n summary_rows = []\n ds_scale = self.scale_pos_weight_map.get(ds_id, 1.0)\n\n for seed in self.seeds:\n seed_pred = {\n \"ID\": df[\"ID\"].values,\n \"dataset\": df[\"dataset\"].values,\n \"label_positive\": y,\n \"fold\": folds[\"fold\"].values,\n \"seed\": seed,\n }\n\n available_models = [\"LR\"]\n if HAS_XGB:\n available_models.append(\"XGB\")\n if HAS_LGB:\n available_models.append(\"LGB\")\n if HAS_CATBOOST:\n available_models.append(\"CAT\")\n\n for m in available_models:\n seed_pred[m] = np.nan\n\n for fold in sorted(folds[\"fold\"].dropna().unique()):\n tr_idx = folds.index[folds[\"fold\"] != fold].to_numpy()\n va_idx = folds.index[folds[\"fold\"] == fold].to_numpy()\n\n X_tr = X_all.iloc[tr_idx].copy()\n X_va = X_all.iloc[va_idx].copy()\n y_tr = y[tr_idx]\n y_va = y[va_idx]\n\n selected_cols, _ = select_features_xgb(\n X_tr, y_tr, top_features=min(self.top_features, X_tr.shape[1]), gpu_ok=self.gpu_ok\n )\n selected_records.append(\n pd.DataFrame({\n \"dataset\": ds_name,\n \"seed\": seed,\n \"fold\": fold,\n \"feature\": selected_cols,\n })\n )\n\n X_tr_np = X_tr[selected_cols].fillna(0.0).astype(np.float32).values\n X_va_np = X_va[selected_cols].fillna(0.0).astype(np.float32).values\n\n _, pred_lr = _train_lr(X_tr_np, y_tr, X_va_np)\n seed_pred[\"LR\"][va_idx] = pred_lr\n summary_rows.append({\"dataset\": ds_name, \"seed\": seed, \"fold\": fold, \"model\": \"LR\", \"auc\": safe_auc(y_va, pred_lr)})\n\n if HAS_XGB:\n _, pred_xgb = _train_xgb(X_tr_np, y_tr, X_va_np, y_va, ds_scale, seed, self.gpu_ok)\n seed_pred[\"XGB\"][va_idx] = pred_xgb\n summary_rows.append({\"dataset\": ds_name, \"seed\": seed, \"fold\": fold, \"model\": \"XGB\", \"auc\": safe_auc(y_va, pred_xgb)})\n\n if HAS_LGB:\n _, pred_lgb = _train_lgb(X_tr_np, y_tr, X_va_np, y_va, ds_scale, seed, self.gpu_ok)\n seed_pred[\"LGB\"][va_idx] = pred_lgb\n summary_rows.append({\"dataset\": ds_name, \"seed\": seed, \"fold\": fold, \"model\": \"LGB\", \"auc\": safe_auc(y_va, pred_lgb)})\n\n if HAS_CATBOOST:\n _, pred_cat = _train_cat(X_tr_np, y_tr, X_va_np, y_va, seed, self.gpu_ok)\n seed_pred[\"CAT\"][va_idx] = pred_cat\n summary_rows.append({\"dataset\": ds_name, \"seed\": seed, \"fold\": fold, \"model\": \"CAT\", \"auc\": safe_auc(y_va, pred_cat)})\n\n pred_df = pd.DataFrame(seed_pred)\n model_cols = [c for c in [\"LR\", \"XGB\", \"LGB\", \"CAT\"] if c in pred_df.columns]\n pred_df[\"AVG\"] = pred_df[model_cols].mean(axis=1)\n oof_frames.append(pred_df)\n\n for m in model_cols + [\"AVG\"]:\n summary_rows.append({\n \"dataset\": ds_name,\n \"seed\": seed,\n \"fold\": -1,\n \"model\": m,\n \"auc\": safe_auc(pred_df[\"label_positive\"].values, pred_df[m].values),\n })\n\n oof_df = pd.concat(oof_frames, ignore_index=True)\n summary_df = pd.DataFrame(summary_rows)\n\n agg_summary = (\n summary_df.groupby([\"dataset\", \"seed\", \"model\"], as_index=False)[\"auc\"]\n .agg(mean_auc=\"mean\", std_auc=\"std\", max_auc=\"max\")\n .sort_values([\"dataset\", \"seed\", \"mean_auc\"], ascending=[True, True, False])\n )\n\n overall_summary = (\n agg_summary.groupby([\"dataset\", \"model\"], as_index=False)[\"mean_auc\"]\n .agg(seed_mean_auc=\"mean\", seed_std_auc=\"std\", seed_max_auc=\"max\")\n .sort_values([\"dataset\", \"seed_mean_auc\"], ascending=[True, False])\n )\n\n selected_df = pd.concat(selected_records, ignore_index=True) if selected_records else pd.DataFrame(columns=[\"dataset\",\"seed\",\"fold\",\"feature\"])\n selected_df.to_csv(out_dir / f\"selected_features_{ds_name}.csv\", index=False)\n\n return oof_df, summary_df, agg_summary, overall_summary\n\"\"\")\n\ntrain_branch_a_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nimport sys\nfrom pathlib import Path\n\nimport pandas as pd\nimport yaml\nfrom joblib import Parallel, delayed\nfrom tqdm import tqdm\n\nPROJECT_ROOT = Path(__file__).resolve().parent\nif str(PROJECT_ROOT) not in sys.path:\n sys.path.insert(0, str(PROJECT_ROOT))\n\nfrom src.branch_a_features import (\n FeatureExtractor,\n build_public_clone_bundle,\n read_repertoire,\n)\nfrom src.branch_a_models import BranchAMultiSeedRunner, check_gpu, run_ablation_xgb\nfrom src.utils import ensure_dir, dataset_id_from_name, seed_everything\n\n\nSCALE_POS_WEIGHT = {\n 1: 1.0, 2: 1.0, 3: 1.0, 4: 1.0, 5: 1.0, 6: 1.0, 7: 5.04, 8: 2.05\n}\n\n\ndef _extract_one(row_dict, ds_path: Path, ds_id: int, extractor: FeatureExtractor, cfg: dict, pub_bundle=None):\n df = read_repertoire(\n ds_path / row_dict[\"filename\"],\n max_seqs=cfg[\"max_sequences_per_file\"],\n random_state=cfg[\"random_state\"],\n )\n meta_row = pd.Series(row_dict)\n feats = extractor.extract_all(df, pub_bundle=pub_bundle, meta_row=meta_row, ds_id=ds_id)\n return {\n \"ID\": row_dict[\"repertoire_id\"],\n \"dataset\": row_dict[\"dataset_name\"],\n \"label_positive\": int(row_dict[\"label_positive\"]) if pd.notna(row_dict[\"label_positive\"]) else pd.NA,\n **feats,\n }\n\n\ndef build_base_feature_table(meta_ds: pd.DataFrame, ds_path: Path, ds_id: int, extractor: FeatureExtractor, cfg: dict):\n rows = meta_ds.to_dict(orient=\"records\")\n feats = Parallel(n_jobs=cfg[\"n_jobs\"], backend=\"loky\")(\n delayed(_extract_one)(r, ds_path, ds_id, extractor, cfg, None)\n for r in tqdm(rows, total=len(rows), leave=False)\n )\n out = pd.DataFrame(feats).fillna(0.0)\n return out\n\n\ndef add_fold_local_public_features(base_df: pd.DataFrame, meta_ds: pd.DataFrame, fold_df: pd.DataFrame, ds_path: Path, ds_id: int, extractor: FeatureExtractor, cfg: dict):\n base_df = base_df.copy().reset_index(drop=True)\n meta_ds = meta_ds.copy().reset_index(drop=True)\n fold_df = fold_df.copy().reset_index(drop=True)\n\n if not ((base_df[\"ID\"].astype(str).tolist() == meta_ds[\"repertoire_id\"].astype(str).tolist()) and\n (meta_ds[\"repertoire_id\"].astype(str).tolist() == fold_df[\"repertoire_id\"].astype(str).tolist())):\n raise ValueError(\"ID alignment mismatch among base_df, meta_ds, and fold_df\")\n\n final_df = base_df.copy()\n pub_cols_seen = set()\n\n for fold in sorted(fold_df[\"fold\"].dropna().unique()):\n tr_meta = meta_ds.loc[fold_df[\"fold\"] != fold].copy()\n va_idx = fold_df.index[fold_df[\"fold\"] == fold].to_numpy()\n\n bundle = build_public_clone_bundle(\n fold_train_meta=tr_meta,\n dataset_path=ds_path,\n max_files=cfg[\"pub_max_files\"],\n min_freq=cfg[\"pub_min_freq\"],\n enrichment=cfg[\"pub_enrich\"],\n top_n=cfg[\"pub_top_n\"],\n max_seqs_per_file=cfg[\"max_sequences_per_file\"],\n random_state=cfg[\"random_state\"],\n )\n\n fold_rows = []\n for idx in va_idx:\n row_dict = meta_ds.iloc[idx].to_dict()\n x = _extract_one(row_dict, ds_path, ds_id, extractor, cfg, pub_bundle=bundle)\n fold_rows.append({\"ID\": row_dict[\"repertoire_id\"], **x})\n\n fold_pub = pd.DataFrame(fold_rows)\n fold_pub = fold_pub.loc[:, ~fold_pub.columns.duplicated()].copy()\n\n protected = {\"ID\", \"dataset\", \"label_positive\"}\n pub_cols = [c for c in fold_pub.columns if c not in protected]\n pub_only_cols = [c for c in pub_cols if c.startswith((\"pub_\", \"overlap_\", \"clone_sharing_\"))]\n\n if len(pub_only_cols) == 0:\n continue\n\n pub_cols_seen.update(pub_only_cols)\n\n for c in pub_only_cols:\n if c not in final_df.columns:\n final_df[c] = 0.0\n\n id_to_values = fold_pub.set_index(\"ID\")[pub_only_cols]\n current_ids = final_df.loc[va_idx, \"ID\"].astype(str).tolist()\n aligned = id_to_values.reindex(current_ids).fillna(0.0).to_numpy()\n\n for j, c in enumerate(pub_only_cols):\n final_df.loc[va_idx, c] = aligned[:, j]\n\n for c in pub_cols_seen:\n final_df[c] = pd.to_numeric(final_df[c], errors=\"coerce\").fillna(0.0)\n\n return final_df\n\n\ndef main():\n cfg_base = yaml.safe_load((PROJECT_ROOT / \"configs\" / \"base.yaml\").read_text())\n cfg_branch = yaml.safe_load((PROJECT_ROOT / \"configs\" / \"branch_a.yaml\").read_text())\n\n runtime = cfg_base[\"runtime\"]\n phase1_paths = cfg_base[\"paths\"]\n cfg = {\n \"random_state\": runtime[\"random_state\"],\n \"n_jobs\": runtime[\"n_jobs\"],\n \"k_list\": cfg_branch[\"features\"][\"k_list\"],\n \"top_kmers_per_k\": cfg_branch[\"features\"][\"top_kmers_per_k\"],\n \"top_pos_motifs\": cfg_branch[\"features\"][\"top_pos_motifs\"],\n \"max_sequences_per_file\": cfg_branch[\"features\"][\"max_sequences_per_file\"],\n \"pub_max_files\": cfg_branch[\"public_clone\"][\"max_files\"],\n \"pub_min_freq\": cfg_branch[\"public_clone\"][\"min_freq\"],\n \"pub_enrich\": cfg_branch[\"public_clone\"][\"enrichment\"],\n \"pub_top_n\": cfg_branch[\"public_clone\"][\"top_n_default\"],\n }\n\n output_root = ensure_dir(cfg_branch[\"paths\"][\"output_root\"])\n phase1_root = Path(phase1_paths[\"output_root\"])\n\n seed_everything(runtime[\"random_state\"])\n gpu_ok = check_gpu()\n\n canonical = pd.read_csv(phase1_root / \"canonical_metadata.csv\")\n all_folds = pd.read_csv(phase1_root / \"all_folds.csv\")\n\n train_root = Path(phase1_paths[\"train_root\"])\n\n seeds = cfg_branch[\"training\"][\"seeds\"]\n top_features = cfg_branch[\"training\"][\"top_features\"]\n\n extractor = FeatureExtractor(\n k_list=cfg[\"k_list\"],\n top_kmers_per_k=cfg[\"top_kmers_per_k\"],\n top_pos_motifs=cfg[\"top_pos_motifs\"],\n )\n\n oof_all = []\n summary_all = []\n ablation_all = []\n\n train_canonical = canonical[canonical[\"source\"] == \"train\"].copy()\n train_canonical = train_canonical[pd.notna(train_canonical[\"label_positive\"])].copy()\n\n dataset_names = sorted(train_canonical[\"dataset_name\"].unique().tolist())\n\n print(\"=\" * 80)\n print(\"PHASE 2: Branch A - repertoire composition baseline\")\n print(\"=\" * 80)\n print(\"Datasets:\", dataset_names)\n print(\"GPU available:\", gpu_ok)\n\n for ds_name in dataset_names:\n ds_id = dataset_id_from_name(ds_name)\n ds_path = train_root / ds_name\n\n print(f\"\\\\n{'-' * 80}\")\n print(f\"Branch A training: {ds_name} (id={ds_id})\")\n print(f\"{'-' * 80}\")\n\n meta_ds = train_canonical[train_canonical[\"dataset_name\"] == ds_name].copy().reset_index(drop=True)\n fold_ds = all_folds[all_folds[\"dataset_name\"] == ds_name].copy().reset_index(drop=True)\n\n keep_cols = [\"dataset_name\", \"repertoire_id\", \"fold\", \"splitter\", \"n_splits_used\"]\n fold_ds = fold_ds[keep_cols].copy()\n meta_ds = meta_ds.merge(fold_ds, on=[\"dataset_name\", \"repertoire_id\"], how=\"inner\")\n fold_ds = meta_ds[[\"dataset_name\", \"repertoire_id\", \"fold\", \"splitter\", \"n_splits_used\"]].copy()\n\n base_df = build_base_feature_table(meta_ds, ds_path, ds_id, extractor, cfg)\n base_cache_path = output_root / f\"branch_a_base_features_{ds_name}.csv\"\n base_df.to_csv(base_cache_path, index=False)\n print(\"Base feature cache saved:\", base_cache_path)\n\n feature_df = add_fold_local_public_features(base_df, meta_ds, fold_ds, ds_path, ds_id, extractor, cfg)\n feature_df.to_csv(output_root / f\"branch_a_features_with_fold_public_{ds_name}.csv\", index=False)\n\n runner = BranchAMultiSeedRunner(\n seeds=seeds,\n scale_pos_weight_map=SCALE_POS_WEIGHT,\n gpu_ok=gpu_ok,\n top_features=top_features,\n )\n\n oof_df, summary_df, agg_summary, overall_summary = runner.run_dataset(\n feature_df=feature_df,\n fold_df=fold_ds,\n ds_name=ds_name,\n ds_id=ds_id,\n out_dir=output_root,\n )\n\n oof_all.append(oof_df)\n summary_all.append(summary_df)\n overall_summary.to_csv(output_root / f\"branch_a_summary_{ds_name}.csv\", index=False)\n\n X = feature_df.drop(columns=[\"ID\", \"dataset\", \"label_positive\"], errors=\"ignore\").fillna(0.0)\n y = feature_df[\"label_positive\"].astype(int).values\n ablation_df = run_ablation_xgb(X, y, fold_ds, gpu_ok=gpu_ok, top_features=top_features)\n ablation_df.insert(0, \"dataset\", ds_name)\n ablation_all.append(ablation_df)\n ablation_df.to_csv(output_root / f\"branch_a_feature_ablation_{ds_name}.csv\", index=False)\n\n print(\"\\\\nSummary:\")\n print(overall_summary.to_string(index=False))\n print(\"\\\\nAblation:\")\n print(ablation_df.to_string(index=False))\n\n final_oof = pd.concat(oof_all, ignore_index=True) if len(oof_all) else pd.DataFrame()\n final_summary_fold = pd.concat(summary_all, ignore_index=True) if len(summary_all) else pd.DataFrame()\n final_ablation = pd.concat(ablation_all, ignore_index=True) if len(ablation_all) else pd.DataFrame()\n\n final_oof.to_csv(output_root / \"branch_a_oof.csv\", index=False)\n\n if len(final_summary_fold):\n summary = (\n final_summary_fold[final_summary_fold[\"fold\"] >= 0]\n .groupby([\"dataset\", \"seed\", \"model\"], as_index=False)[\"auc\"]\n .agg(mean_auc=\"mean\", std_auc=\"std\", max_auc=\"max\")\n )\n summary.to_csv(output_root / \"branch_a_summary.csv\", index=False)\n else:\n pd.DataFrame().to_csv(output_root / \"branch_a_summary.csv\", index=False)\n\n final_ablation.to_csv(output_root / \"branch_a_feature_ablation.csv\", index=False)\n\n print(\"\\\\n\" + \"=\" * 80)\n print(\"PHASE 2 DONE\")\n print(\"Saved:\")\n print(output_root / \"branch_a_oof.csv\")\n print(output_root / \"branch_a_summary.csv\")\n print(output_root / \"branch_a_feature_ablation.csv\")\n print(\"=\" * 80)\n\n\nif __name__ == \"__main__\":\n main()\n\"\"\")\n\n(PROJECT_ROOT / \"src\" / \"branch_a_features.py\").write_text(branch_a_features_py, encoding=\"utf-8\")\n(PROJECT_ROOT / \"src\" / \"branch_a_models.py\").write_text(branch_a_models_py, encoding=\"utf-8\")\n(PROJECT_ROOT / \"train_branch_a.py\").write_text(train_branch_a_py, encoding=\"utf-8\")\n\nprint(\"Written:\")\nprint(\"-\", PROJECT_ROOT / \"src\" / \"branch_a_features.py\")\nprint(\"-\", PROJECT_ROOT / \"src\" / \"branch_a_models.py\")\nprint(\"-\", PROJECT_ROOT / \"train_branch_a.py\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 3 — imports verify করো\nimport sys\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\nif str(PROJECT_ROOT) not in sys.path:\n sys.path.insert(0, str(PROJECT_ROOT))\n\nfrom src.branch_a_features import FeatureExtractor, build_public_clone_bundle, read_repertoire\nfrom src.branch_a_models import BranchAMultiSeedRunner, run_ablation_xgb, check_gpu\n\nprint(\"Phase 2 imports OK\")\nprint(\"GPU available:\", check_gpu())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 4 — Phase 2 run করো\n!python /kaggle/working/project/train_branch_a.py","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 5 — outputs inspect করো\nfrom pathlib import Path\nimport pandas as pd\n\nOUT_DIR = Path(\"/kaggle/working/project/artifacts/phase2_branch_a\")\n\nprint(\"Files:\")\nfor p in sorted(OUT_DIR.glob(\"*\")):\n print(\"-\", p.name)\n\noof_path = OUT_DIR / \"branch_a_oof.csv\"\nsummary_path = OUT_DIR / \"branch_a_summary.csv\"\nablation_path = OUT_DIR / \"branch_a_feature_ablation.csv\"\n\nprint(\"\\nbranch_a_oof.csv\")\ndisplay(pd.read_csv(oof_path).head())\n\nprint(\"\\nbranch_a_summary.csv\")\ndisplay(pd.read_csv(summary_path))\n\nprint(\"\\nbranch_a_feature_ablation.csv\")\ndisplay(pd.read_csv(ablation_path))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# phase 3 ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 1 — branch_b.yaml write করো\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nbranch_b_yaml = \"\"\"\nenabled: true\n\npaths:\n output_root: /kaggle/working/project/artifacts/phase3_branch_b\n\ndictionary:\n max_sequences_per_file: 30000\n max_repertoires_per_class: 150\n min_freq: 0.10\n enrichment: 3.0\n top_exact: 3000\n top_clusters: 5000\n\ntraining:\n seeds: [42, 52, 62]\n\"\"\"\n\n(PROJECT_ROOT / \"configs\" / \"branch_b.yaml\").write_text(branch_b_yaml.strip() + \"\\n\", encoding=\"utf-8\")\nprint(\"Written:\", PROJECT_ROOT / \"configs\" / \"branch_b.yaml\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 2 — src/branch_b_cluster.py, src/branch_b_public.py, train_branch_b.py write করো\nfrom pathlib import Path\nfrom textwrap import dedent\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nbranch_b_cluster_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nimport itertools\nfrom typing import List, Set\n\nimport numpy as np\n\nfrom .utils import stable_hash\n\n\ndef kmer_set(seq: str, k: int = 3) -> List[str]:\n if not seq or len(seq) < k:\n return [seq] if seq else []\n return list({seq[i:i+k] for i in range(len(seq) - k + 1)})\n\n\ndef minhash_cluster_key(seq: str, k: int = 3, num_hash: int = 6) -> str:\n kms = kmer_set(seq, k=k)\n if len(kms) == 0:\n return f\"mh|L{len(seq)}|EMPTY\"\n hs = sorted(stable_hash(km, mod=10_000_019) for km in kms)\n hs = hs[:num_hash]\n return f\"mh|L{len(seq)}|\" + \"_\".join(map(str, hs))\n\n\ndef deletion_proxy_keys(seq: str, max_del1: int = 4, max_del2: int = 2) -> Set[str]:\n keys = set()\n n = len(seq)\n if n <= 2:\n return keys\n\n # 1-deletion proxies\n idxs = np.linspace(0, n - 1, min(max_del1, n), dtype=int).tolist()\n for i in sorted(set(idxs)):\n s = seq[:i] + seq[i+1:]\n keys.add(f\"d1|L{len(s)}|{stable_hash(s, mod=10_000_019)}\")\n\n # 2-deletion proxies\n if n >= 5:\n cand = sorted(set(idxs))\n pairs = list(itertools.combinations(cand, 2))[:max_del2]\n for i, j in pairs:\n if i == j:\n continue\n keep = [ch for idx, ch in enumerate(seq) if idx not in {i, j}]\n s = \"\".join(keep)\n keys.add(f\"d2|L{len(s)}|{stable_hash(s, mod=10_000_019)}\")\n\n return keys\n\n\ndef signature_key(seq: str) -> str:\n if not seq:\n return \"sig|EMPTY\"\n p2 = seq[:2] if len(seq) >= 2 else seq\n s2 = seq[-2:] if len(seq) >= 2 else seq\n p3 = seq[:3] if len(seq) >= 3 else seq\n s3 = seq[-3:] if len(seq) >= 3 else seq\n return f\"sig|L{len(seq)}|P2{p2}|S2{s2}|P3{p3}|S3{s3}\"\n\n\ndef approx_cluster_keys(seq: str) -> Set[str]:\n keys = set()\n if not seq:\n return keys\n keys.add(signature_key(seq))\n keys.add(minhash_cluster_key(seq, k=3, num_hash=6))\n keys.update(deletion_proxy_keys(seq, max_del1=4, max_del2=2))\n return keys\n\"\"\")\n\nbranch_b_public_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nfrom collections import Counter\nfrom pathlib import Path\nfrom typing import Dict, Tuple\n\nimport numpy as np\nimport pandas as pd\nfrom joblib import Parallel, delayed\n\nfrom .branch_a_features import read_repertoire, counter_from_df\nfrom .branch_b_cluster import approx_cluster_keys\nfrom .utils import ensure_dir\n\n\ndef _cache_one(row_dict: dict, ds_path: Path, max_sequences_per_file: int, random_state: int):\n rep_id = row_dict[\"repertoire_id\"]\n fn = row_dict[\"filename\"]\n df = read_repertoire(ds_path / fn, max_seqs=max_sequences_per_file, random_state=random_state)\n seq_counter, _ = counter_from_df(df)\n total_weight = float(sum(seq_counter.values()))\n return rep_id, {\n \"seq_counter\": seq_counter,\n \"total_weight\": total_weight,\n }\n\n\ndef build_dataset_cache(meta_ds: pd.DataFrame, ds_path: Path, max_sequences_per_file: int = 30000, random_state: int = 42, n_jobs: int = 4):\n rows = meta_ds[[\"repertoire_id\", \"filename\"]].drop_duplicates().to_dict(orient=\"records\")\n cached = Parallel(n_jobs=n_jobs, backend=\"loky\")(\n delayed(_cache_one)(r, ds_path, max_sequences_per_file, random_state)\n for r in rows\n )\n return dict(cached)\n\n\ndef _maybe_cap_rows(df: pd.DataFrame, max_repertoires_per_class: int | None, random_state: int = 42):\n if max_repertoires_per_class is None:\n return df.copy()\n if len(df) <= max_repertoires_per_class:\n return df.copy()\n return df.sample(n=max_repertoires_per_class, random_state=random_state).copy()\n\n\ndef build_fold_signal_catalog(\n fold_train_meta: pd.DataFrame,\n cache: Dict[str, Dict],\n min_freq: float = 0.10,\n enrichment: float = 3.0,\n top_exact: int = 3000,\n top_clusters: int = 5000,\n max_repertoires_per_class: int | None = 150,\n random_state: int = 42,\n):\n meta = fold_train_meta.copy()\n meta[\"label_positive\"] = pd.to_numeric(meta[\"label_positive\"], errors=\"coerce\").fillna(0).astype(int)\n\n pos_rows = _maybe_cap_rows(meta[meta[\"label_positive\"] == 1], max_repertoires_per_class, random_state)\n neg_rows = _maybe_cap_rows(meta[meta[\"label_positive\"] == 0], max_repertoires_per_class, random_state)\n\n pos_exact = Counter()\n neg_exact = Counter()\n\n pos_exact_weight = Counter()\n neg_exact_weight = Counter()\n\n pos_cluster = Counter()\n neg_cluster = Counter()\n\n def update_from_rows(rows, exact_counter, exact_weight_counter, cluster_counter):\n for _, row in rows.iterrows():\n rep_id = row[\"repertoire_id\"]\n if rep_id not in cache:\n continue\n seq_counter = cache[rep_id][\"seq_counter\"]\n seqs = list(seq_counter.keys())\n\n exact_counter.update(seqs)\n exact_weight_counter.update(seq_counter)\n\n rep_cluster_keys = set()\n for seq in seqs:\n rep_cluster_keys.update(approx_cluster_keys(seq))\n cluster_counter.update(rep_cluster_keys)\n\n update_from_rows(pos_rows, pos_exact, pos_exact_weight, pos_cluster)\n update_from_rows(neg_rows, neg_exact, neg_exact_weight, neg_cluster)\n\n n_pos = max(1, len(pos_rows))\n n_neg = max(1, len(neg_rows))\n\n exact_rows = []\n for seq, count in pos_exact.items():\n pf = count / n_pos\n nf = neg_exact.get(seq, 0) / n_neg\n if pf >= min_freq and pf > nf * enrichment:\n score = float(np.log((pf + 1e-6) / (nf + 1e-6)))\n exact_rows.append({\n \"key\": seq,\n \"score\": score,\n \"pos_freq\": pf,\n \"neg_freq\": nf,\n \"pos_weight_sum\": float(pos_exact_weight.get(seq, 0.0)),\n \"neg_weight_sum\": float(neg_exact_weight.get(seq, 0.0)),\n \"kind\": \"exact_sequence\",\n })\n\n cluster_rows = []\n for key, count in pos_cluster.items():\n pf = count / n_pos\n nf = neg_cluster.get(key, 0) / n_neg\n if pf >= min_freq and pf > nf * enrichment:\n score = float(np.log((pf + 1e-6) / (nf + 1e-6)))\n cluster_rows.append({\n \"key\": key,\n \"score\": score,\n \"pos_freq\": pf,\n \"neg_freq\": nf,\n \"kind\": \"approx_cluster\",\n })\n\n exact_df = pd.DataFrame(exact_rows).sort_values(\"score\", ascending=False).head(top_exact) if len(exact_rows) else pd.DataFrame(columns=[\"key\", \"score\", \"pos_freq\", \"neg_freq\", \"pos_weight_sum\", \"neg_weight_sum\", \"kind\"])\n cluster_df = pd.DataFrame(cluster_rows).sort_values(\"score\", ascending=False).head(top_clusters) if len(cluster_rows) else pd.DataFrame(columns=[\"key\", \"score\", \"pos_freq\", \"neg_freq\", \"kind\"])\n\n exact_catalog = {row[\"key\"]: row for row in exact_df.to_dict(orient=\"records\")}\n cluster_catalog = {row[\"key\"]: row for row in cluster_df.to_dict(orient=\"records\")}\n\n bundle = {\n \"exact_catalog\": exact_catalog,\n \"cluster_catalog\": cluster_catalog,\n \"n_pos_repertoires\": n_pos,\n \"n_neg_repertoires\": n_neg,\n }\n return bundle, exact_df, cluster_df\n\n\ndef extract_branch_b_features_from_counter(seq_counter: Counter, bundle: dict) -> dict:\n features = {\n \"public_seq_count\": 0.0,\n \"public_seq_weighted_score\": 0.0,\n \"public_seq_hit_weight\": 0.0,\n \"cluster_hit_count\": 0.0,\n \"cluster_hit_weight\": 0.0,\n \"cluster_enrichment_sum\": 0.0,\n \"max_enriched_cluster_score\": 0.0,\n \"exact_or_cluster_hit_count\": 0.0,\n \"cluster_density\": 0.0,\n \"public_cluster_combo_score\": 0.0,\n }\n\n if seq_counter is None or len(seq_counter) == 0:\n return features\n\n exact_catalog = bundle.get(\"exact_catalog\", {})\n cluster_catalog = bundle.get(\"cluster_catalog\", {})\n\n total_weight = float(sum(seq_counter.values()))\n if total_weight <= 0:\n return features\n\n exact_hit_count = 0\n exact_hit_weight = 0.0\n exact_score_sum = 0.0\n\n cluster_hit_count = 0\n cluster_hit_weight = 0.0\n cluster_score_sum = 0.0\n max_cluster_score = 0.0\n\n combined_hit_count = 0\n\n for seq, cw in seq_counter.items():\n exact_hit = False\n cluster_hit = False\n\n if seq in exact_catalog:\n sc = float(exact_catalog[seq][\"score\"])\n exact_hit = True\n exact_hit_count += 1\n exact_hit_weight += cw\n exact_score_sum += sc * np.log1p(cw)\n\n seq_best_cluster = 0.0\n for key in approx_cluster_keys(seq):\n if key in cluster_catalog:\n cluster_hit = True\n seq_best_cluster = max(seq_best_cluster, float(cluster_catalog[key][\"score\"]))\n\n if cluster_hit:\n cluster_hit_count += 1\n cluster_hit_weight += cw\n cluster_score_sum += seq_best_cluster * np.log1p(cw)\n max_cluster_score = max(max_cluster_score, seq_best_cluster)\n\n if exact_hit or cluster_hit:\n combined_hit_count += 1\n\n n_unique = max(1, len(seq_counter))\n features[\"public_seq_count\"] = float(exact_hit_count)\n features[\"public_seq_weighted_score\"] = float(exact_score_sum)\n features[\"public_seq_hit_weight\"] = float(exact_hit_weight / total_weight)\n\n features[\"cluster_hit_count\"] = float(cluster_hit_count)\n features[\"cluster_hit_weight\"] = float(cluster_hit_weight / total_weight)\n features[\"cluster_enrichment_sum\"] = float(cluster_score_sum)\n features[\"max_enriched_cluster_score\"] = float(max_cluster_score)\n\n features[\"exact_or_cluster_hit_count\"] = float(combined_hit_count)\n features[\"cluster_density\"] = float(cluster_hit_count / n_unique)\n features[\"public_cluster_combo_score\"] = float(exact_score_sum + cluster_score_sum)\n return features\n\n\ndef build_feature_table_from_meta(meta_df: pd.DataFrame, cache: Dict[str, Dict], bundle: dict):\n rows = []\n for _, row in meta_df.iterrows():\n rep_id = row[\"repertoire_id\"]\n seq_counter = cache.get(rep_id, {}).get(\"seq_counter\", Counter())\n feats = extract_branch_b_features_from_counter(seq_counter, bundle)\n rows.append({\n \"ID\": rep_id,\n \"dataset\": row[\"dataset_name\"],\n \"label_positive\": int(row[\"label_positive\"]) if pd.notna(row[\"label_positive\"]) else pd.NA,\n **feats,\n })\n return pd.DataFrame(rows)\n\n\ndef save_catalogs(exact_df: pd.DataFrame, cluster_df: pd.DataFrame, out_dir: Path, ds_name: str):\n out_dir = ensure_dir(out_dir)\n exact_df.to_csv(out_dir / f\"disease_enriched_sequences_{ds_name}.csv\", index=False)\n cluster_df.to_csv(out_dir / f\"cluster_catalog_{ds_name}.csv\", index=False)\n\"\"\")\n\ntrain_branch_b_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nimport sys\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport yaml\n\nPROJECT_ROOT = Path(__file__).resolve().parent\nif str(PROJECT_ROOT) not in sys.path:\n sys.path.insert(0, str(PROJECT_ROOT))\n\nfrom src.branch_a_models import (\n _train_cat,\n _train_lgb,\n _train_lr,\n _train_xgb,\n check_gpu,\n safe_auc,\n)\nfrom src.branch_b_public import (\n build_dataset_cache,\n build_feature_table_from_meta,\n build_fold_signal_catalog,\n)\nfrom src.utils import ensure_dir, seed_everything, dataset_id_from_name\n\n\nSCALE_POS_WEIGHT = {\n 1: 1.0, 2: 1.0, 3: 1.0, 4: 1.0, 5: 1.0, 6: 1.0, 7: 5.04, 8: 2.05\n}\n\n\ndef main():\n cfg_base = yaml.safe_load((PROJECT_ROOT / \"configs\" / \"base.yaml\").read_text())\n cfg_branch = yaml.safe_load((PROJECT_ROOT / \"configs\" / \"branch_b.yaml\").read_text())\n\n runtime = cfg_base[\"runtime\"]\n phase1_paths = cfg_base[\"paths\"]\n out_dir = ensure_dir(cfg_branch[\"paths\"][\"output_root\"])\n\n seed_everything(runtime[\"random_state\"])\n gpu_ok = check_gpu()\n\n canonical = pd.read_csv(Path(phase1_paths[\"output_root\"]) / \"canonical_metadata.csv\")\n all_folds = pd.read_csv(Path(phase1_paths[\"output_root\"]) / \"all_folds.csv\")\n\n train_root = Path(phase1_paths[\"train_root\"])\n seeds = list(cfg_branch[\"training\"][\"seeds\"])\n\n train_meta_all = canonical[canonical[\"source\"] == \"train\"].copy()\n train_meta_all = train_meta_all[pd.notna(train_meta_all[\"label_positive\"])].copy()\n\n dataset_names = sorted(train_meta_all[\"dataset_name\"].unique().tolist())\n\n all_oof = []\n all_summary = []\n\n print(\"=\" * 80)\n print(\"PHASE 3: Branch B - sequence cluster / public signal\")\n print(\"=\" * 80)\n print(\"Datasets:\", dataset_names)\n print(\"GPU available:\", gpu_ok)\n\n for ds_name in dataset_names:\n ds_id = dataset_id_from_name(ds_name)\n ds_scale = SCALE_POS_WEIGHT.get(ds_id, 1.0)\n ds_path = train_root / ds_name\n\n print(f\"\\\\n{'-' * 80}\")\n print(f\"Branch B training: {ds_name} (id={ds_id})\")\n print(f\"{'-' * 80}\")\n\n meta_ds = train_meta_all[train_meta_all[\"dataset_name\"] == ds_name].copy()\n fold_ds = all_folds[all_folds[\"dataset_name\"] == ds_name].copy()\n\n meta_ds = meta_ds.merge(\n fold_ds[[\"dataset_name\", \"repertoire_id\", \"fold\", \"splitter\", \"n_splits_used\"]],\n on=[\"dataset_name\", \"repertoire_id\"],\n how=\"inner\",\n ).reset_index(drop=True)\n\n cache = build_dataset_cache(\n meta_ds=meta_ds,\n ds_path=ds_path,\n max_sequences_per_file=cfg_branch[\"dictionary\"][\"max_sequences_per_file\"],\n random_state=runtime[\"random_state\"],\n n_jobs=runtime[\"n_jobs\"],\n )\n\n seed_frames = []\n summary_rows = []\n all_exact_rows = []\n all_cluster_rows = []\n\n unique_folds = sorted(meta_ds[\"fold\"].dropna().unique().tolist())\n model_names = [\"LR\", \"XGB\", \"LGB\", \"CAT\"]\n\n for seed in seeds:\n pred_store = {\n \"ID\": meta_ds[\"repertoire_id\"].astype(str).tolist(),\n \"dataset\": meta_ds[\"dataset_name\"].astype(str).tolist(),\n \"label_positive\": meta_ds[\"label_positive\"].astype(int).tolist(),\n \"fold\": meta_ds[\"fold\"].tolist(),\n \"seed\": [seed] * len(meta_ds),\n }\n for m in model_names:\n pred_store[m] = [np.nan] * len(meta_ds)\n\n for fold in unique_folds:\n tr_meta = meta_ds[meta_ds[\"fold\"] != fold].copy().reset_index(drop=True)\n va_meta = meta_ds[meta_ds[\"fold\"] == fold].copy().reset_index(drop=True)\n va_global_idx = meta_ds.index[meta_ds[\"fold\"] == fold].to_numpy()\n\n bundle, exact_df, cluster_df = build_fold_signal_catalog(\n fold_train_meta=tr_meta,\n cache=cache,\n min_freq=cfg_branch[\"dictionary\"][\"min_freq\"],\n enrichment=cfg_branch[\"dictionary\"][\"enrichment\"],\n top_exact=cfg_branch[\"dictionary\"][\"top_exact\"],\n top_clusters=cfg_branch[\"dictionary\"][\"top_clusters\"],\n max_repertoires_per_class=cfg_branch[\"dictionary\"][\"max_repertoires_per_class\"],\n random_state=seed,\n )\n\n if len(exact_df):\n exact_df = exact_df.copy()\n exact_df.insert(0, \"dataset\", ds_name)\n exact_df.insert(1, \"fold\", fold)\n all_exact_rows.append(exact_df)\n\n if len(cluster_df):\n cluster_df = cluster_df.copy()\n cluster_df.insert(0, \"dataset\", ds_name)\n cluster_df.insert(1, \"fold\", fold)\n all_cluster_rows.append(cluster_df)\n\n X_tr_df = build_feature_table_from_meta(tr_meta, cache, bundle)\n X_va_df = build_feature_table_from_meta(va_meta, cache, bundle)\n\n feature_cols = [c for c in X_tr_df.columns if c not in [\"ID\", \"dataset\", \"label_positive\"]]\n X_tr = X_tr_df[feature_cols].fillna(0.0).astype(np.float32).values\n X_va = X_va_df[feature_cols].fillna(0.0).astype(np.float32).values\n y_tr = X_tr_df[\"label_positive\"].astype(int).values\n y_va = X_va_df[\"label_positive\"].astype(int).values\n\n _, pred_lr = _train_lr(X_tr, y_tr, X_va)\n for gidx, p in zip(va_global_idx, pred_lr):\n pred_store[\"LR\"][gidx] = float(p)\n summary_rows.append({\"dataset\": ds_name, \"seed\": seed, \"fold\": fold, \"model\": \"LR\", \"auc\": safe_auc(y_va, pred_lr)})\n\n model, pred_xgb = _train_xgb(X_tr, y_tr, X_va, y_va, ds_scale, seed, gpu_ok)\n if pred_xgb is not None:\n for gidx, p in zip(va_global_idx, pred_xgb):\n pred_store[\"XGB\"][gidx] = float(p)\n summary_rows.append({\"dataset\": ds_name, \"seed\": seed, \"fold\": fold, \"model\": \"XGB\", \"auc\": safe_auc(y_va, pred_xgb)})\n\n model, pred_lgb = _train_lgb(X_tr, y_tr, X_va, y_va, ds_scale, seed, gpu_ok)\n if pred_lgb is not None:\n for gidx, p in zip(va_global_idx, pred_lgb):\n pred_store[\"LGB\"][gidx] = float(p)\n summary_rows.append({\"dataset\": ds_name, \"seed\": seed, \"fold\": fold, \"model\": \"LGB\", \"auc\": safe_auc(y_va, pred_lgb)})\n\n model, pred_cat = _train_cat(X_tr, y_tr, X_va, y_va, seed, gpu_ok)\n if pred_cat is not None:\n for gidx, p in zip(va_global_idx, pred_cat):\n pred_store[\"CAT\"][gidx] = float(p)\n summary_rows.append({\"dataset\": ds_name, \"seed\": seed, \"fold\": fold, \"model\": \"CAT\", \"auc\": safe_auc(y_va, pred_cat)})\n\n pred_df = pd.DataFrame(pred_store)\n valid_model_cols = [c for c in model_names if c in pred_df.columns and pred_df[c].notna().any()]\n pred_df[\"AVG\"] = pred_df[valid_model_cols].mean(axis=1)\n seed_frames.append(pred_df)\n\n for m in valid_model_cols + [\"AVG\"]:\n auc_full = safe_auc(pred_df[\"label_positive\"].values, pred_df[m].values)\n summary_rows.append({\"dataset\": ds_name, \"seed\": seed, \"fold\": -1, \"model\": m, \"auc\": auc_full})\n\n ds_oof = pd.concat(seed_frames, ignore_index=True)\n ds_oof.to_csv(out_dir / f\"branch_b_oof_{ds_name}.csv\", index=False)\n all_oof.append(ds_oof)\n\n summary_df = pd.DataFrame(summary_rows)\n if len(summary_df):\n summary_df.to_csv(out_dir / f\"branch_b_summary_{ds_name}.csv\", index=False)\n all_summary.append(summary_df)\n\n exact_all = pd.concat(all_exact_rows, ignore_index=True) if len(all_exact_rows) else pd.DataFrame(columns=[\"dataset\", \"fold\", \"key\", \"score\", \"pos_freq\", \"neg_freq\", \"pos_weight_sum\", \"neg_weight_sum\", \"kind\"])\n cluster_all = pd.concat(all_cluster_rows, ignore_index=True) if len(all_cluster_rows) else pd.DataFrame(columns=[\"dataset\", \"fold\", \"key\", \"score\", \"pos_freq\", \"neg_freq\", \"kind\"])\n\n exact_all.to_csv(out_dir / f\"disease_enriched_sequences_{ds_name}.csv\", index=False)\n cluster_all.to_csv(out_dir / f\"cluster_catalog_{ds_name}.csv\", index=False)\n\n print(\"\\\\nOOF preview:\")\n print(ds_oof.head().to_string(index=False))\n\n if len(summary_df):\n overall = (\n summary_df[summary_df[\"fold\"] >= 0]\n .groupby([\"dataset\", \"seed\", \"model\"], as_index=False)[\"auc\"]\n .agg(mean_auc=\"mean\", std_auc=\"std\", max_auc=\"max\")\n .sort_values([\"dataset\", \"seed\", \"mean_auc\"], ascending=[True, True, False])\n )\n print(\"\\\\nSummary:\")\n print(overall.to_string(index=False))\n\n print(f\"Saved exact catalog -> {out_dir / f'disease_enriched_sequences_{ds_name}.csv'}\")\n print(f\"Saved cluster catalog -> {out_dir / f'cluster_catalog_{ds_name}.csv'}\")\n\n final_oof = pd.concat(all_oof, ignore_index=True) if len(all_oof) else pd.DataFrame()\n final_oof.to_csv(out_dir / \"branch_b_oof.csv\", index=False)\n\n final_summary = pd.concat(all_summary, ignore_index=True) if len(all_summary) else pd.DataFrame()\n if len(final_summary):\n agg = (\n final_summary[final_summary[\"fold\"] >= 0]\n .groupby([\"dataset\", \"seed\", \"model\"], as_index=False)[\"auc\"]\n .agg(mean_auc=\"mean\", std_auc=\"std\", max_auc=\"max\")\n .sort_values([\"dataset\", \"seed\", \"mean_auc\"], ascending=[True, True, False])\n )\n agg.to_csv(out_dir / \"branch_b_summary.csv\", index=False)\n else:\n pd.DataFrame().to_csv(out_dir / \"branch_b_summary.csv\", index=False)\n\n print(\"\\\\n\" + \"=\" * 80)\n print(\"PHASE 3 DONE\")\n print(\"Saved:\")\n print(out_dir / \"branch_b_oof.csv\")\n print(out_dir / \"branch_b_summary.csv\")\n print(\"=\" * 80)\n\n\nif __name__ == \"__main__\":\n main()\n\"\"\")\n\n(PROJECT_ROOT / \"src\" / \"branch_b_cluster.py\").write_text(branch_b_cluster_py, encoding=\"utf-8\")\n(PROJECT_ROOT / \"src\" / \"branch_b_public.py\").write_text(branch_b_public_py, encoding=\"utf-8\")\n(PROJECT_ROOT / \"train_branch_b.py\").write_text(train_branch_b_py, encoding=\"utf-8\")\n\nprint(\"Written:\")\nprint(\"-\", PROJECT_ROOT / \"src\" / \"branch_b_cluster.py\")\nprint(\"-\", PROJECT_ROOT / \"src\" / \"branch_b_public.py\")\nprint(\"-\", PROJECT_ROOT / \"train_branch_b.py\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 3 — imports verify করো\nimport sys\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\nif str(PROJECT_ROOT) not in sys.path:\n sys.path.insert(0, str(PROJECT_ROOT))\n\nfrom src.branch_b_cluster import approx_cluster_keys, signature_key, minhash_cluster_key\nfrom src.branch_b_public import build_dataset_cache, build_fold_signal_catalog, build_feature_table_from_meta\n\nprint(\"Phase 3 imports OK\")\nprint(\"Example keys:\", list(sorted(approx_cluster_keys(\"CASSLGQETQYF\")))[:5])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 4 — Phase 3 run করো\n!python /kaggle/working/project/train_branch_b.py","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 5 — outputs inspect করো\nfrom pathlib import Path\nimport pandas as pd\n\nOUT_DIR = Path(\"/kaggle/working/project/artifacts/phase3_branch_b\")\n\nprint(\"Files:\")\nfor p in sorted(OUT_DIR.glob(\"*\")):\n print(\"-\", p.name)\n\nprint(\"\\nbranch_b_oof.csv\")\ndisplay(pd.read_csv(OUT_DIR / \"branch_b_oof.csv\").head())\n\nsummary_path = OUT_DIR / \"branch_b_summary.csv\"\nif summary_path.exists():\n print(\"\\nbranch_b_summary.csv\")\n display(pd.read_csv(summary_path))\n\n# first available exact catalog\nexact_files = sorted(OUT_DIR.glob(\"disease_enriched_sequences_*.csv\"))\nif exact_files:\n print(f\"\\n{exact_files[0].name}\")\n display(pd.read_csv(exact_files[0]).head())\n\ncluster_files = sorted(OUT_DIR.glob(\"cluster_catalog_*.csv\"))\nif cluster_files:\n print(f\"\\n{cluster_files[0].name}\")\n display(pd.read_csv(cluster_files[0]).head())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Phase 4 ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 1 — branch_c.yaml write করো\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nbranch_c_yaml = \"\"\"\nenabled: true\n\npaths:\n output_root: /kaggle/working/project/artifacts/phase4_branch_c\n\nmodel:\n route: auto\n local_model_dir: \"\"\n batch_size: 16\n max_length: 64\n fallback_dim: 128\n\nembedding:\n max_sequences_per_file: 20000\n top_k_pool: 64\n\ntraining:\n seeds: [42, 52, 62]\n\"\"\"\n\n(PROJECT_ROOT / \"configs\" / \"branch_c.yaml\").write_text(branch_c_yaml.strip() + \"\\n\", encoding=\"utf-8\")\nprint(\"Written:\", PROJECT_ROOT / \"configs\" / \"branch_c.yaml\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 2 — src/branch_c_embeddings.py আর train_branch_c.py write করো\nfrom pathlib import Path\nfrom textwrap import dedent\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nbranch_c_embeddings_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nimport math\nfrom pathlib import Path\nfrom typing import Dict, List, Optional, Tuple\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\n\nfrom .branch_a_features import read_repertoire\nfrom .utils import stable_hash\n\nAA_PROPERTIES = {\n \"A\": {\"hydro\": 1.8, \"vol\": 88.6, \"charge\": 0.0, \"polar\": 0},\n \"R\": {\"hydro\": -4.5, \"vol\": 173.4, \"charge\": 1.0, \"polar\": 1},\n \"N\": {\"hydro\": -3.5, \"vol\": 114.1, \"charge\": 0.0, \"polar\": 1},\n \"D\": {\"hydro\": -3.5, \"vol\": 111.1, \"charge\": -1.0, \"polar\": 1},\n \"C\": {\"hydro\": 2.5, \"vol\": 108.5, \"charge\": 0.0, \"polar\": 0},\n \"Q\": {\"hydro\": -3.5, \"vol\": 143.8, \"charge\": 0.0, \"polar\": 1},\n \"E\": {\"hydro\": -3.5, \"vol\": 138.4, \"charge\": -1.0, \"polar\": 1},\n \"G\": {\"hydro\": -0.4, \"vol\": 60.1, \"charge\": 0.0, \"polar\": 0},\n \"H\": {\"hydro\": -3.2, \"vol\": 153.2, \"charge\": 0.5, \"polar\": 1},\n \"I\": {\"hydro\": 4.5, \"vol\": 166.7, \"charge\": 0.0, \"polar\": 0},\n \"L\": {\"hydro\": 3.8, \"vol\": 166.7, \"charge\": 0.0, \"polar\": 0},\n \"K\": {\"hydro\": -3.9, \"vol\": 168.6, \"charge\": 1.0, \"polar\": 1},\n \"M\": {\"hydro\": 1.9, \"vol\": 162.9, \"charge\": 0.0, \"polar\": 0},\n \"F\": {\"hydro\": 2.8, \"vol\": 189.9, \"charge\": 0.0, \"polar\": 0},\n \"P\": {\"hydro\": -1.6, \"vol\": 112.7, \"charge\": 0.0, \"polar\": 0},\n \"S\": {\"hydro\": -0.8, \"vol\": 89.0, \"charge\": 0.0, \"polar\": 1},\n \"T\": {\"hydro\": -0.7, \"vol\": 116.1, \"charge\": 0.0, \"polar\": 1},\n \"W\": {\"hydro\": -0.9, \"vol\": 227.8, \"charge\": 0.0, \"polar\": 0},\n \"Y\": {\"hydro\": -1.3, \"vol\": 193.6, \"charge\": 0.0, \"polar\": 1},\n \"V\": {\"hydro\": 4.2, \"vol\": 140.0, \"charge\": 0.0, \"polar\": 0},\n}\nAA_LIST = list(\"ARNDCQEGHILKMFPSTWYV\")\nAA_INDEX = {aa: i for i, aa in enumerate(AA_LIST)}\nAROMATIC = set([\"F\", \"W\", \"Y\"])\nHYDROPHOBIC = set([\"A\", \"V\", \"I\", \"L\", \"M\", \"F\", \"W\", \"Y\", \"C\"])\nPOLAR = set([aa for aa, p in AA_PROPERTIES.items() if p[\"polar\"] == 1])\n\n\ndef softmax_np(x):\n x = np.asarray(x, dtype=float)\n if x.ndim == 0:\n return np.array([1.0], dtype=float)\n z = x - np.max(x)\n ez = np.exp(z)\n s = ez.sum()\n if s <= 0:\n return np.ones_like(ez) / max(1, len(ez))\n return ez / s\n\n\ndef aggregate_unique_sequences(df: pd.DataFrame) -> pd.DataFrame:\n if df is None or len(df) == 0:\n return pd.DataFrame(columns=[\"junction_aa\", \"templates\", \"v_call\", \"j_call\"])\n\n x = df.copy()\n x[\"junction_aa\"] = x[\"junction_aa\"].fillna(\"\").astype(str)\n x[\"templates\"] = pd.to_numeric(x[\"templates\"], errors=\"coerce\").fillna(1.0).clip(lower=1.0)\n\n if \"v_call\" not in x.columns:\n x[\"v_call\"] = \"\"\n if \"j_call\" not in x.columns:\n x[\"j_call\"] = \"\"\n\n x = x[x[\"junction_aa\"] != \"\"].copy()\n if len(x) == 0:\n return pd.DataFrame(columns=[\"junction_aa\", \"templates\", \"v_call\", \"j_call\"])\n\n grouped = (\n x.groupby(\"junction_aa\", as_index=False)\n .agg({\n \"templates\": \"sum\",\n \"v_call\": \"first\",\n \"j_call\": \"first\",\n })\n .sort_values(\"templates\", ascending=False)\n .reset_index(drop=True)\n )\n return grouped\n\n\nclass FallbackEmbedder:\n def __init__(self, dim: int = 128):\n self.dim = int(dim)\n self.name = f\"fallback_{self.dim}d\"\n self.base_hash_dim = 96\n if self.dim != 128:\n self.base_hash_dim = max(32, self.dim - 32)\n\n def _hash_kmer(self, kmer: str, mod: int):\n return stable_hash(kmer, mod=mod)\n\n def embed_one(self, seq: str, v_call: str = \"\", j_call: str = \"\") -> np.ndarray:\n vec_parts = []\n\n # 20 AA composition\n aa_vec = np.zeros(20, dtype=np.float32)\n valid = 0\n hydro = vol = charge = charge_abs = 0.0\n aromatic = hydrophobic = polar = 0.0\n\n for aa in seq:\n if aa in AA_INDEX:\n aa_vec[AA_INDEX[aa]] += 1.0\n valid += 1\n hydro += AA_PROPERTIES[aa][\"hydro\"]\n vol += AA_PROPERTIES[aa][\"vol\"]\n charge += AA_PROPERTIES[aa][\"charge\"]\n charge_abs += abs(AA_PROPERTIES[aa][\"charge\"])\n aromatic += float(aa in AROMATIC)\n hydrophobic += float(aa in HYDROPHOBIC)\n polar += float(aa in POLAR)\n\n if valid > 0:\n aa_vec /= valid\n hydro /= valid\n vol /= valid\n charge /= valid\n charge_abs /= valid\n aromatic /= valid\n hydrophobic /= valid\n polar /= valid\n\n vec_parts.append(aa_vec.astype(np.float32))\n\n # 96 hashed 3-mer vector\n hdim = self.base_hash_dim\n hash_vec = np.zeros(hdim, dtype=np.float32)\n if len(seq) >= 3:\n total_k = 0\n for i in range(len(seq) - 3 + 1):\n km = seq[i:i+3]\n idx = self._hash_kmer(km, mod=hdim)\n hash_vec[idx] += 1.0\n total_k += 1\n if total_k > 0:\n hash_vec /= total_k\n vec_parts.append(hash_vec)\n\n # 8 physicochemical / shape features\n phys = np.array([\n min(len(seq), 40) / 40.0,\n hydro,\n vol / 250.0,\n charge,\n charge_abs,\n aromatic,\n hydrophobic,\n polar,\n ], dtype=np.float32)\n vec_parts.append(phys)\n\n # 4 gene hashes\n gene = np.zeros(4, dtype=np.float32)\n if v_call:\n gene[0] = (stable_hash(v_call + \"|0\", mod=10007) % 1000) / 1000.0\n gene[1] = (stable_hash(v_call + \"|1\", mod=10007) % 1000) / 1000.0\n if j_call:\n gene[2] = (stable_hash(j_call + \"|0\", mod=10007) % 1000) / 1000.0\n gene[3] = (stable_hash(j_call + \"|1\", mod=10007) % 1000) / 1000.0\n vec_parts.append(gene.astype(np.float32))\n\n out = np.concatenate(vec_parts).astype(np.float32)\n\n if len(out) < self.dim:\n pad = np.zeros(self.dim - len(out), dtype=np.float32)\n out = np.concatenate([out, pad]).astype(np.float32)\n elif len(out) > self.dim:\n out = out[:self.dim].astype(np.float32)\n return out\n\n def embed_many(self, seqs: List[str], v_calls: Optional[List[str]] = None, j_calls: Optional[List[str]] = None) -> np.ndarray:\n v_calls = v_calls or [\"\"] * len(seqs)\n j_calls = j_calls or [\"\"] * len(seqs)\n mats = [self.embed_one(s, v, j) for s, v, j in zip(seqs, v_calls, j_calls)]\n if len(mats) == 0:\n return np.zeros((0, self.dim), dtype=np.float32)\n return np.vstack(mats).astype(np.float32)\n\n\nclass LocalHFEmbedder:\n def __init__(self, model_dir: str, batch_size: int = 16, max_length: int = 64):\n self.model_dir = str(model_dir)\n self.batch_size = int(batch_size)\n self.max_length = int(max_length)\n self.name = f\"local_hf::{Path(model_dir).name}\"\n\n import torch\n from transformers import AutoModel, AutoTokenizer\n\n self.torch = torch\n self.device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n self.tokenizer = AutoTokenizer.from_pretrained(self.model_dir, local_files_only=True, trust_remote_code=True)\n self.model = AutoModel.from_pretrained(self.model_dir, local_files_only=True, trust_remote_code=True)\n self.model.to(self.device)\n self.model.eval()\n\n hidden = getattr(self.model.config, \"hidden_size\", None)\n if hidden is None:\n hidden = getattr(self.model.config, \"d_model\", None)\n self.dim = int(hidden)\n\n def _format_seq(self, seq: str) -> str:\n # ProtBERT style often works better with spaced chars.\n return \" \".join(list(seq))\n\n def embed_many(self, seqs: List[str], v_calls: Optional[List[str]] = None, j_calls: Optional[List[str]] = None) -> np.ndarray:\n if len(seqs) == 0:\n return np.zeros((0, self.dim), dtype=np.float32)\n\n out_chunks = []\n with self.torch.no_grad():\n for i in range(0, len(seqs), self.batch_size):\n batch = seqs[i:i+self.batch_size]\n batch_in = [self._format_seq(s) for s in batch]\n\n toks = self.tokenizer(\n batch_in,\n padding=True,\n truncation=True,\n max_length=self.max_length,\n return_tensors=\"pt\",\n )\n toks = {k: v.to(self.device) for k, v in toks.items()}\n res = self.model(**toks)\n\n if hasattr(res, \"last_hidden_state\"):\n hs = res.last_hidden_state\n else:\n hs = res[0]\n\n attn = toks[\"attention_mask\"].unsqueeze(-1).float()\n pooled = (hs * attn).sum(dim=1) / attn.sum(dim=1).clamp(min=1.0)\n out_chunks.append(pooled.detach().cpu().numpy().astype(np.float32))\n\n return np.vstack(out_chunks).astype(np.float32)\n\n\ndef create_embedder(route: str = \"auto\", local_model_dir: str = \"\", batch_size: int = 16, max_length: int = 64, fallback_dim: int = 128):\n route = str(route).lower().strip()\n local_model_dir = str(local_model_dir).strip()\n\n if route in {\"local_hf\", \"auto\"} and local_model_dir and Path(local_model_dir).exists():\n try:\n embedder = LocalHFEmbedder(local_model_dir, batch_size=batch_size, max_length=max_length)\n return embedder, {\"route_used\": \"local_hf\", \"embedder_name\": embedder.name, \"embedding_dim\": embedder.dim}\n except Exception as e:\n fallback = FallbackEmbedder(dim=fallback_dim)\n return fallback, {\"route_used\": \"fallback_after_hf_error\", \"embedder_name\": fallback.name, \"embedding_dim\": fallback.dim, \"warning\": str(e)}\n\n fallback = FallbackEmbedder(dim=fallback_dim)\n return fallback, {\"route_used\": \"fallback\", \"embedder_name\": fallback.name, \"embedding_dim\": fallback.dim}\n\n\ndef build_repertoire_vector(\n tsv_path: Path,\n embedder,\n max_sequences_per_file: int = 20000,\n top_k_pool: int = 64,\n random_state: int = 42,\n):\n df = read_repertoire(tsv_path, max_seqs=max_sequences_per_file, random_state=random_state)\n x = aggregate_unique_sequences(df)\n\n if len(x) == 0:\n dim = int(embedder.dim)\n pooled = np.zeros(dim * 4 + 6, dtype=np.float32)\n return pooled, {\n \"n_unique_seq\": 0.0,\n \"total_templates\": 0.0,\n \"pool_topk_used\": 0.0,\n \"embed_model\": embedder.name,\n \"embed_dim\": float(dim),\n }\n\n seqs = x[\"junction_aa\"].astype(str).tolist()\n v_calls = x[\"v_call\"].fillna(\"\").astype(str).tolist() if \"v_call\" in x.columns else [\"\"] * len(seqs)\n j_calls = x[\"j_call\"].fillna(\"\").astype(str).tolist() if \"j_call\" in x.columns else [\"\"] * len(seqs)\n weights = pd.to_numeric(x[\"templates\"], errors=\"coerce\").fillna(1.0).clip(lower=1.0).values.astype(np.float32)\n\n E = embedder.embed_many(seqs, v_calls=v_calls, j_calls=j_calls)\n if len(E) == 0:\n dim = int(embedder.dim)\n pooled = np.zeros(dim * 4 + 6, dtype=np.float32)\n return pooled, {\n \"n_unique_seq\": 0.0,\n \"total_templates\": 0.0,\n \"pool_topk_used\": 0.0,\n \"embed_model\": embedder.name,\n \"embed_dim\": float(dim),\n }\n\n w = weights / max(1e-9, weights.sum())\n weighted_mean = np.average(E, axis=0, weights=w)\n\n topk = min(int(top_k_pool), len(E))\n idx_sorted = np.argsort(-weights)[:topk]\n topk_mean = E[idx_sorted].mean(axis=0)\n\n max_pool = E.max(axis=0)\n\n centroid = weighted_mean\n scores = (E @ centroid) / max(1e-6, np.linalg.norm(centroid)) + np.log1p(weights)\n attn = softmax_np(scores)\n attention_pool = np.average(E, axis=0, weights=attn)\n\n stats = np.array([\n float(len(seqs)),\n float(weights.sum()),\n float(topk),\n float(np.mean([len(s) for s in seqs])),\n float(np.std([len(s) for s in seqs])),\n float(np.max(weights) / max(1e-9, weights.sum())),\n ], dtype=np.float32)\n\n pooled = np.concatenate([\n weighted_mean.astype(np.float32),\n topk_mean.astype(np.float32),\n max_pool.astype(np.float32),\n attention_pool.astype(np.float32),\n stats.astype(np.float32),\n ]).astype(np.float32)\n\n meta = {\n \"n_unique_seq\": float(len(seqs)),\n \"total_templates\": float(weights.sum()),\n \"pool_topk_used\": float(topk),\n \"embed_model\": embedder.name,\n \"embed_dim\": float(embedder.dim),\n }\n return pooled, meta\n\n\ndef build_embedding_table(\n meta_df: pd.DataFrame,\n ds_path: Path,\n embedder,\n max_sequences_per_file: int = 20000,\n top_k_pool: int = 64,\n random_state: int = 42,\n):\n rows = []\n for _, row in tqdm(meta_df.iterrows(), total=len(meta_df), leave=False):\n vec, meta = build_repertoire_vector(\n tsv_path=ds_path / row[\"filename\"],\n embedder=embedder,\n max_sequences_per_file=max_sequences_per_file,\n top_k_pool=top_k_pool,\n random_state=random_state,\n )\n record = {\n \"ID\": row[\"repertoire_id\"],\n \"dataset\": row[\"dataset_name\"],\n \"label_positive\": int(row[\"label_positive\"]) if pd.notna(row[\"label_positive\"]) else pd.NA,\n **meta,\n }\n for i, v in enumerate(vec):\n record[f\"emb_{i:04d}\"] = float(v)\n rows.append(record)\n return pd.DataFrame(rows)\n\"\"\")\n\ntrain_branch_c_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nimport sys\nfrom pathlib import Path\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport yaml\nfrom sklearn.decomposition import PCA\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.neural_network import MLPClassifier\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler\n\nPROJECT_ROOT = Path(__file__).resolve().parent\nif str(PROJECT_ROOT) not in sys.path:\n sys.path.insert(0, str(PROJECT_ROOT))\n\nfrom src.branch_a_models import check_gpu, safe_auc\nfrom src.branch_c_embeddings import build_embedding_table, create_embedder\nfrom src.utils import ensure_dir, seed_everything, dataset_id_from_name\n\ntry:\n import xgboost as xgb\n HAS_XGB = True\nexcept Exception:\n HAS_XGB = False\n\nSCALE_POS_WEIGHT = {\n 1: 1.0, 2: 1.0, 3: 1.0, 4: 1.0, 5: 1.0, 6: 1.0, 7: 5.04, 8: 2.05\n}\n\n\ndef train_lr(X_tr, y_tr, X_va):\n pipe = Pipeline([\n (\"scaler\", StandardScaler()),\n (\"lr\", LogisticRegression(\n C=0.8,\n max_iter=3000,\n solver=\"liblinear\",\n class_weight=\"balanced\",\n random_state=42,\n )),\n ])\n pipe.fit(X_tr, y_tr)\n pred = pipe.predict_proba(X_va)[:, 1]\n return pipe, pred\n\n\ndef train_mlp(X_tr, y_tr, X_va, seed=42):\n pipe = Pipeline([\n (\"scaler\", StandardScaler()),\n (\"mlp\", MLPClassifier(\n hidden_layer_sizes=(128, 64),\n activation=\"relu\",\n alpha=1e-4,\n learning_rate_init=1e-3,\n batch_size=32,\n max_iter=300,\n early_stopping=True,\n random_state=seed,\n )),\n ])\n pipe.fit(X_tr, y_tr)\n pred = pipe.predict_proba(X_va)[:, 1]\n return pipe, pred\n\n\ndef train_xgb(X_tr, y_tr, X_va, y_va, ds_scale, seed=42, gpu_ok=False):\n if not HAS_XGB:\n return None, None\n\n dtr = xgb.DMatrix(X_tr, label=y_tr)\n dva = xgb.DMatrix(X_va, label=y_va)\n\n params = {\n \"objective\": \"binary:logistic\",\n \"eval_metric\": \"auc\",\n \"max_depth\": 5,\n \"learning_rate\": 0.05,\n \"subsample\": 0.9,\n \"colsample_bytree\": 0.85,\n \"min_child_weight\": 5,\n \"reg_alpha\": 0.10,\n \"reg_lambda\": 1.00,\n \"scale_pos_weight\": ds_scale,\n \"tree_method\": \"hist\",\n \"device\": \"cuda\" if gpu_ok else \"cpu\",\n \"seed\": seed,\n \"verbosity\": 0,\n }\n\n try:\n bst = xgb.train(\n params,\n dtr,\n num_boost_round=1000,\n evals=[(dva, \"valid\")],\n early_stopping_rounds=60,\n verbose_eval=False,\n )\n except Exception:\n params[\"device\"] = \"cpu\"\n bst = xgb.train(\n params,\n dtr,\n num_boost_round=1000,\n evals=[(dva, \"valid\")],\n early_stopping_rounds=60,\n verbose_eval=False,\n )\n\n pred = bst.predict(dva)\n return bst, pred\n\n\ndef save_pca_plot(emb_df: pd.DataFrame, out_path: Path):\n feature_cols = [c for c in emb_df.columns if c.startswith(\"emb_\")]\n if len(feature_cols) < 2 or len(emb_df) == 0:\n return\n\n X = emb_df[feature_cols].fillna(0.0).values\n y = pd.to_numeric(emb_df[\"label_positive\"], errors=\"coerce\").fillna(-1).astype(int).values\n\n pca = PCA(n_components=2, random_state=42)\n Z = pca.fit_transform(X)\n\n plt.figure(figsize=(8, 6))\n mask_pos = y == 1\n mask_neg = y == 0\n mask_other = ~(mask_pos | mask_neg)\n\n if mask_neg.any():\n plt.scatter(Z[mask_neg, 0], Z[mask_neg, 1], s=15, alpha=0.6, label=\"negative\")\n if mask_pos.any():\n plt.scatter(Z[mask_pos, 0], Z[mask_pos, 1], s=15, alpha=0.6, label=\"positive\")\n if mask_other.any():\n plt.scatter(Z[mask_other, 0], Z[mask_other, 1], s=15, alpha=0.6, label=\"unknown\")\n\n plt.xlabel(f\"PC1 ({pca.explained_variance_ratio_[0]*100:.1f}%)\")\n plt.ylabel(f\"PC2 ({pca.explained_variance_ratio_[1]*100:.1f}%)\")\n plt.title(\"Branch C Embedding PCA\")\n plt.legend()\n plt.tight_layout()\n plt.savefig(out_path, dpi=180, bbox_inches=\"tight\")\n plt.close()\n\n\ndef main():\n cfg_base = yaml.safe_load((PROJECT_ROOT / \"configs\" / \"base.yaml\").read_text())\n cfg_branch = yaml.safe_load((PROJECT_ROOT / \"configs\" / \"branch_c.yaml\").read_text())\n\n runtime = cfg_base[\"runtime\"]\n phase1_paths = cfg_base[\"paths\"]\n out_dir = ensure_dir(cfg_branch[\"paths\"][\"output_root\"])\n\n seed_everything(runtime[\"random_state\"])\n gpu_ok = check_gpu()\n\n canonical = pd.read_csv(Path(phase1_paths[\"output_root\"]) / \"canonical_metadata.csv\")\n all_folds = pd.read_csv(Path(phase1_paths[\"output_root\"]) / \"all_folds.csv\")\n train_root = Path(phase1_paths[\"train_root\"])\n\n embedder, embedder_info = create_embedder(\n route=cfg_branch[\"model\"][\"route\"],\n local_model_dir=cfg_branch[\"model\"][\"local_model_dir\"],\n batch_size=cfg_branch[\"model\"][\"batch_size\"],\n max_length=cfg_branch[\"model\"][\"max_length\"],\n fallback_dim=cfg_branch[\"model\"][\"fallback_dim\"],\n )\n\n print(\"=\" * 80)\n print(\"PHASE 4: Branch C - pretrained sequence embedding model\")\n print(\"=\" * 80)\n print(\"Embedder info:\", embedder_info)\n print(\"GPU available:\", gpu_ok)\n\n train_meta_all = canonical[canonical[\"source\"] == \"train\"].copy()\n train_meta_all = train_meta_all[pd.notna(train_meta_all[\"label_positive\"])].copy()\n dataset_names = sorted(train_meta_all[\"dataset_name\"].unique().tolist())\n\n seeds = list(cfg_branch[\"training\"][\"seeds\"])\n\n all_emb = []\n all_oof = []\n all_summary = []\n\n for ds_name in dataset_names:\n ds_id = dataset_id_from_name(ds_name)\n ds_scale = SCALE_POS_WEIGHT.get(ds_id, 1.0)\n ds_path = train_root / ds_name\n\n print(f\"\\\\n{'-' * 80}\")\n print(f\"Branch C training: {ds_name} (id={ds_id})\")\n print(f\"{'-' * 80}\")\n\n meta_ds = train_meta_all[train_meta_all[\"dataset_name\"] == ds_name].copy()\n fold_ds = all_folds[all_folds[\"dataset_name\"] == ds_name].copy()\n\n meta_ds = meta_ds.merge(\n fold_ds[[\"dataset_name\", \"repertoire_id\", \"fold\", \"splitter\", \"n_splits_used\"]],\n on=[\"dataset_name\", \"repertoire_id\"],\n how=\"inner\",\n ).reset_index(drop=True)\n\n emb_df = build_embedding_table(\n meta_df=meta_ds,\n ds_path=ds_path,\n embedder=embedder,\n max_sequences_per_file=cfg_branch[\"embedding\"][\"max_sequences_per_file\"],\n top_k_pool=cfg_branch[\"embedding\"][\"top_k_pool\"],\n random_state=runtime[\"random_state\"],\n )\n emb_df[\"dataset_id\"] = ds_id\n emb_df[\"embed_route_used\"] = embedder_info.get(\"route_used\", \"unknown\")\n emb_df[\"fold\"] = meta_ds[\"fold\"].values\n\n emb_df.to_csv(out_dir / f\"embedding_vectors_{ds_name}.csv\", index=False)\n all_emb.append(emb_df)\n\n feature_cols = [c for c in emb_df.columns if c.startswith(\"emb_\")]\n X_all = emb_df[feature_cols].fillna(0.0).astype(np.float32).values\n y = emb_df[\"label_positive\"].astype(int).values\n\n seed_frames = []\n summary_rows = []\n model_cols = [\"LR\", \"XGB\", \"MLP\"]\n\n unique_folds = sorted(emb_df[\"fold\"].dropna().unique().tolist())\n\n for seed in seeds:\n pred_store = {\n \"ID\": emb_df[\"ID\"].astype(str).tolist(),\n \"dataset\": emb_df[\"dataset\"].astype(str).tolist(),\n \"label_positive\": y.tolist(),\n \"fold\": emb_df[\"fold\"].tolist(),\n \"seed\": [seed] * len(emb_df),\n }\n for m in model_cols:\n pred_store[m] = [np.nan] * len(emb_df)\n\n for fold in unique_folds:\n tr_idx = emb_df.index[emb_df[\"fold\"] != fold].to_numpy()\n va_idx = emb_df.index[emb_df[\"fold\"] == fold].to_numpy()\n\n X_tr = X_all[tr_idx]\n X_va = X_all[va_idx]\n y_tr = y[tr_idx]\n y_va = y[va_idx]\n\n _, pred_lr = train_lr(X_tr, y_tr, X_va)\n for idx, p in zip(va_idx, pred_lr):\n pred_store[\"LR\"][idx] = float(p)\n summary_rows.append({\"dataset\": ds_name, \"seed\": seed, \"fold\": fold, \"model\": \"LR\", \"auc\": safe_auc(y_va, pred_lr)})\n\n model, pred_xgb = train_xgb(X_tr, y_tr, X_va, y_va, ds_scale, seed=seed, gpu_ok=gpu_ok)\n if pred_xgb is not None:\n for idx, p in zip(va_idx, pred_xgb):\n pred_store[\"XGB\"][idx] = float(p)\n summary_rows.append({\"dataset\": ds_name, \"seed\": seed, \"fold\": fold, \"model\": \"XGB\", \"auc\": safe_auc(y_va, pred_xgb)})\n\n _, pred_mlp = train_mlp(X_tr, y_tr, X_va, seed=seed)\n for idx, p in zip(va_idx, pred_mlp):\n pred_store[\"MLP\"][idx] = float(p)\n summary_rows.append({\"dataset\": ds_name, \"seed\": seed, \"fold\": fold, \"model\": \"MLP\", \"auc\": safe_auc(y_va, pred_mlp)})\n\n pred_df = pd.DataFrame(pred_store)\n valid_model_cols = [c for c in model_cols if pred_df[c].notna().any()]\n pred_df[\"AVG\"] = pred_df[valid_model_cols].mean(axis=1)\n seed_frames.append(pred_df)\n\n for m in valid_model_cols + [\"AVG\"]:\n auc_full = safe_auc(pred_df[\"label_positive\"].values, pred_df[m].values)\n summary_rows.append({\"dataset\": ds_name, \"seed\": seed, \"fold\": -1, \"model\": m, \"auc\": auc_full})\n\n ds_oof = pd.concat(seed_frames, ignore_index=True)\n ds_oof.to_csv(out_dir / f\"branch_c_oof_{ds_name}.csv\", index=False)\n all_oof.append(ds_oof)\n\n summary_df = pd.DataFrame(summary_rows)\n if len(summary_df):\n summary_df.to_csv(out_dir / f\"branch_c_summary_{ds_name}.csv\", index=False)\n overall = (\n summary_df[summary_df[\"fold\"] >= 0]\n .groupby([\"dataset\", \"seed\", \"model\"], as_index=False)[\"auc\"]\n .agg(mean_auc=\"mean\", std_auc=\"std\", max_auc=\"max\")\n .sort_values([\"dataset\", \"seed\", \"mean_auc\"], ascending=[True, True, False])\n )\n print(\"\\\\nSummary:\")\n print(overall.to_string(index=False))\n all_summary.append(summary_df)\n\n final_emb = pd.concat(all_emb, ignore_index=True) if len(all_emb) else pd.DataFrame()\n final_oof = pd.concat(all_oof, ignore_index=True) if len(all_oof) else pd.DataFrame()\n final_summary = pd.concat(all_summary, ignore_index=True) if len(all_summary) else pd.DataFrame()\n\n # save parquet with safe fallback\n parquet_path = out_dir / \"embedding_repertoire_vectors.parquet\"\n if len(final_emb):\n try:\n final_emb.to_parquet(parquet_path, index=False)\n except Exception:\n final_emb.to_csv(out_dir / \"embedding_repertoire_vectors_fallback.csv\", index=False)\n\n final_oof.to_csv(out_dir / \"branch_c_oof.csv\", index=False)\n\n if len(final_summary):\n agg = (\n final_summary[final_summary[\"fold\"] >= 0]\n .groupby([\"dataset\", \"seed\", \"model\"], as_index=False)[\"auc\"]\n .agg(mean_auc=\"mean\", std_auc=\"std\", max_auc=\"max\")\n .sort_values([\"dataset\", \"seed\", \"mean_auc\"], ascending=[True, True, False])\n )\n agg.to_csv(out_dir / \"branch_c_summary.csv\", index=False)\n else:\n pd.DataFrame().to_csv(out_dir / \"branch_c_summary.csv\", index=False)\n\n if len(final_emb):\n save_pca_plot(final_emb, out_dir / \"embedding_pca_plot.png\")\n\n # optional UMAP if installed\n try:\n import umap\n feature_cols = [c for c in final_emb.columns if c.startswith(\"emb_\")]\n X = final_emb[feature_cols].fillna(0.0).values\n y = pd.to_numeric(final_emb[\"label_positive\"], errors=\"coerce\").fillna(-1).astype(int).values\n reducer = umap.UMAP(n_components=2, random_state=42)\n Z = reducer.fit_transform(X)\n\n plt.figure(figsize=(8, 6))\n mask_pos = y == 1\n mask_neg = y == 0\n if mask_neg.any():\n plt.scatter(Z[mask_neg, 0], Z[mask_neg, 1], s=15, alpha=0.6, label=\"negative\")\n if mask_pos.any():\n plt.scatter(Z[mask_pos, 0], Z[mask_pos, 1], s=15, alpha=0.6, label=\"positive\")\n plt.title(\"Branch C Embedding UMAP\")\n plt.legend()\n plt.tight_layout()\n plt.savefig(out_dir / \"embedding_umap_plot.png\", dpi=180, bbox_inches=\"tight\")\n plt.close()\n except Exception:\n pass\n\n print(\"\\\\n\" + \"=\" * 80)\n print(\"PHASE 4 DONE\")\n print(\"Saved:\")\n print(out_dir / \"branch_c_oof.csv\")\n print(out_dir / \"embedding_repertoire_vectors.parquet\")\n print(out_dir / \"embedding_pca_plot.png\")\n print(\"=\" * 80)\n\n\nif __name__ == \"__main__\":\n main()\n\"\"\")\n\n(PROJECT_ROOT / \"src\" / \"branch_c_embeddings.py\").write_text(branch_c_embeddings_py, encoding=\"utf-8\")\n(PROJECT_ROOT / \"train_branch_c.py\").write_text(train_branch_c_py, encoding=\"utf-8\")\n\nprint(\"Written:\")\nprint(\"-\", PROJECT_ROOT / \"src\" / \"branch_c_embeddings.py\")\nprint(\"-\", PROJECT_ROOT / \"train_branch_c.py\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 3 — imports verify করো\nimport sys\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\nif str(PROJECT_ROOT) not in sys.path:\n sys.path.insert(0, str(PROJECT_ROOT))\n\nfrom src.branch_c_embeddings import create_embedder, build_embedding_table\n\nembedder, info = create_embedder(\n route=\"auto\",\n local_model_dir=\"\",\n batch_size=8,\n max_length=64,\n fallback_dim=128,\n)\nprint(\"Phase 4 imports OK\")\nprint(\"Embedder info:\", info)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 4 — Phase 4 run করো\n!python /kaggle/working/project/train_branch_c.py","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 5 — outputs inspect করো\nfrom pathlib import Path\nimport pandas as pd\n\nOUT_DIR = Path(\"/kaggle/working/project/artifacts/phase4_branch_c\")\n\nprint(\"Files:\")\nfor p in sorted(OUT_DIR.glob(\"*\")):\n print(\"-\", p.name)\n\noof_path = OUT_DIR / \"branch_c_oof.csv\"\nsummary_path = OUT_DIR / \"branch_c_summary.csv\"\nparquet_path = OUT_DIR / \"embedding_repertoire_vectors.parquet\"\n\nprint(\"\\nbranch_c_oof.csv\")\ndisplay(pd.read_csv(oof_path).head())\n\nprint(\"\\nbranch_c_summary.csv\")\ndisplay(pd.read_csv(summary_path))\n\nif parquet_path.exists():\n print(\"\\nembedding_repertoire_vectors.parquet\")\n display(pd.read_parquet(parquet_path).head())\nelse:\n csv_fallback = OUT_DIR / \"embedding_repertoire_vectors_fallback.csv\"\n if csv_fallback.exists():\n print(\"\\nembedding_repertoire_vectors_fallback.csv\")\n display(pd.read_csv(csv_fallback).head())\n else:\n print(\"No embedding table file found.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Phase 5","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 1 — branch_d.yaml write করো\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nbranch_d_yaml = \"\"\"\nenabled: true\n\npaths:\n output_root: /kaggle/working/project/artifacts/phase5_branch_d\n\ndictionary:\n max_sequences_per_file: 30000\n max_repertoires_per_class: 150\n min_freq: 0.10\n enrichment: 3.0\n top_exact: 3000\n top_clusters: 5000\n\nbag:\n top_k_abundant: 256\n random_tail: 128\n force_public_top: 32\n max_seq_len: 32\n\nmodel:\n aa_emb_dim: 32\n gene_emb_dim: 8\n conv_channels: 64\n hidden_dim: 128\n dropout: 0.20\n\ntraining:\n seeds: [42, 52, 62]\n batch_size: 16\n epochs: 8\n lr: 0.001\n weight_decay: 0.0001\n lambda_attr: 0.25\n lambda_sparse: 0.01\n\"\"\"\n\n(PROJECT_ROOT / \"configs\" / \"branch_d.yaml\").write_text(branch_d_yaml.strip() + \"\\n\", encoding=\"utf-8\")\nprint(\"Written:\", PROJECT_ROOT / \"configs\" / \"branch_d.yaml\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 2 — src/branch_d_mil.py আর train_branch_d.py write করো\nfrom pathlib import Path\nfrom textwrap import dedent\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nbranch_d_mil_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nimport random\nfrom pathlib import Path\nfrom typing import Dict, List, Tuple\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset\n\nfrom .branch_a_features import read_repertoire\nfrom .branch_b_cluster import approx_cluster_keys\nfrom .utils import stable_hash\n\nAA_VOCAB = [\"\"] + list(\"ARNDCQEGHILKMFPSTWYV\")\nAA_TO_ID = {aa: i for i, aa in enumerate(AA_VOCAB)}\nPAD_ID = 0\n\n\ndef seed_torch(seed: int = 42):\n random.seed(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed_all(seed)\n\n\ndef gene_family(gene_call: str) -> str:\n if not isinstance(gene_call, str) or not gene_call:\n return \"UNK\"\n return gene_call.split(\"*\")[0].split(\"-\")[0].upper() or \"UNK\"\n\n\ndef gene_hash_id(gene_call: str, mod: int = 512) -> int:\n fam = gene_family(gene_call)\n return int(stable_hash(fam, mod=10_000_019) % mod)\n\n\ndef aggregate_unique_sequences(df: pd.DataFrame) -> pd.DataFrame:\n if df is None or len(df) == 0:\n return pd.DataFrame(columns=[\"junction_aa\", \"templates\", \"v_call\", \"j_call\"])\n\n x = df.copy()\n x[\"junction_aa\"] = x[\"junction_aa\"].fillna(\"\").astype(str)\n x[\"templates\"] = pd.to_numeric(x[\"templates\"], errors=\"coerce\").fillna(1.0).clip(lower=1.0)\n\n if \"v_call\" not in x.columns:\n x[\"v_call\"] = \"\"\n if \"j_call\" not in x.columns:\n x[\"j_call\"] = \"\"\n\n x = x[x[\"junction_aa\"] != \"\"].copy()\n if len(x) == 0:\n return pd.DataFrame(columns=[\"junction_aa\", \"templates\", \"v_call\", \"j_call\"])\n\n grouped = (\n x.groupby(\"junction_aa\", as_index=False)\n .agg({\n \"templates\": \"sum\",\n \"v_call\": \"first\",\n \"j_call\": \"first\",\n })\n .sort_values(\"templates\", ascending=False)\n .reset_index(drop=True)\n )\n return grouped\n\n\ndef build_repertoire_table_cache(meta_ds: pd.DataFrame, ds_path: Path, max_sequences_per_file: int = 30000, random_state: int = 42):\n cache = {}\n for _, row in meta_ds.iterrows():\n rep_id = row[\"repertoire_id\"]\n fn = row[\"filename\"]\n df = read_repertoire(ds_path / fn, max_seqs=max_sequences_per_file, random_state=random_state)\n agg = aggregate_unique_sequences(df)\n cache[rep_id] = agg\n return cache\n\n\ndef weak_target_score(seq: str, bundle: dict) -> float:\n exact_catalog = bundle.get(\"exact_catalog\", {})\n cluster_catalog = bundle.get(\"cluster_catalog\", {})\n\n score = 0.0\n if seq in exact_catalog:\n score += float(exact_catalog[seq][\"score\"])\n\n best_cluster = 0.0\n for key in approx_cluster_keys(seq):\n if key in cluster_catalog:\n best_cluster = max(best_cluster, float(cluster_catalog[key][\"score\"]))\n score += best_cluster\n return float(score)\n\n\ndef encode_seq(seq: str, max_len: int = 32) -> np.ndarray:\n toks = np.zeros(max_len, dtype=np.int64)\n if not seq:\n return toks\n seq = seq[:max_len]\n for i, aa in enumerate(seq):\n toks[i] = AA_TO_ID.get(aa, PAD_ID)\n return toks\n\n\ndef build_bag_from_table(\n agg_df: pd.DataFrame,\n bundle: dict,\n top_k_abundant: int = 256,\n random_tail: int = 128,\n force_public_top: int = 32,\n max_seq_len: int = 32,\n random_state: int = 42,\n):\n rng = np.random.RandomState(random_state)\n\n bag_size = int(top_k_abundant + random_tail)\n\n if agg_df is None or len(agg_df) == 0:\n return {\n \"tokens\": np.zeros((bag_size, max_seq_len), dtype=np.int64),\n \"v_ids\": np.zeros(bag_size, dtype=np.int64),\n \"j_ids\": np.zeros(bag_size, dtype=np.int64),\n \"abundance\": np.zeros(bag_size, dtype=np.float32),\n \"lengths\": np.zeros(bag_size, dtype=np.float32),\n \"weak_targets\": np.zeros(bag_size, dtype=np.float32),\n \"mask\": np.zeros(bag_size, dtype=np.float32),\n \"seqs\": [\"\"] * bag_size,\n }\n\n x = agg_df.copy().reset_index(drop=True)\n x[\"templates\"] = pd.to_numeric(x[\"templates\"], errors=\"coerce\").fillna(1.0).clip(lower=1.0)\n x = x.sort_values(\"templates\", ascending=False).reset_index(drop=True)\n\n weak_scores = np.array([weak_target_score(seq, bundle) for seq in x[\"junction_aa\"].astype(str)], dtype=np.float32)\n x[\"weak_score\"] = weak_scores\n\n public_idx = np.where(weak_scores > 0)[0].tolist()\n public_idx = sorted(public_idx, key=lambda i: (-weak_scores[i], -float(x.iloc[i][\"templates\"])))\n public_idx = public_idx[:force_public_top]\n\n top_idx = list(range(min(top_k_abundant, len(x))))\n selected = []\n seen = set()\n\n for idx in public_idx + top_idx:\n if idx not in seen:\n selected.append(idx)\n seen.add(idx)\n\n remaining = [i for i in range(len(x)) if i not in seen]\n tail_need = max(0, bag_size - len(selected))\n if tail_need > 0 and len(remaining) > 0:\n if len(remaining) <= tail_need:\n tail_idx = remaining\n else:\n tail_idx = rng.choice(remaining, size=tail_need, replace=False).tolist()\n for idx in tail_idx:\n if idx not in seen:\n selected.append(idx)\n seen.add(idx)\n\n selected = selected[:bag_size]\n bag = x.iloc[selected].copy().reset_index(drop=True)\n\n n_real = len(bag)\n if n_real < bag_size:\n pad_rows = pd.DataFrame({\n \"junction_aa\": [\"\"] * (bag_size - n_real),\n \"templates\": [0.0] * (bag_size - n_real),\n \"v_call\": [\"\"] * (bag_size - n_real),\n \"j_call\": [\"\"] * (bag_size - n_real),\n \"weak_score\": [0.0] * (bag_size - n_real),\n })\n bag = pd.concat([bag, pad_rows], ignore_index=True)\n\n seqs = bag[\"junction_aa\"].astype(str).tolist()\n tokens = np.vstack([encode_seq(s, max_len=max_seq_len) for s in seqs]).astype(np.int64)\n\n total_templates = float(max(1e-9, bag[\"templates\"].sum()))\n abundance = np.log1p(pd.to_numeric(bag[\"templates\"], errors=\"coerce\").fillna(0.0).values.astype(np.float32))\n abundance = abundance / max(1e-9, abundance.max() if abundance.max() > 0 else 1.0)\n\n lengths = np.array([min(len(s), max_seq_len) / max_seq_len for s in seqs], dtype=np.float32)\n weak = bag[\"weak_score\"].fillna(0.0).astype(np.float32).values\n weak = weak / max(1e-9, weak.max() if weak.max() > 0 else 1.0)\n\n v_ids = np.array([gene_hash_id(v, mod=512) for v in bag[\"v_call\"].fillna(\"\").astype(str)], dtype=np.int64)\n j_ids = np.array([gene_hash_id(j, mod=512) for j in bag[\"j_call\"].fillna(\"\").astype(str)], dtype=np.int64)\n\n mask = np.array([1.0 if s else 0.0 for s in seqs], dtype=np.float32)\n\n return {\n \"tokens\": tokens,\n \"v_ids\": v_ids,\n \"j_ids\": j_ids,\n \"abundance\": abundance.astype(np.float32),\n \"lengths\": lengths.astype(np.float32),\n \"weak_targets\": weak.astype(np.float32),\n \"mask\": mask.astype(np.float32),\n \"seqs\": seqs,\n }\n\n\nclass RepertoireBagDataset(Dataset):\n def __init__(self, bag_records: List[dict]):\n self.bag_records = bag_records\n\n def __len__(self):\n return len(self.bag_records)\n\n def __getitem__(self, idx):\n r = self.bag_records[idx]\n return {\n \"ID\": r[\"ID\"],\n \"dataset\": r[\"dataset\"],\n \"label\": torch.tensor(float(r[\"label_positive\"]), dtype=torch.float32),\n \"tokens\": torch.tensor(r[\"tokens\"], dtype=torch.long),\n \"v_ids\": torch.tensor(r[\"v_ids\"], dtype=torch.long),\n \"j_ids\": torch.tensor(r[\"j_ids\"], dtype=torch.long),\n \"abundance\": torch.tensor(r[\"abundance\"], dtype=torch.float32),\n \"lengths\": torch.tensor(r[\"lengths\"], dtype=torch.float32),\n \"weak_targets\": torch.tensor(r[\"weak_targets\"], dtype=torch.float32),\n \"mask\": torch.tensor(r[\"mask\"], dtype=torch.float32),\n \"seqs\": r[\"seqs\"],\n }\n\n\nclass SequenceEncoder(nn.Module):\n def __init__(self, aa_vocab_size=21, aa_emb_dim=32, gene_vocab_size=512, gene_emb_dim=8, conv_channels=64, hidden_dim=128, dropout=0.2):\n super().__init__()\n self.aa_emb = nn.Embedding(aa_vocab_size, aa_emb_dim, padding_idx=0)\n self.v_emb = nn.Embedding(gene_vocab_size, gene_emb_dim)\n self.j_emb = nn.Embedding(gene_vocab_size, gene_emb_dim)\n\n self.conv1 = nn.Conv1d(aa_emb_dim, conv_channels, kernel_size=3, padding=1)\n self.conv2 = nn.Conv1d(conv_channels, conv_channels, kernel_size=5, padding=2)\n self.proj = nn.Sequential(\n nn.Linear(conv_channels * 2 + gene_emb_dim * 2 + 2, hidden_dim),\n nn.ReLU(),\n nn.Dropout(dropout),\n nn.Linear(hidden_dim, hidden_dim),\n nn.ReLU(),\n )\n\n def forward(self, tokens, v_ids, j_ids, abundance, lengths):\n # tokens: [B, N, L]\n B, N, L = tokens.shape\n x = self.aa_emb(tokens) # [B, N, L, E]\n x = x.view(B * N, L, -1).transpose(1, 2) # [B*N, E, L]\n x = F.relu(self.conv1(x))\n x = F.relu(self.conv2(x))\n\n x_max = F.adaptive_max_pool1d(x, 1).squeeze(-1)\n x_mean = F.adaptive_avg_pool1d(x, 1).squeeze(-1)\n\n v = self.v_emb(v_ids.view(B * N))\n j = self.j_emb(j_ids.view(B * N))\n\n aux = torch.stack([\n abundance.view(B * N),\n lengths.view(B * N),\n ], dim=1)\n\n out = torch.cat([x_max, x_mean, v, j, aux], dim=1)\n out = self.proj(out)\n out = out.view(B, N, -1)\n return out\n\n\nclass DeepMILModel(nn.Module):\n def __init__(self, aa_emb_dim=32, gene_emb_dim=8, conv_channels=64, hidden_dim=128, dropout=0.2):\n super().__init__()\n self.encoder = SequenceEncoder(\n aa_emb_dim=aa_emb_dim,\n gene_emb_dim=gene_emb_dim,\n conv_channels=conv_channels,\n hidden_dim=hidden_dim,\n dropout=dropout,\n )\n self.attn = nn.Sequential(\n nn.Linear(hidden_dim, hidden_dim),\n nn.Tanh(),\n nn.Linear(hidden_dim, 1),\n )\n self.attr_head = nn.Linear(hidden_dim, 1)\n self.cls_head = nn.Sequential(\n nn.Linear(hidden_dim, hidden_dim),\n nn.ReLU(),\n nn.Dropout(dropout),\n nn.Linear(hidden_dim, 1),\n )\n\n def forward(self, tokens, v_ids, j_ids, abundance, lengths, mask):\n seq_repr = self.encoder(tokens, v_ids, j_ids, abundance, lengths) # [B, N, H]\n attn_logits = self.attn(seq_repr).squeeze(-1) # [B, N]\n attn_logits = attn_logits.masked_fill(mask <= 0, -1e9)\n attn_weights = torch.softmax(attn_logits, dim=1) # [B, N]\n\n bag_repr = torch.sum(seq_repr * attn_weights.unsqueeze(-1), dim=1)\n bag_logit = self.cls_head(bag_repr).squeeze(-1) # [B]\n\n attr_logits = self.attr_head(seq_repr).squeeze(-1) # [B, N]\n attr_probs = torch.sigmoid(attr_logits) * mask\n\n final_seq_score = attr_probs * attn_weights * mask\n return {\n \"bag_logit\": bag_logit,\n \"attn_weights\": attn_weights,\n \"attr_logits\": attr_logits,\n \"attr_probs\": attr_probs,\n \"final_seq_score\": final_seq_score,\n }\n\n\ndef compute_losses(outputs, labels, weak_targets, mask, lambda_attr=0.25, lambda_sparse=0.01):\n bag_logit = outputs[\"bag_logit\"]\n attn = outputs[\"attn_weights\"]\n attr_logits = outputs[\"attr_logits\"]\n\n cls_loss = F.binary_cross_entropy_with_logits(bag_logit, labels)\n\n weak_binary = (weak_targets > 0).float()\n attr_bce = F.binary_cross_entropy_with_logits(attr_logits, weak_binary, reduction=\"none\")\n attr_bce = (attr_bce * mask).sum() / mask.sum().clamp_min(1.0)\n\n weak_sum = weak_targets.sum(dim=1, keepdim=True)\n weak_norm = weak_targets / weak_sum.clamp_min(1e-8)\n\n valid_rank = (weak_sum.squeeze(1) > 0).float()\n kl = weak_norm * (torch.log(weak_norm.clamp_min(1e-8)) - torch.log(attn.clamp_min(1e-8)))\n kl = kl.sum(dim=1)\n rank_consistency = (kl * valid_rank).sum() / valid_rank.sum().clamp_min(1.0)\n\n attr_loss = attr_bce + rank_consistency\n\n entropy = -(attn * torch.log(attn.clamp_min(1e-8)) * mask).sum(dim=1) / mask.sum(dim=1).clamp_min(1.0)\n sparse_loss = entropy.mean()\n\n total = cls_loss + lambda_attr * attr_loss + lambda_sparse * sparse_loss\n\n return {\n \"total\": total,\n \"cls\": cls_loss.detach().item(),\n \"attr\": attr_loss.detach().item(),\n \"sparse\": sparse_loss.detach().item(),\n }\n\n\ndef build_bag_records_from_meta(\n meta_df: pd.DataFrame,\n table_cache: dict,\n bundle: dict,\n bag_cfg: dict,\n random_state: int = 42,\n):\n rows = []\n for i, (_, row) in enumerate(meta_df.iterrows()):\n rep_id = row[\"repertoire_id\"]\n agg_df = table_cache.get(rep_id, pd.DataFrame(columns=[\"junction_aa\", \"templates\", \"v_call\", \"j_call\"]))\n bag = build_bag_from_table(\n agg_df=agg_df,\n bundle=bundle,\n top_k_abundant=bag_cfg[\"top_k_abundant\"],\n random_tail=bag_cfg[\"random_tail\"],\n force_public_top=bag_cfg[\"force_public_top\"],\n max_seq_len=bag_cfg[\"max_seq_len\"],\n random_state=random_state + i,\n )\n rows.append({\n \"ID\": rep_id,\n \"dataset\": row[\"dataset_name\"],\n \"label_positive\": int(row[\"label_positive\"]) if pd.notna(row[\"label_positive\"]) else 0,\n **bag,\n })\n return rows\n\"\"\")\n\ntrain_branch_d_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nimport sys\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom torch.utils.data import DataLoader\nimport yaml\n\nPROJECT_ROOT = Path(__file__).resolve().parent\nif str(PROJECT_ROOT) not in sys.path:\n sys.path.insert(0, str(PROJECT_ROOT))\n\nfrom src.branch_b_public import build_dataset_cache, build_fold_signal_catalog\nfrom src.branch_d_mil import (\n DeepMILModel,\n RepertoireBagDataset,\n build_bag_records_from_meta,\n build_repertoire_table_cache,\n compute_losses,\n seed_torch,\n)\nfrom src.branch_a_models import safe_auc\nfrom src.utils import ensure_dir, seed_everything, dataset_id_from_name\n\nSCALE_POS_WEIGHT = {\n 1: 1.0, 2: 1.0, 3: 1.0, 4: 1.0, 5: 1.0, 6: 1.0, 7: 5.04, 8: 2.05\n}\n\n\ndef collate_fn(batch):\n return {\n \"ID\": [x[\"ID\"] for x in batch],\n \"dataset\": [x[\"dataset\"] for x in batch],\n \"label\": torch.stack([x[\"label\"] for x in batch]),\n \"tokens\": torch.stack([x[\"tokens\"] for x in batch]),\n \"v_ids\": torch.stack([x[\"v_ids\"] for x in batch]),\n \"j_ids\": torch.stack([x[\"j_ids\"] for x in batch]),\n \"abundance\": torch.stack([x[\"abundance\"] for x in batch]),\n \"lengths\": torch.stack([x[\"lengths\"] for x in batch]),\n \"weak_targets\": torch.stack([x[\"weak_targets\"] for x in batch]),\n \"mask\": torch.stack([x[\"mask\"] for x in batch]),\n \"seqs\": [x[\"seqs\"] for x in batch],\n }\n\n\ndef move_to_device(batch, device):\n out = {}\n for k, v in batch.items():\n if torch.is_tensor(v):\n out[k] = v.to(device)\n else:\n out[k] = v\n return out\n\n\ndef evaluate_model(model, loader, device):\n model.eval()\n bag_probs = []\n bag_labels = []\n\n seq_rows = []\n attn_rows = []\n\n with torch.no_grad():\n for batch in loader:\n batch_dev = move_to_device(batch, device)\n outputs = model(\n tokens=batch_dev[\"tokens\"],\n v_ids=batch_dev[\"v_ids\"],\n j_ids=batch_dev[\"j_ids\"],\n abundance=batch_dev[\"abundance\"],\n lengths=batch_dev[\"lengths\"],\n mask=batch_dev[\"mask\"],\n )\n\n probs = torch.sigmoid(outputs[\"bag_logit\"]).detach().cpu().numpy()\n attn = outputs[\"attn_weights\"].detach().cpu().numpy()\n attr = outputs[\"attr_probs\"].detach().cpu().numpy()\n final_seq = outputs[\"final_seq_score\"].detach().cpu().numpy()\n weak = batch[\"weak_targets\"].numpy()\n mask = batch[\"mask\"].numpy()\n\n bag_probs.extend(probs.tolist())\n bag_labels.extend(batch[\"label\"].numpy().tolist())\n\n for i, rep_id in enumerate(batch[\"ID\"]):\n attn_rows.append({\n \"ID\": rep_id,\n \"attn\": attn[i].astype(np.float32),\n \"attr\": attr[i].astype(np.float32),\n \"final_seq\": final_seq[i].astype(np.float32),\n \"mask\": mask[i].astype(np.float32),\n })\n\n valid_idx = np.where(mask[i] > 0)[0].tolist()\n seq_entries = []\n for j in valid_idx:\n seq_entries.append({\n \"ID\": rep_id,\n \"dataset\": batch[\"dataset\"][i],\n \"rank\": int(j + 1),\n \"sequence\": batch[\"seqs\"][i][j],\n \"attention_weight\": float(attn[i][j]),\n \"attr_prob\": float(attr[i][j]),\n \"final_seq_score\": float(final_seq[i][j]),\n \"weak_target\": float(weak[i][j]),\n })\n seq_rows.extend(seq_entries)\n\n auc = safe_auc(np.array(bag_labels), np.array(bag_probs))\n return auc, np.array(bag_probs), seq_rows, attn_rows\n\n\ndef train_one_fold(train_records, val_records, cfg_model, cfg_train, device, seed):\n seed_torch(seed)\n\n train_ds = RepertoireBagDataset(train_records)\n val_ds = RepertoireBagDataset(val_records)\n\n train_loader = DataLoader(train_ds, batch_size=cfg_train[\"batch_size\"], shuffle=True, collate_fn=collate_fn)\n val_loader = DataLoader(val_ds, batch_size=cfg_train[\"batch_size\"], shuffle=False, collate_fn=collate_fn)\n\n model = DeepMILModel(\n aa_emb_dim=cfg_model[\"aa_emb_dim\"],\n gene_emb_dim=cfg_model[\"gene_emb_dim\"],\n conv_channels=cfg_model[\"conv_channels\"],\n hidden_dim=cfg_model[\"hidden_dim\"],\n dropout=cfg_model[\"dropout\"],\n ).to(device)\n\n optimizer = torch.optim.AdamW(\n model.parameters(),\n lr=cfg_train[\"lr\"],\n weight_decay=cfg_train[\"weight_decay\"],\n )\n\n best_state = None\n best_auc = -np.inf\n best_epoch = -1\n\n for epoch in range(cfg_train[\"epochs\"]):\n model.train()\n for batch in train_loader:\n batch_dev = move_to_device(batch, device)\n outputs = model(\n tokens=batch_dev[\"tokens\"],\n v_ids=batch_dev[\"v_ids\"],\n j_ids=batch_dev[\"j_ids\"],\n abundance=batch_dev[\"abundance\"],\n lengths=batch_dev[\"lengths\"],\n mask=batch_dev[\"mask\"],\n )\n losses = compute_losses(\n outputs=outputs,\n labels=batch_dev[\"label\"],\n weak_targets=batch_dev[\"weak_targets\"],\n mask=batch_dev[\"mask\"],\n lambda_attr=cfg_train[\"lambda_attr\"],\n lambda_sparse=cfg_train[\"lambda_sparse\"],\n )\n\n optimizer.zero_grad()\n losses[\"total\"].backward()\n torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=5.0)\n optimizer.step()\n\n val_auc, _, _, _ = evaluate_model(model, val_loader, device)\n if np.isnan(val_auc):\n val_auc = -1.0\n\n if val_auc > best_auc:\n best_auc = val_auc\n best_epoch = epoch\n best_state = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()}\n\n if best_state is not None:\n model.load_state_dict(best_state)\n\n val_auc, val_probs, seq_rows, attn_rows = evaluate_model(model, val_loader, device)\n return model, val_auc, val_probs, seq_rows, attn_rows, best_epoch\n\n\ndef main():\n cfg_base = yaml.safe_load((PROJECT_ROOT / \"configs\" / \"base.yaml\").read_text())\n cfg_branch = yaml.safe_load((PROJECT_ROOT / \"configs\" / \"branch_d.yaml\").read_text())\n\n runtime = cfg_base[\"runtime\"]\n phase1_paths = cfg_base[\"paths\"]\n out_dir = ensure_dir(cfg_branch[\"paths\"][\"output_root\"])\n\n seed_everything(runtime[\"random_state\"])\n device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n canonical = pd.read_csv(Path(phase1_paths[\"output_root\"]) / \"canonical_metadata.csv\")\n all_folds = pd.read_csv(Path(phase1_paths[\"output_root\"]) / \"all_folds.csv\")\n train_root = Path(phase1_paths[\"train_root\"])\n\n train_meta_all = canonical[canonical[\"source\"] == \"train\"].copy()\n train_meta_all = train_meta_all[pd.notna(train_meta_all[\"label_positive\"])].copy()\n\n dataset_names = sorted(train_meta_all[\"dataset_name\"].unique().tolist())\n seeds = list(cfg_branch[\"training\"][\"seeds\"])\n\n cfg_dict = cfg_branch[\"dictionary\"]\n cfg_bag = cfg_branch[\"bag\"]\n cfg_model = cfg_branch[\"model\"]\n cfg_train = cfg_branch[\"training\"]\n\n all_oof = []\n\n print(\"=\" * 80)\n print(\"PHASE 5: Branch D - Deep MIL repertoire model\")\n print(\"=\" * 80)\n print(\"Device:\", device)\n print(\"Datasets:\", dataset_names)\n\n for ds_name in dataset_names:\n ds_id = dataset_id_from_name(ds_name)\n ds_path = train_root / ds_name\n\n print(f\"\\\\n{'-' * 80}\")\n print(f\"Branch D training: {ds_name} (id={ds_id})\")\n print(f\"{'-' * 80}\")\n\n meta_ds = train_meta_all[train_meta_all[\"dataset_name\"] == ds_name].copy()\n fold_ds = all_folds[all_folds[\"dataset_name\"] == ds_name].copy()\n\n meta_ds = meta_ds.merge(\n fold_ds[[\"dataset_name\", \"repertoire_id\", \"fold\", \"splitter\", \"n_splits_used\"]],\n on=[\"dataset_name\", \"repertoire_id\"],\n how=\"inner\",\n ).reset_index(drop=True)\n\n signal_cache = build_dataset_cache(\n meta_ds=meta_ds,\n ds_path=ds_path,\n max_sequences_per_file=cfg_dict[\"max_sequences_per_file\"],\n random_state=runtime[\"random_state\"],\n n_jobs=runtime[\"n_jobs\"],\n )\n table_cache = build_repertoire_table_cache(\n meta_ds=meta_ds,\n ds_path=ds_path,\n max_sequences_per_file=cfg_dict[\"max_sequences_per_file\"],\n random_state=runtime[\"random_state\"],\n )\n\n unique_folds = sorted(meta_ds[\"fold\"].dropna().unique().tolist())\n ds_seed_frames = []\n ds_seq_rows = []\n ds_attn_maps = []\n ds_summary_rows = []\n\n for seed in seeds:\n pred_store = {\n \"ID\": meta_ds[\"repertoire_id\"].astype(str).tolist(),\n \"dataset\": meta_ds[\"dataset_name\"].astype(str).tolist(),\n \"label_positive\": meta_ds[\"label_positive\"].astype(int).tolist(),\n \"fold\": meta_ds[\"fold\"].tolist(),\n \"seed\": [seed] * len(meta_ds),\n \"MIL_PROB\": [np.nan] * len(meta_ds),\n }\n\n for fold in unique_folds:\n tr_meta = meta_ds[meta_ds[\"fold\"] != fold].copy().reset_index(drop=True)\n va_meta = meta_ds[meta_ds[\"fold\"] == fold].copy().reset_index(drop=True)\n va_global_idx = meta_ds.index[meta_ds[\"fold\"] == fold].to_numpy()\n\n bundle, exact_df, cluster_df = build_fold_signal_catalog(\n fold_train_meta=tr_meta,\n cache=signal_cache,\n min_freq=cfg_dict[\"min_freq\"],\n enrichment=cfg_dict[\"enrichment\"],\n top_exact=cfg_dict[\"top_exact\"],\n top_clusters=cfg_dict[\"top_clusters\"],\n max_repertoires_per_class=cfg_dict[\"max_repertoires_per_class\"],\n random_state=seed,\n )\n\n train_records = build_bag_records_from_meta(\n meta_df=tr_meta,\n table_cache=table_cache,\n bundle=bundle,\n bag_cfg=cfg_bag,\n random_state=seed,\n )\n val_records = build_bag_records_from_meta(\n meta_df=va_meta,\n table_cache=table_cache,\n bundle=bundle,\n bag_cfg=cfg_bag,\n random_state=seed + 1000,\n )\n\n model, val_auc, val_probs, seq_rows, attn_rows, best_epoch = train_one_fold(\n train_records=train_records,\n val_records=val_records,\n cfg_model=cfg_model,\n cfg_train=cfg_train,\n device=device,\n seed=seed,\n )\n\n for idx, p in zip(va_global_idx, val_probs):\n pred_store[\"MIL_PROB\"][idx] = float(p)\n\n ds_summary_rows.append({\n \"dataset\": ds_name,\n \"seed\": seed,\n \"fold\": fold,\n \"model\": \"MIL\",\n \"auc\": val_auc,\n \"best_epoch\": best_epoch,\n })\n\n for row in seq_rows:\n row[\"seed\"] = seed\n row[\"fold\"] = fold\n ds_seq_rows.extend(seq_rows)\n\n for row in attn_rows:\n row[\"seed\"] = seed\n row[\"fold\"] = fold\n ds_attn_maps.extend(attn_rows)\n\n seed_df = pd.DataFrame(pred_store)\n ds_seed_frames.append(seed_df)\n\n ds_summary_rows.append({\n \"dataset\": ds_name,\n \"seed\": seed,\n \"fold\": -1,\n \"model\": \"MIL\",\n \"auc\": safe_auc(seed_df[\"label_positive\"].values, seed_df[\"MIL_PROB\"].values),\n \"best_epoch\": -1,\n })\n\n ds_oof = pd.concat(ds_seed_frames, ignore_index=True)\n ds_oof.to_csv(out_dir / f\"branch_d_oof_{ds_name}.csv\", index=False)\n all_oof.append(ds_oof)\n\n # save sequence-level scores\n seq_df = pd.DataFrame(ds_seq_rows)\n if len(seq_df):\n seq_path = out_dir / f\"mil_sequence_scores_{ds_name}.parquet\"\n try:\n seq_df.to_parquet(seq_path, index=False)\n except Exception:\n seq_df.to_csv(out_dir / f\"mil_sequence_scores_{ds_name}_fallback.csv\", index=False)\n\n # save attention maps\n if len(ds_attn_maps):\n attn_ids = [x[\"ID\"] for x in ds_attn_maps]\n attn_seed = [x[\"seed\"] for x in ds_attn_maps]\n attn_fold = [x[\"fold\"] for x in ds_attn_maps]\n attn_arr = np.vstack([x[\"attn\"] for x in ds_attn_maps]).astype(np.float32)\n attr_arr = np.vstack([x[\"attr\"] for x in ds_attn_maps]).astype(np.float32)\n final_arr = np.vstack([x[\"final_seq\"] for x in ds_attn_maps]).astype(np.float32)\n mask_arr = np.vstack([x[\"mask\"] for x in ds_attn_maps]).astype(np.float32)\n\n np.savez_compressed(\n out_dir / f\"mil_attention_maps_{ds_name}.npz\",\n repertoire_ids=np.array(attn_ids, dtype=object),\n seeds=np.array(attn_seed, dtype=np.int64),\n folds=np.array(attn_fold, dtype=np.int64),\n attention=attn_arr,\n attr_probs=attr_arr,\n final_seq_scores=final_arr,\n mask=mask_arr,\n )\n\n summary_df = pd.DataFrame(ds_summary_rows)\n if len(summary_df):\n summary_df.to_csv(out_dir / f\"branch_d_summary_{ds_name}.csv\", index=False)\n\n overall = (\n summary_df[summary_df[\"fold\"] >= 0]\n .groupby([\"dataset\", \"seed\", \"model\"], as_index=False)\n .agg(mean_auc=(\"auc\", \"mean\"), std_auc=(\"auc\", \"std\"), max_auc=(\"auc\", \"max\"))\n .sort_values([\"dataset\", \"seed\", \"mean_auc\"], ascending=[True, True, False])\n )\n print(\"\\\\nSummary:\")\n print(overall.to_string(index=False))\n\n print(f\"Saved OOF -> {out_dir / f'branch_d_oof_{ds_name}.csv'}\")\n print(f\"Saved seq parquet -> {out_dir / f'mil_sequence_scores_{ds_name}.parquet'}\")\n print(f\"Saved attn npz -> {out_dir / f'mil_attention_maps_{ds_name}.npz'}\")\n\n final_oof = pd.concat(all_oof, ignore_index=True) if len(all_oof) else pd.DataFrame()\n final_oof.to_csv(out_dir / \"branch_d_oof.csv\", index=False)\n\n print(\"\\\\n\" + \"=\" * 80)\n print(\"PHASE 5 DONE\")\n print(\"Saved:\")\n print(out_dir / \"branch_d_oof.csv\")\n print(\"=\" * 80)\n\n\nif __name__ == \"__main__\":\n main()\n\"\"\")\n\n(PROJECT_ROOT / \"src\" / \"branch_d_mil.py\").write_text(branch_d_mil_py, encoding=\"utf-8\")\n(PROJECT_ROOT / \"train_branch_d.py\").write_text(train_branch_d_py, encoding=\"utf-8\")\n\nprint(\"Written:\")\nprint(\"-\", PROJECT_ROOT / \"src\" / \"branch_d_mil.py\")\nprint(\"-\", PROJECT_ROOT / \"train_branch_d.py\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 3 — imports verify করো\nimport sys\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\nif str(PROJECT_ROOT) not in sys.path:\n sys.path.insert(0, str(PROJECT_ROOT))\n\nfrom src.branch_d_mil import (\n DeepMILModel,\n build_bag_from_table,\n build_repertoire_table_cache,\n compute_losses,\n)\n\nprint(\"Phase 5 imports OK\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 4 — Phase 5 run করো\n!python /kaggle/working/project/train_branch_d.py","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 5 — outputs inspect করো\nfrom pathlib import Path\nimport pandas as pd\nimport numpy as np\n\nOUT_DIR = Path(\"/kaggle/working/project/artifacts/phase5_branch_d\")\n\nprint(\"Files:\")\nfor p in sorted(OUT_DIR.glob(\"*\")):\n print(\"-\", p.name)\n\nprint(\"\\nbranch_d_oof.csv\")\ndisplay(pd.read_csv(OUT_DIR / \"branch_d_oof.csv\").head())\n\n# first parquet if exists\nparquets = sorted(OUT_DIR.glob(\"mil_sequence_scores_*.parquet\"))\nif parquets:\n print(f\"\\n{parquets[0].name}\")\n display(pd.read_parquet(parquets[0]).head())\nelse:\n fallbacks = sorted(OUT_DIR.glob(\"mil_sequence_scores_*_fallback.csv\"))\n if fallbacks:\n print(f\"\\n{fallbacks[0].name}\")\n display(pd.read_csv(fallbacks[0]).head())\n\nnpzs = sorted(OUT_DIR.glob(\"mil_attention_maps_*.npz\"))\nif npzs:\n z = np.load(npzs[0], allow_pickle=True)\n print(f\"\\n{npzs[0].name}\")\n print(\"keys:\", z.files)\n print(\"attention shape:\", z[\"attention\"].shape)\n print(\"mask shape:\", z[\"mask\"].shape)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Phase 6 — Explicit sequence attribution head\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 1 — attribution.yaml write করো\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nattribution_yaml = \"\"\"\nenabled: true\n\npaths:\n output_root: /kaggle/working/project/artifacts/phase6_attribution\n\ndictionary:\n max_sequences_per_file: 30000\n max_repertoires_per_class: 150\n min_freq: 0.10\n enrichment: 3.0\n top_exact: 3000\n top_clusters: 5000\n\nbag:\n top_k_abundant: 256\n random_tail: 128\n force_public_top: 32\n max_seq_len: 32\n\nmodel:\n aa_emb_dim: 32\n gene_emb_dim: 8\n conv_channels: 64\n hidden_dim: 128\n dropout: 0.20\n\ntraining:\n seeds: [42, 52, 62]\n batch_size: 16\n epochs: 8\n lr: 0.001\n weight_decay: 0.0001\n lambda_attr: 0.25\n lambda_sparse: 0.01\n\nattribution:\n ig_steps: 8\n max_loo_sequences: 128\n rank_weights:\n attention: 0.30\n loo_delta: 0.20\n grad: 0.20\n public: 0.15\n cluster: 0.15\n\"\"\"\n\n(PROJECT_ROOT / \"configs\" / \"attribution.yaml\").write_text(attribution_yaml.strip() + \"\\n\", encoding=\"utf-8\")\nprint(\"Written:\", PROJECT_ROOT / \"configs\" / \"attribution.yaml\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 2 — src/attribution.py আর train_attribution.py write করো\nfrom pathlib import Path\nfrom textwrap import dedent\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nattribution_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nfrom pathlib import Path\nfrom typing import Dict, List\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader\n\nfrom .branch_b_cluster import approx_cluster_keys\nfrom .branch_b_public import build_dataset_cache, build_fold_signal_catalog\nfrom .branch_d_mil import (\n DeepMILModel,\n RepertoireBagDataset,\n build_bag_records_from_meta,\n build_repertoire_table_cache,\n compute_losses,\n seed_torch,\n)\n\n\ndef collate_fn(batch):\n return {\n \"ID\": [x[\"ID\"] for x in batch],\n \"dataset\": [x[\"dataset\"] for x in batch],\n \"label\": torch.stack([x[\"label\"] for x in batch]),\n \"tokens\": torch.stack([x[\"tokens\"] for x in batch]),\n \"v_ids\": torch.stack([x[\"v_ids\"] for x in batch]),\n \"j_ids\": torch.stack([x[\"j_ids\"] for x in batch]),\n \"abundance\": torch.stack([x[\"abundance\"] for x in batch]),\n \"lengths\": torch.stack([x[\"lengths\"] for x in batch]),\n \"weak_targets\": torch.stack([x[\"weak_targets\"] for x in batch]),\n \"mask\": torch.stack([x[\"mask\"] for x in batch]),\n \"seqs\": [x[\"seqs\"] for x in batch],\n }\n\n\ndef move_to_device(batch, device):\n out = {}\n for k, v in batch.items():\n if torch.is_tensor(v):\n out[k] = v.to(device)\n else:\n out[k] = v\n return out\n\n\ndef train_mil_one_fold(train_records, val_records, cfg_model, cfg_train, device, seed):\n seed_torch(seed)\n\n train_ds = RepertoireBagDataset(train_records)\n val_ds = RepertoireBagDataset(val_records)\n\n train_loader = DataLoader(train_ds, batch_size=cfg_train[\"batch_size\"], shuffle=True, collate_fn=collate_fn)\n val_loader = DataLoader(val_ds, batch_size=cfg_train[\"batch_size\"], shuffle=False, collate_fn=collate_fn)\n\n model = DeepMILModel(\n aa_emb_dim=cfg_model[\"aa_emb_dim\"],\n gene_emb_dim=cfg_model[\"gene_emb_dim\"],\n conv_channels=cfg_model[\"conv_channels\"],\n hidden_dim=cfg_model[\"hidden_dim\"],\n dropout=cfg_model[\"dropout\"],\n ).to(device)\n\n optimizer = torch.optim.AdamW(\n model.parameters(),\n lr=cfg_train[\"lr\"],\n weight_decay=cfg_train[\"weight_decay\"],\n )\n\n best_state = None\n best_auc = -np.inf\n best_epoch = -1\n\n for epoch in range(cfg_train[\"epochs\"]):\n model.train()\n for batch in train_loader:\n batch_dev = move_to_device(batch, device)\n outputs = model(\n tokens=batch_dev[\"tokens\"],\n v_ids=batch_dev[\"v_ids\"],\n j_ids=batch_dev[\"j_ids\"],\n abundance=batch_dev[\"abundance\"],\n lengths=batch_dev[\"lengths\"],\n mask=batch_dev[\"mask\"],\n )\n losses = compute_losses(\n outputs=outputs,\n labels=batch_dev[\"label\"],\n weak_targets=batch_dev[\"weak_targets\"],\n mask=batch_dev[\"mask\"],\n lambda_attr=cfg_train[\"lambda_attr\"],\n lambda_sparse=cfg_train[\"lambda_sparse\"],\n )\n\n optimizer.zero_grad()\n losses[\"total\"].backward()\n torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=5.0)\n optimizer.step()\n\n val_auc, _ = evaluate_bag_auc(model, val_loader, device)\n if np.isnan(val_auc):\n val_auc = -1.0\n\n if val_auc > best_auc:\n best_auc = val_auc\n best_epoch = epoch\n best_state = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()}\n\n if best_state is not None:\n model.load_state_dict(best_state)\n\n final_auc, final_probs = evaluate_bag_auc(model, val_loader, device)\n return model, final_auc, final_probs, best_epoch\n\n\ndef evaluate_bag_auc(model, loader, device):\n model.eval()\n bag_probs = []\n bag_labels = []\n\n with torch.no_grad():\n for batch in loader:\n batch_dev = move_to_device(batch, device)\n outputs = model(\n tokens=batch_dev[\"tokens\"],\n v_ids=batch_dev[\"v_ids\"],\n j_ids=batch_dev[\"j_ids\"],\n abundance=batch_dev[\"abundance\"],\n lengths=batch_dev[\"lengths\"],\n mask=batch_dev[\"mask\"],\n )\n probs = torch.sigmoid(outputs[\"bag_logit\"]).detach().cpu().numpy()\n bag_probs.extend(probs.tolist())\n bag_labels.extend(batch[\"label\"].numpy().tolist())\n\n bag_probs = np.asarray(bag_probs, dtype=float)\n bag_labels = np.asarray(bag_labels, dtype=float)\n\n if len(np.unique(bag_labels)) < 2:\n return np.nan, bag_probs\n\n from sklearn.metrics import roc_auc_score\n auc = float(roc_auc_score(bag_labels, bag_probs))\n return auc, bag_probs\n\n\ndef forward_from_seq_repr(model, seq_repr, mask):\n attn_logits = model.attn(seq_repr).squeeze(-1)\n attn_logits = attn_logits.masked_fill(mask <= 0, -1e9)\n attn_weights = torch.softmax(attn_logits, dim=1)\n bag_repr = torch.sum(seq_repr * attn_weights.unsqueeze(-1), dim=1)\n bag_logit = model.cls_head(bag_repr).squeeze(-1)\n return bag_logit, attn_weights\n\n\ndef integrated_gradient_scores(model, tokens, v_ids, j_ids, abundance, lengths, mask, steps=8):\n model.eval()\n\n seq_repr = model.encoder(tokens, v_ids, j_ids, abundance, lengths) # [1, N, H]\n baseline = torch.zeros_like(seq_repr)\n total_grad = torch.zeros_like(seq_repr)\n\n alphas = torch.linspace(1.0 / steps, 1.0, steps, device=seq_repr.device)\n\n for alpha in alphas:\n interp = (baseline + alpha * (seq_repr - baseline)).detach().requires_grad_(True)\n bag_logit, _ = forward_from_seq_repr(model, interp, mask)\n grads = torch.autograd.grad(bag_logit.sum(), interp, retain_graph=False, create_graph=False)[0]\n total_grad += grads\n\n ig = (seq_repr - baseline) * total_grad / float(steps)\n score = ig.abs().sum(dim=-1).squeeze(0) * mask.squeeze(0)\n return score.detach().cpu().numpy().astype(np.float32)\n\n\ndef public_cluster_scores_for_sequences(seqs: List[str], bundle: dict):\n exact_catalog = bundle.get(\"exact_catalog\", {})\n cluster_catalog = bundle.get(\"cluster_catalog\", {})\n\n public_scores = []\n cluster_scores = []\n\n for seq in seqs:\n ps = 0.0\n cs = 0.0\n\n if seq in exact_catalog:\n ps = float(exact_catalog[seq][\"score\"])\n\n for key in approx_cluster_keys(seq):\n if key in cluster_catalog:\n cs = max(cs, float(cluster_catalog[key][\"score\"]))\n\n public_scores.append(ps)\n cluster_scores.append(cs)\n\n return np.asarray(public_scores, dtype=np.float32), np.asarray(cluster_scores, dtype=np.float32)\n\n\ndef leave_one_out_scores(model, tokens, v_ids, j_ids, abundance, lengths, mask, attention_scores, weak_targets, max_loo_sequences=128):\n model.eval()\n\n with torch.no_grad():\n base_out = model(\n tokens=tokens,\n v_ids=v_ids,\n j_ids=j_ids,\n abundance=abundance,\n lengths=lengths,\n mask=mask,\n )\n base_prob = torch.sigmoid(base_out[\"bag_logit\"]).detach().cpu().item()\n\n valid_idx = np.where(mask.squeeze(0).detach().cpu().numpy() > 0)[0].tolist()\n if len(valid_idx) == 0:\n return np.zeros(mask.shape[1], dtype=np.float32)\n\n att = np.asarray(attention_scores, dtype=float)\n weak = np.asarray(weak_targets, dtype=float)\n priority = att + weak\n cand = sorted(valid_idx, key=lambda i: -priority[i])[:max_loo_sequences]\n\n loo = np.zeros(mask.shape[1], dtype=np.float32)\n\n with torch.no_grad():\n for j in cand:\n mod_mask = mask.clone()\n mod_mask[:, j] = 0.0\n out = model(\n tokens=tokens,\n v_ids=v_ids,\n j_ids=j_ids,\n abundance=abundance,\n lengths=lengths,\n mask=mod_mask,\n )\n p = torch.sigmoid(out[\"bag_logit\"]).detach().cpu().item()\n loo[j] = max(0.0, float(base_prob - p))\n\n return loo.astype(np.float32)\n\n\ndef rank_normalize(scores: np.ndarray, mask: np.ndarray) -> np.ndarray:\n scores = np.asarray(scores, dtype=float)\n mask = np.asarray(mask, dtype=float)\n out = np.zeros_like(scores, dtype=np.float32)\n\n valid = np.where(mask > 0)[0]\n if len(valid) == 0:\n return out\n\n vals = scores[valid]\n order = np.argsort(-vals, kind=\"mergesort\")\n if len(valid) == 1:\n out[valid[order[0]]] = 1.0\n return out\n\n for rank_pos, idx_pos in enumerate(order):\n score_norm = 1.0 - (rank_pos / (len(valid) - 1))\n out[valid[idx_pos]] = float(score_norm)\n return out\n\n\ndef fuse_rank_scores(attention_score, loo_delta, grad_score, public_score, cluster_score, mask, rank_weights):\n att_rank = rank_normalize(attention_score, mask)\n loo_rank = rank_normalize(loo_delta, mask)\n grad_rank = rank_normalize(grad_score, mask)\n pub_rank = rank_normalize(public_score, mask)\n clu_rank = rank_normalize(cluster_score, mask)\n\n final = (\n rank_weights[\"attention\"] * att_rank +\n rank_weights[\"loo_delta\"] * loo_rank +\n rank_weights[\"grad\"] * grad_rank +\n rank_weights[\"public\"] * pub_rank +\n rank_weights[\"cluster\"] * clu_rank\n ) * mask\n\n return {\n \"attention_rank\": att_rank.astype(np.float32),\n \"loo_rank\": loo_rank.astype(np.float32),\n \"grad_rank\": grad_rank.astype(np.float32),\n \"public_rank\": pub_rank.astype(np.float32),\n \"cluster_rank\": clu_rank.astype(np.float32),\n \"final_rank_score\": final.astype(np.float32),\n }\n\n\ndef attribute_one_record(model, record: dict, bundle: dict, device, ig_steps=8, max_loo_sequences=128, rank_weights=None):\n rank_weights = rank_weights or {\n \"attention\": 0.30,\n \"loo_delta\": 0.20,\n \"grad\": 0.20,\n \"public\": 0.15,\n \"cluster\": 0.15,\n }\n\n tokens = torch.tensor(record[\"tokens\"], dtype=torch.long, device=device).unsqueeze(0)\n v_ids = torch.tensor(record[\"v_ids\"], dtype=torch.long, device=device).unsqueeze(0)\n j_ids = torch.tensor(record[\"j_ids\"], dtype=torch.long, device=device).unsqueeze(0)\n abundance = torch.tensor(record[\"abundance\"], dtype=torch.float32, device=device).unsqueeze(0)\n lengths = torch.tensor(record[\"lengths\"], dtype=torch.float32, device=device).unsqueeze(0)\n weak_targets = np.asarray(record[\"weak_targets\"], dtype=np.float32)\n mask = torch.tensor(record[\"mask\"], dtype=torch.float32, device=device).unsqueeze(0)\n\n model.eval()\n with torch.no_grad():\n out = model(\n tokens=tokens,\n v_ids=v_ids,\n j_ids=j_ids,\n abundance=abundance,\n lengths=lengths,\n mask=mask,\n )\n bag_prob = torch.sigmoid(out[\"bag_logit\"]).detach().cpu().item()\n attention = out[\"attn_weights\"].detach().cpu().numpy().squeeze(0).astype(np.float32)\n\n grad_score = integrated_gradient_scores(\n model=model,\n tokens=tokens,\n v_ids=v_ids,\n j_ids=j_ids,\n abundance=abundance,\n lengths=lengths,\n mask=mask,\n steps=ig_steps,\n )\n\n loo_delta = leave_one_out_scores(\n model=model,\n tokens=tokens,\n v_ids=v_ids,\n j_ids=j_ids,\n abundance=abundance,\n lengths=lengths,\n mask=mask,\n attention_scores=attention,\n weak_targets=weak_targets,\n max_loo_sequences=max_loo_sequences,\n )\n\n public_score, cluster_score = public_cluster_scores_for_sequences(record[\"seqs\"], bundle)\n mask_np = np.asarray(record[\"mask\"], dtype=np.float32)\n\n fused = fuse_rank_scores(\n attention_score=attention,\n loo_delta=loo_delta,\n grad_score=grad_score,\n public_score=public_score,\n cluster_score=cluster_score,\n mask=mask_np,\n rank_weights=rank_weights,\n )\n\n valid_idx = np.where(mask_np > 0)[0].tolist()\n rows = []\n for j in valid_idx:\n rows.append({\n \"ID\": record[\"ID\"],\n \"dataset\": record[\"dataset\"],\n \"label_positive\": int(record[\"label_positive\"]),\n \"sequence\": record[\"seqs\"][j],\n \"attention_score\": float(attention[j]),\n \"loo_delta\": float(loo_delta[j]),\n \"grad_score\": float(grad_score[j]),\n \"public_score\": float(public_score[j]),\n \"cluster_score\": float(cluster_score[j]),\n \"attention_rank\": float(fused[\"attention_rank\"][j]),\n \"loo_rank\": float(fused[\"loo_rank\"][j]),\n \"grad_rank\": float(fused[\"grad_rank\"][j]),\n \"public_rank\": float(fused[\"public_rank\"][j]),\n \"cluster_rank\": float(fused[\"cluster_rank\"][j]),\n \"rank_score\": float(fused[\"final_rank_score\"][j]),\n \"bag_probability\": float(bag_prob),\n })\n\n rows = sorted(rows, key=lambda x: (-x[\"rank_score\"], -x[\"attention_score\"], -x[\"public_score\"]))\n for k, row in enumerate(rows, start=1):\n row[\"rank\"] = int(k)\n\n return rows, {\n \"ID\": record[\"ID\"],\n \"attention\": attention.astype(np.float32),\n \"loo_delta\": loo_delta.astype(np.float32),\n \"grad_score\": grad_score.astype(np.float32),\n \"public_score\": public_score.astype(np.float32),\n \"cluster_score\": cluster_score.astype(np.float32),\n \"mask\": mask_np.astype(np.float32),\n \"final_rank_score\": fused[\"final_rank_score\"].astype(np.float32),\n }\n\"\"\")\n\ntrain_attribution_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nimport sys\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport yaml\n\nPROJECT_ROOT = Path(__file__).resolve().parent\nif str(PROJECT_ROOT) not in sys.path:\n sys.path.insert(0, str(PROJECT_ROOT))\n\nfrom src.attribution import (\n attribute_one_record,\n build_dataset_cache,\n build_fold_signal_catalog,\n build_repertoire_table_cache,\n build_bag_records_from_meta,\n train_mil_one_fold,\n)\nfrom src.utils import ensure_dir, seed_everything, dataset_id_from_name\n\n\ndef safe_auc(y_true, y_prob):\n from sklearn.metrics import roc_auc_score\n y_true = np.asarray(y_true)\n y_prob = np.asarray(y_prob)\n if len(np.unique(y_true)) < 2:\n return np.nan\n return float(roc_auc_score(y_true, y_prob))\n\n\ndef main():\n cfg_base = yaml.safe_load((PROJECT_ROOT / \"configs\" / \"base.yaml\").read_text())\n cfg_branch = yaml.safe_load((PROJECT_ROOT / \"configs\" / \"attribution.yaml\").read_text())\n\n runtime = cfg_base[\"runtime\"]\n phase1_paths = cfg_base[\"paths\"]\n out_dir = ensure_dir(cfg_branch[\"paths\"][\"output_root\"])\n\n seed_everything(runtime[\"random_state\"])\n device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n canonical = pd.read_csv(Path(phase1_paths[\"output_root\"]) / \"canonical_metadata.csv\")\n all_folds = pd.read_csv(Path(phase1_paths[\"output_root\"]) / \"all_folds.csv\")\n train_root = Path(phase1_paths[\"train_root\"])\n\n train_meta_all = canonical[canonical[\"source\"] == \"train\"].copy()\n train_meta_all = train_meta_all[pd.notna(train_meta_all[\"label_positive\"])].copy()\n\n dataset_names = sorted(train_meta_all[\"dataset_name\"].unique().tolist())\n seeds = list(cfg_branch[\"training\"][\"seeds\"])\n\n cfg_dict = cfg_branch[\"dictionary\"]\n cfg_bag = cfg_branch[\"bag\"]\n cfg_model = cfg_branch[\"model\"]\n cfg_train = cfg_branch[\"training\"]\n cfg_attr = cfg_branch[\"attribution\"]\n\n all_oof = []\n all_summary = []\n\n print(\"=\" * 80)\n print(\"PHASE 6: Explicit sequence attribution head\")\n print(\"=\" * 80)\n print(\"Device:\", device)\n print(\"Datasets:\", dataset_names)\n\n for ds_name in dataset_names:\n ds_id = dataset_id_from_name(ds_name)\n ds_path = train_root / ds_name\n\n print(f\"\\\\n{'-' * 80}\")\n print(f\"Attribution training: {ds_name} (id={ds_id})\")\n print(f\"{'-' * 80}\")\n\n meta_ds = train_meta_all[train_meta_all[\"dataset_name\"] == ds_name].copy()\n fold_ds = all_folds[all_folds[\"dataset_name\"] == ds_name].copy()\n\n meta_ds = meta_ds.merge(\n fold_ds[[\"dataset_name\", \"repertoire_id\", \"fold\", \"splitter\", \"n_splits_used\"]],\n on=[\"dataset_name\", \"repertoire_id\"],\n how=\"inner\",\n ).reset_index(drop=True)\n\n signal_cache = build_dataset_cache(\n meta_ds=meta_ds,\n ds_path=ds_path,\n max_sequences_per_file=cfg_dict[\"max_sequences_per_file\"],\n random_state=runtime[\"random_state\"],\n n_jobs=runtime[\"n_jobs\"],\n )\n\n table_cache = build_repertoire_table_cache(\n meta_ds=meta_ds,\n ds_path=ds_path,\n max_sequences_per_file=cfg_dict[\"max_sequences_per_file\"],\n random_state=runtime[\"random_state\"],\n )\n\n unique_folds = sorted(meta_ds[\"fold\"].dropna().unique().tolist())\n ds_seed_frames = []\n ds_summary_rows = []\n ds_ranked_rows = []\n ds_map_rows = []\n\n for seed in seeds:\n pred_store = {\n \"ID\": meta_ds[\"repertoire_id\"].astype(str).tolist(),\n \"dataset\": meta_ds[\"dataset_name\"].astype(str).tolist(),\n \"label_positive\": meta_ds[\"label_positive\"].astype(int).tolist(),\n \"fold\": meta_ds[\"fold\"].tolist(),\n \"seed\": [seed] * len(meta_ds),\n \"MIL_ATTR_PROB\": [np.nan] * len(meta_ds),\n }\n\n for fold in unique_folds:\n tr_meta = meta_ds[meta_ds[\"fold\"] != fold].copy().reset_index(drop=True)\n va_meta = meta_ds[meta_ds[\"fold\"] == fold].copy().reset_index(drop=True)\n va_global_idx = meta_ds.index[meta_ds[\"fold\"] == fold].to_numpy()\n\n bundle, _, _ = build_fold_signal_catalog(\n fold_train_meta=tr_meta,\n cache=signal_cache,\n min_freq=cfg_dict[\"min_freq\"],\n enrichment=cfg_dict[\"enrichment\"],\n top_exact=cfg_dict[\"top_exact\"],\n top_clusters=cfg_dict[\"top_clusters\"],\n max_repertoires_per_class=cfg_dict[\"max_repertoires_per_class\"],\n random_state=seed,\n )\n\n train_records = build_bag_records_from_meta(\n meta_df=tr_meta,\n table_cache=table_cache,\n bundle=bundle,\n bag_cfg=cfg_bag,\n random_state=seed,\n )\n val_records = build_bag_records_from_meta(\n meta_df=va_meta,\n table_cache=table_cache,\n bundle=bundle,\n bag_cfg=cfg_bag,\n random_state=seed + 1000,\n )\n\n model, val_auc, val_probs, best_epoch = train_mil_one_fold(\n train_records=train_records,\n val_records=val_records,\n cfg_model=cfg_model,\n cfg_train=cfg_train,\n device=device,\n seed=seed,\n )\n\n for idx, p in zip(va_global_idx, val_probs):\n pred_store[\"MIL_ATTR_PROB\"][idx] = float(p)\n\n ds_summary_rows.append({\n \"dataset\": ds_name,\n \"seed\": seed,\n \"fold\": fold,\n \"model\": \"MIL_ATTR\",\n \"auc\": val_auc,\n \"best_epoch\": best_epoch,\n })\n\n for rec in val_records:\n rows, maps = attribute_one_record(\n model=model,\n record=rec,\n bundle=bundle,\n device=device,\n ig_steps=cfg_attr[\"ig_steps\"],\n max_loo_sequences=cfg_attr[\"max_loo_sequences\"],\n rank_weights=cfg_attr[\"rank_weights\"],\n )\n for row in rows:\n row[\"seed\"] = seed\n row[\"fold\"] = fold\n ds_ranked_rows.extend(rows)\n\n maps[\"seed\"] = seed\n maps[\"fold\"] = fold\n ds_map_rows.append(maps)\n\n seed_df = pd.DataFrame(pred_store)\n ds_seed_frames.append(seed_df)\n\n ds_summary_rows.append({\n \"dataset\": ds_name,\n \"seed\": seed,\n \"fold\": -1,\n \"model\": \"MIL_ATTR\",\n \"auc\": safe_auc(seed_df[\"label_positive\"].values, seed_df[\"MIL_ATTR_PROB\"].values),\n \"best_epoch\": -1,\n })\n\n ds_oof = pd.concat(ds_seed_frames, ignore_index=True)\n ds_oof.to_csv(out_dir / f\"phase6_attribution_oof_{ds_name}.csv\", index=False)\n all_oof.append(ds_oof)\n\n ranked_df = pd.DataFrame(ds_ranked_rows)\n if len(ranked_df):\n ranked_df = ranked_df.sort_values([\"ID\", \"seed\", \"fold\", \"rank\"], ascending=[True, True, True, True]).reset_index(drop=True)\n try:\n ranked_df.to_parquet(out_dir / f\"ranked_sequence_attribution_{ds_name}.parquet\", index=False)\n except Exception:\n ranked_df.to_csv(out_dir / f\"ranked_sequence_attribution_{ds_name}_fallback.csv\", index=False)\n\n if len(ds_map_rows):\n np.savez_compressed(\n out_dir / f\"attribution_maps_{ds_name}.npz\",\n repertoire_ids=np.array([x[\"ID\"] for x in ds_map_rows], dtype=object),\n seeds=np.array([x[\"seed\"] for x in ds_map_rows], dtype=np.int64),\n folds=np.array([x[\"fold\"] for x in ds_map_rows], dtype=np.int64),\n attention=np.vstack([x[\"attention\"] for x in ds_map_rows]).astype(np.float32),\n loo_delta=np.vstack([x[\"loo_delta\"] for x in ds_map_rows]).astype(np.float32),\n grad_score=np.vstack([x[\"grad_score\"] for x in ds_map_rows]).astype(np.float32),\n public_score=np.vstack([x[\"public_score\"] for x in ds_map_rows]).astype(np.float32),\n cluster_score=np.vstack([x[\"cluster_score\"] for x in ds_map_rows]).astype(np.float32),\n final_rank_score=np.vstack([x[\"final_rank_score\"] for x in ds_map_rows]).astype(np.float32),\n mask=np.vstack([x[\"mask\"] for x in ds_map_rows]).astype(np.float32),\n )\n\n summary_df = pd.DataFrame(ds_summary_rows)\n if len(summary_df):\n summary_df.to_csv(out_dir / f\"phase6_attribution_summary_{ds_name}.csv\", index=False)\n\n overall = (\n summary_df[summary_df[\"fold\"] >= 0]\n .groupby([\"dataset\", \"seed\", \"model\"], as_index=False)\n .agg(mean_auc=(\"auc\", \"mean\"), std_auc=(\"auc\", \"std\"), max_auc=(\"auc\", \"max\"))\n .sort_values([\"dataset\", \"seed\", \"mean_auc\"], ascending=[True, True, False])\n )\n print(\"\\\\nSummary:\")\n print(overall.to_string(index=False))\n\n print(f\"Saved ranked attribution -> {out_dir / f'ranked_sequence_attribution_{ds_name}.parquet'}\")\n print(f\"Saved attribution maps -> {out_dir / f'attribution_maps_{ds_name}.npz'}\")\n\n final_oof = pd.concat(all_oof, ignore_index=True) if len(all_oof) else pd.DataFrame()\n final_oof.to_csv(out_dir / \"phase6_attribution_oof.csv\", index=False)\n\n print(\"\\\\n\" + \"=\" * 80)\n print(\"PHASE 6 DONE\")\n print(\"Saved:\")\n print(out_dir / \"phase6_attribution_oof.csv\")\n print(\"=\" * 80)\n\n\nif __name__ == \"__main__\":\n main()\n\"\"\")\n\n(PROJECT_ROOT / \"src\" / \"attribution.py\").write_text(attribution_py, encoding=\"utf-8\")\n(PROJECT_ROOT / \"train_attribution.py\").write_text(train_attribution_py, encoding=\"utf-8\")\n\nprint(\"Written:\")\nprint(\"-\", PROJECT_ROOT / \"src\" / \"attribution.py\")\nprint(\"-\", PROJECT_ROOT / \"train_attribution.py\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 3 — imports verify করো\nimport sys\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\nif str(PROJECT_ROOT) not in sys.path:\n sys.path.insert(0, str(PROJECT_ROOT))\n\nfrom src.attribution import (\n attribute_one_record,\n integrated_gradient_scores,\n leave_one_out_scores,\n fuse_rank_scores,\n)\n\nprint(\"Phase 6 imports OK\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 4 — Phase 6 run করো\n!python /kaggle/working/project/train_attribution.py","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 5 — outputs inspect করো\nfrom pathlib import Path\nimport pandas as pd\nimport numpy as np\n\nOUT_DIR = Path(\"/kaggle/working/project/artifacts/phase6_attribution\")\n\nprint(\"Files:\")\nfor p in sorted(OUT_DIR.glob(\"*\")):\n print(\"-\", p.name)\n\nprint(\"\\nphase6_attribution_oof.csv\")\ndisplay(pd.read_csv(OUT_DIR / \"phase6_attribution_oof.csv\").head())\n\nparquets = sorted(OUT_DIR.glob(\"ranked_sequence_attribution_*.parquet\"))\nif parquets:\n print(f\"\\n{parquets[0].name}\")\n display(pd.read_parquet(parquets[0]).head(20))\nelse:\n fallbacks = sorted(OUT_DIR.glob(\"ranked_sequence_attribution_*_fallback.csv\"))\n if fallbacks:\n print(f\"\\n{fallbacks[0].name}\")\n display(pd.read_csv(fallbacks[0]).head(20))\n\nnpzs = sorted(OUT_DIR.glob(\"attribution_maps_*.npz\"))\nif npzs:\n z = np.load(npzs[0], allow_pickle=True)\n print(f\"\\n{npzs[0].name}\")\n print(\"keys:\", z.files)\n print(\"attention shape:\", z[\"attention\"].shape)\n print(\"final_rank_score shape:\", z[\"final_rank_score\"].shape)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Phase 7 — Final fusion layer","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 1 — Phase 7 config file write করো\n# Cell 1: Phase 7 fusion config file write\n\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nfusion_yaml = \"\"\"\nenabled: true\n\npaths:\n output_root: /kaggle/working/project/artifacts/phase7_fusion\n branch_a_oof: /kaggle/working/project/artifacts/phase2_branch_a/branch_a_oof.csv\n branch_b_oof: /kaggle/working/project/artifacts/phase3_branch_b/branch_b_oof.csv\n branch_c_oof: /kaggle/working/project/artifacts/phase4_branch_c/branch_c_oof.csv\n branch_d_oof: /kaggle/working/project/artifacts/phase5_branch_d/branch_d_oof.csv\n\ntraining:\n random_state: 42\n\"\"\"\n\n(PROJECT_ROOT / \"configs\" / \"fusion.yaml\").write_text(fusion_yaml.strip() + \"\\n\", encoding=\"utf-8\")\nprint(\"Written:\", PROJECT_ROOT / \"configs\" / \"fusion.yaml\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 2 — fusion code files write করো\n# Cell 2: src/fusion.py and train_fusion.py write\n\nfrom pathlib import Path\nfrom textwrap import dedent\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nfusion_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nfrom pathlib import Path\nfrom typing import Dict, List, Tuple\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.linear_model import LogisticRegression\n\n\nMETA_COLS = {\"ID\", \"dataset\", \"label_positive\", \"fold\", \"seed\"}\n\n\ndef safe_auc(y_true, y_prob):\n from sklearn.metrics import roc_auc_score\n\n y_true = np.asarray(y_true)\n y_prob = np.asarray(y_prob)\n\n valid = ~pd.isna(y_true) & ~pd.isna(y_prob)\n y_true = y_true[valid]\n y_prob = y_prob[valid]\n\n if len(y_true) == 0 or len(np.unique(y_true)) < 2:\n return np.nan\n return float(roc_auc_score(y_true, y_prob))\n\n\ndef rank_normalize_array(x: np.ndarray) -> np.ndarray:\n x = np.asarray(x, dtype=float)\n out = np.zeros_like(x, dtype=np.float32)\n\n if len(x) == 0:\n return out\n\n valid = ~np.isnan(x)\n idx = np.where(valid)[0]\n if len(idx) == 0:\n return out\n if len(idx) == 1:\n out[idx[0]] = 1.0\n return out\n\n vals = x[idx]\n order = np.argsort(-vals, kind=\"mergesort\")\n for rank_pos, pos in enumerate(order):\n out[idx[pos]] = float(1.0 - rank_pos / (len(idx) - 1))\n return out.astype(np.float32)\n\n\ndef rank_normalize_by_dataset(df: pd.DataFrame, pred_cols: List[str]) -> pd.DataFrame:\n out = df.copy()\n for ds_name, g in out.groupby(\"dataset\"):\n idx = g.index\n for c in pred_cols:\n out.loc[idx, c] = rank_normalize_array(g[c].values)\n return out\n\n\ndef _pick_branch_pred_column(df: pd.DataFrame) -> str:\n preferred = [\"AVG\", \"MIL_PROB\", \"MIL_ATTR_PROB\"]\n for c in preferred:\n if c in df.columns:\n return c\n\n candidate_cols = [c for c in df.columns if c not in META_COLS]\n numeric_cols = [c for c in candidate_cols if pd.api.types.is_numeric_dtype(df[c])]\n if len(numeric_cols) == 0:\n raise ValueError(\"No prediction column found in OOF file.\")\n return numeric_cols[-1]\n\n\ndef load_branch_oof(path: str | Path, branch_name: str) -> pd.DataFrame:\n path = Path(path)\n if not path.exists():\n return pd.DataFrame(columns=[\"ID\", \"dataset\", \"label_positive\", \"fold\", f\"{branch_name}_pred\"])\n\n df = pd.read_csv(path)\n\n required = [\"ID\", \"dataset\"]\n for c in required:\n if c not in df.columns:\n raise ValueError(f\"{path} missing required column: {c}\")\n\n pred_col = _pick_branch_pred_column(df)\n\n keep_cols = [\"ID\", \"dataset\"]\n if \"label_positive\" in df.columns:\n keep_cols.append(\"label_positive\")\n if \"fold\" in df.columns:\n keep_cols.append(\"fold\")\n keep_cols.append(pred_col)\n\n x = df[keep_cols].copy()\n x = x.rename(columns={pred_col: f\"{branch_name}_pred\"})\n\n # aggregate across seeds if duplicated\n group_cols = [\"ID\", \"dataset\"]\n agg_map = {f\"{branch_name}_pred\": \"mean\"}\n\n if \"label_positive\" in x.columns:\n group_cols.append(\"label_positive\")\n if \"fold\" in x.columns:\n group_cols.append(\"fold\")\n\n x = x.groupby(group_cols, as_index=False).agg(agg_map)\n\n if \"label_positive\" not in x.columns:\n x[\"label_positive\"] = np.nan\n if \"fold\" not in x.columns:\n x[\"fold\"] = np.nan\n\n return x\n\n\ndef merge_branch_oofs(branch_frames: Dict[str, pd.DataFrame]) -> pd.DataFrame:\n valid_items = [(k, v.copy()) for k, v in branch_frames.items() if len(v) > 0]\n if len(valid_items) == 0:\n return pd.DataFrame(columns=[\"ID\", \"dataset\", \"label_positive\", \"fold\"])\n\n base_name, base_df = valid_items[0]\n merged = base_df.copy()\n\n for name, df in valid_items[1:]:\n pred_col = f\"{name}_pred\"\n use_cols = [\"ID\", \"dataset\", pred_col]\n if pred_col not in df.columns:\n continue\n merged = merged.merge(df[use_cols], on=[\"ID\", \"dataset\"], how=\"outer\")\n\n if \"label_positive\" in df.columns and \"label_positive\" in merged.columns:\n lab_map = df[[\"ID\", \"dataset\", \"label_positive\"]].drop_duplicates()\n merged = merged.merge(\n lab_map.rename(columns={\"label_positive\": \"label_positive_tmp\"}),\n on=[\"ID\", \"dataset\"],\n how=\"left\",\n )\n merged[\"label_positive\"] = merged[\"label_positive\"].fillna(merged[\"label_positive_tmp\"])\n merged = merged.drop(columns=[\"label_positive_tmp\"])\n\n if \"fold\" in df.columns and \"fold\" in merged.columns:\n fold_map = df[[\"ID\", \"dataset\", \"fold\"]].drop_duplicates()\n merged = merged.merge(\n fold_map.rename(columns={\"fold\": \"fold_tmp\"}),\n on=[\"ID\", \"dataset\"],\n how=\"left\",\n )\n merged[\"fold\"] = merged[\"fold\"].fillna(merged[\"fold_tmp\"])\n merged = merged.drop(columns=[\"fold_tmp\"])\n\n if \"label_positive\" not in merged.columns:\n merged[\"label_positive\"] = np.nan\n if \"fold\" not in merged.columns:\n merged[\"fold\"] = np.nan\n\n pred_cols = [c for c in merged.columns if c.endswith(\"_pred\")]\n\n for c in pred_cols:\n if merged[c].isna().all():\n merged[c] = 0.5\n else:\n merged[c] = merged[c].fillna(merged[c].mean())\n\n merged = merged.sort_values([\"dataset\", \"ID\"]).reset_index(drop=True)\n return merged\n\n\ndef simple_mean_pred(df: pd.DataFrame, pred_cols: List[str]) -> np.ndarray:\n return df[pred_cols].mean(axis=1).values.astype(float)\n\n\ndef rank_mean_pred(df: pd.DataFrame, pred_cols: List[str]) -> np.ndarray:\n tmp = rank_normalize_by_dataset(df[[\"dataset\"] + pred_cols].copy(), pred_cols)\n return tmp[pred_cols].mean(axis=1).values.astype(float)\n\n\ndef fit_nonnegative_weights(X: np.ndarray, y: np.ndarray, random_state: int = 42) -> np.ndarray:\n X = np.asarray(X, dtype=float)\n y = np.asarray(y, dtype=int)\n\n valid = ~np.isnan(X).any(axis=1) & ~np.isnan(y)\n X = X[valid]\n y = y[valid]\n\n if len(X) == 0 or X.shape[1] == 0 or len(np.unique(y)) < 2:\n w = np.ones(X.shape[1], dtype=float) / max(1, X.shape[1])\n return w\n\n clf = LogisticRegression(max_iter=4000, random_state=random_state)\n clf.fit(X, y)\n\n coef = np.clip(np.asarray(clf.coef_[0], dtype=float), 0, None)\n if coef.sum() <= 0:\n coef = np.ones_like(coef, dtype=float)\n coef = coef / coef.sum()\n return coef.astype(float)\n\n\ndef global_stack_oof(df: pd.DataFrame, pred_cols: List[str], random_state: int = 42) -> Tuple[np.ndarray, np.ndarray]:\n out = np.zeros(len(df), dtype=float)\n weights_per_fold = []\n\n usable = df.dropna(subset=[\"label_positive\"]).copy()\n usable[\"label_positive\"] = usable[\"label_positive\"].astype(int)\n\n folds = sorted([f for f in usable[\"fold\"].dropna().unique().tolist()])\n if len(folds) == 0:\n w = fit_nonnegative_weights(usable[pred_cols].values, usable[\"label_positive\"].values, random_state=random_state)\n out[:] = np.dot(df[pred_cols].values, w)\n return out, w\n\n for fold in folds:\n tr = usable[usable[\"fold\"] != fold].copy()\n va_idx = usable.index[usable[\"fold\"] == fold].to_numpy()\n\n if len(tr) == 0 or len(va_idx) == 0:\n continue\n\n w = fit_nonnegative_weights(tr[pred_cols].values, tr[\"label_positive\"].values, random_state=random_state)\n weights_per_fold.append(w)\n out[va_idx] = np.dot(df.loc[va_idx, pred_cols].values, w)\n\n if len(weights_per_fold) == 0:\n full_w = fit_nonnegative_weights(usable[pred_cols].values, usable[\"label_positive\"].values, random_state=random_state)\n out[:] = np.dot(df[pred_cols].values, full_w)\n return out, full_w\n\n full_w = fit_nonnegative_weights(usable[pred_cols].values, usable[\"label_positive\"].values, random_state=random_state)\n return out, full_w\n\n\ndef dataset_stack_oof(df: pd.DataFrame, pred_cols: List[str], random_state: int = 42) -> Tuple[np.ndarray, pd.DataFrame]:\n out = np.zeros(len(df), dtype=float)\n weight_rows = []\n\n usable = df.dropna(subset=[\"label_positive\"]).copy()\n usable[\"label_positive\"] = usable[\"label_positive\"].astype(int)\n\n for ds_name, g in usable.groupby(\"dataset\"):\n folds = sorted([f for f in g[\"fold\"].dropna().unique().tolist()])\n if len(folds) == 0:\n w = fit_nonnegative_weights(g[pred_cols].values, g[\"label_positive\"].values, random_state=random_state)\n out[g.index] = np.dot(df.loc[g.index, pred_cols].values, w)\n else:\n for fold in folds:\n tr = g[g[\"fold\"] != fold].copy()\n va_idx = g.index[g[\"fold\"] == fold].to_numpy()\n if len(tr) == 0 or len(va_idx) == 0:\n continue\n w = fit_nonnegative_weights(tr[pred_cols].values, tr[\"label_positive\"].values, random_state=random_state)\n out[va_idx] = np.dot(df.loc[va_idx, pred_cols].values, w)\n\n full_w = fit_nonnegative_weights(g[pred_cols].values, g[\"label_positive\"].values, random_state=random_state)\n for c, wv in zip(pred_cols, full_w):\n weight_rows.append({\n \"scope\": \"dataset\",\n \"dataset\": ds_name,\n \"method\": \"STACK_DATASET\",\n \"branch_feature\": c,\n \"weight\": float(wv),\n })\n\n weights_df = pd.DataFrame(weight_rows)\n return out, weights_df\n\n\ndef build_fusion_outputs(merged_df: pd.DataFrame, random_state: int = 42):\n df = merged_df.copy()\n pred_cols = [c for c in df.columns if c.endswith(\"_pred\")]\n\n if len(pred_cols) == 0:\n raise ValueError(\"No branch prediction columns found for fusion.\")\n\n df[\"SIMPLE_MEAN\"] = simple_mean_pred(df, pred_cols)\n df[\"RANK_MEAN\"] = rank_mean_pred(df, pred_cols)\n\n global_oof, global_w = global_stack_oof(df, pred_cols, random_state=random_state)\n df[\"STACK_GLOBAL\"] = global_oof\n\n dataset_oof, dataset_weights_df = dataset_stack_oof(df, pred_cols, random_state=random_state)\n df[\"STACK_DATASET\"] = dataset_oof\n\n global_weights_df = pd.DataFrame({\n \"scope\": [\"global\"] * len(pred_cols),\n \"dataset\": [\"ALL\"] * len(pred_cols),\n \"method\": [\"STACK_GLOBAL\"] * len(pred_cols),\n \"branch_feature\": pred_cols,\n \"weight\": global_w.astype(float),\n })\n\n weights_df = pd.concat([global_weights_df, dataset_weights_df], ignore_index=True)\n\n summary_rows = []\n methods = [\"SIMPLE_MEAN\", \"RANK_MEAN\", \"STACK_GLOBAL\", \"STACK_DATASET\"]\n\n usable = df.dropna(subset=[\"label_positive\"]).copy()\n usable[\"label_positive\"] = usable[\"label_positive\"].astype(int)\n\n for ds_name, g in usable.groupby(\"dataset\"):\n for m in methods:\n summary_rows.append({\n \"dataset\": ds_name,\n \"method\": m,\n \"auc\": safe_auc(g[\"label_positive\"].values, g[m].values),\n \"n_samples\": int(len(g)),\n })\n\n for m in methods:\n summary_rows.append({\n \"dataset\": \"ALL\",\n \"method\": m,\n \"auc\": safe_auc(usable[\"label_positive\"].values, usable[m].values),\n \"n_samples\": int(len(usable)),\n })\n\n summary_df = pd.DataFrame(summary_rows).sort_values([\"dataset\", \"auc\"], ascending=[True, False]).reset_index(drop=True)\n return df, weights_df, summary_df\n\"\"\")\n\ntrain_fusion_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nimport sys\nfrom pathlib import Path\n\nimport pandas as pd\nimport yaml\n\nPROJECT_ROOT = Path(__file__).resolve().parent\nif str(PROJECT_ROOT) not in sys.path:\n sys.path.insert(0, str(PROJECT_ROOT))\n\nfrom src.fusion import (\n build_fusion_outputs,\n load_branch_oof,\n merge_branch_oofs,\n)\nfrom src.utils import ensure_dir\n\n\ndef main():\n cfg = yaml.safe_load((PROJECT_ROOT / \"configs\" / \"fusion.yaml\").read_text())\n out_dir = ensure_dir(cfg[\"paths\"][\"output_root\"])\n random_state = cfg[\"training\"][\"random_state\"]\n\n branch_paths = {\n \"branch_a\": cfg[\"paths\"][\"branch_a_oof\"],\n \"branch_b\": cfg[\"paths\"][\"branch_b_oof\"],\n \"branch_c\": cfg[\"paths\"][\"branch_c_oof\"],\n \"branch_d\": cfg[\"paths\"][\"branch_d_oof\"],\n }\n\n print(\"=\" * 80)\n print(\"PHASE 7: Final fusion layer\")\n print(\"=\" * 80)\n\n branch_frames = {}\n for name, path in branch_paths.items():\n df = load_branch_oof(path, branch_name=name)\n branch_frames[name] = df\n print(f\"{name}: rows={len(df)} | path={path}\")\n\n merged = merge_branch_oofs(branch_frames)\n print(\"\\\\nMerged fusion table shape:\", merged.shape)\n print(\"Columns:\", merged.columns.tolist())\n\n fusion_oof, weights_df, summary_df = build_fusion_outputs(merged, random_state=random_state)\n\n fusion_oof.to_csv(out_dir / \"fusion_oof.csv\", index=False)\n weights_df.to_csv(out_dir / \"fusion_weights.csv\", index=False)\n summary_df.to_csv(out_dir / \"fusion_summary.csv\", index=False)\n\n print(\"\\\\nFusion summary:\")\n print(summary_df.to_string(index=False))\n\n print(\"\\\\nSaved:\")\n print(out_dir / \"fusion_oof.csv\")\n print(out_dir / \"fusion_weights.csv\")\n print(out_dir / \"fusion_summary.csv\")\n print(\"=\" * 80)\n\n\nif __name__ == \"__main__\":\n main()\n\"\"\")\n\n(PROJECT_ROOT / \"src\" / \"fusion.py\").write_text(fusion_py, encoding=\"utf-8\")\n(PROJECT_ROOT / \"train_fusion.py\").write_text(train_fusion_py, encoding=\"utf-8\")\n\nprint(\"Written:\")\nprint(\"-\", PROJECT_ROOT / \"src\" / \"fusion.py\")\nprint(\"-\", PROJECT_ROOT / \"train_fusion.py\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Cell 3 — imports verify করো\n# Cell 3: Fusion imports verify\n\nimport sys\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\nif str(PROJECT_ROOT) not in sys.path:\n sys.path.insert(0, str(PROJECT_ROOT))\n\nfrom src.fusion import (\n load_branch_oof,\n merge_branch_oofs,\n build_fusion_outputs,\n)\n\nprint(\"Phase 7 imports OK\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 4: Run Phase 7 fusion training on OOF predictions\n\n!python /kaggle/working/project/train_fusion.py","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 5: Inspect fusion outputs\n\nfrom pathlib import Path\nimport pandas as pd\n\nOUT_DIR = Path(\"/kaggle/working/project/artifacts/phase7_fusion\")\n\nprint(\"Files:\")\nfor p in sorted(OUT_DIR.glob(\"*\")):\n print(\"-\", p.name)\n\nprint(\"\\nfusion_oof.csv\")\ndisplay(pd.read_csv(OUT_DIR / \"fusion_oof.csv\").head())\n\nprint(\"\\nfusion_weights.csv\")\ndisplay(pd.read_csv(OUT_DIR / \"fusion_weights.csv\"))\n\nprint(\"\\nfusion_summary.csv\")\ndisplay(pd.read_csv(OUT_DIR / \"fusion_summary.csv\"))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Phase 8 — Benchmarking section","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 1: Phase 8 benchmark config file write\n\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nbenchmark_yaml = \"\"\"\nenabled: true\n\npaths:\n output_root: /kaggle/working/project/artifacts/phase8_benchmarks\n canonical_metadata: /kaggle/working/project/artifacts/phase1/canonical_metadata.csv\n fusion_oof: /kaggle/working/project/artifacts/phase7_fusion/fusion_oof.csv\n phase6_dir: /kaggle/working/project/artifacts/phase6_attribution\n\nreal:\n topk_list: [5, 10, 20]\n\nsimairr:\n n_pos: 120\n n_neg: 120\n n_background_per_rep: 220\n n_public_background: 180\n n_disease_public: 60\n public_common_prob: 0.18\n disease_prob_pos: 0.22\n disease_prob_neg: 0.03\n random_state: 42\n\nligo:\n n_pos: 120\n n_neg: 120\n n_background_per_rep: 240\n implanted_motifs: [\"CASSLG\", \"ASSIR\", \"GGQET\", \"QYFDT\"]\n implants_per_positive: 3\n implant_prob_positive: 1.0\n implant_prob_negative: 0.02\n recovery_k_list: [1, 5, 10]\n random_state: 42\n\ndictionary:\n max_repertoires_per_class: 150\n min_freq: 0.10\n enrichment: 3.0\n top_exact: 3000\n top_clusters: 5000\n\"\"\"\n\n(PROJECT_ROOT / \"configs\" / \"benchmark.yaml\").write_text(benchmark_yaml.strip() + \"\\n\", encoding=\"utf-8\")\nprint(\"Written:\", PROJECT_ROOT / \"configs\" / \"benchmark.yaml\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 2: Write src/simulation.py, src/evaluation.py, and train_benchmarks.py\n\nfrom pathlib import Path\nfrom textwrap import dedent\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nsimulation_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nfrom collections import Counter\nfrom pathlib import Path\nfrom typing import Dict, List, Tuple\n\nimport numpy as np\nimport pandas as pd\n\nfrom .branch_a_features import read_repertoire\nfrom .branch_b_cluster import approx_cluster_keys\nfrom .utils import stable_hash\n\n\nAA_LIST = list(\"ARNDCQEGHILKMFPSTWYV\")\n\n\ndef estimate_empirical_background(\n canonical_train_meta: pd.DataFrame,\n train_root: Path,\n max_repertoires: int = 16,\n max_sequences_per_file: int = 4000,\n random_state: int = 42,\n):\n rng = np.random.RandomState(random_state)\n if len(canonical_train_meta) == 0:\n aa_probs = np.ones(len(AA_LIST), dtype=float) / len(AA_LIST)\n lengths = np.arange(10, 21)\n return aa_probs, lengths\n\n sample_meta = canonical_train_meta.sample(\n n=min(max_repertoires, len(canonical_train_meta)),\n random_state=random_state,\n replace=False,\n )\n\n aa_counter = Counter()\n lengths = []\n\n for _, row in sample_meta.iterrows():\n ds_path = train_root / row[\"dataset_name\"]\n df = read_repertoire(ds_path / row[\"filename\"], max_seqs=max_sequences_per_file, random_state=random_state)\n if len(df) == 0:\n continue\n seqs = df[\"junction_aa\"].fillna(\"\").astype(str).tolist()\n for s in seqs:\n if not s:\n continue\n lengths.append(len(s))\n for ch in s:\n if ch in AA_LIST:\n aa_counter[ch] += 1\n\n if len(lengths) == 0:\n lengths = list(range(10, 21))\n\n aa_probs = np.array([aa_counter.get(aa, 1) for aa in AA_LIST], dtype=float)\n aa_probs = aa_probs / aa_probs.sum()\n lengths = np.asarray(lengths, dtype=int)\n return aa_probs, lengths\n\n\ndef random_seq(length: int, aa_probs: np.ndarray, rng: np.random.RandomState) -> str:\n toks = rng.choice(AA_LIST, size=int(length), replace=True, p=aa_probs)\n return \"\".join(toks.tolist())\n\n\ndef random_lengths(length_source: np.ndarray, n: int, rng: np.random.RandomState) -> np.ndarray:\n if len(length_source) == 0:\n length_source = np.arange(10, 21)\n idx = rng.choice(np.arange(len(length_source)), size=n, replace=True)\n return np.asarray(length_source[idx], dtype=int)\n\n\ndef make_public_pool(n_seq: int, aa_probs: np.ndarray, length_source: np.ndarray, rng: np.random.RandomState) -> List[str]:\n lengths = random_lengths(length_source, n_seq, rng)\n return [random_seq(int(L), aa_probs, rng) for L in lengths]\n\n\ndef build_counter_from_seq_list(seqs: List[str], weights: List[float]) -> Counter:\n c = Counter()\n for s, w in zip(seqs, weights):\n if s:\n c[s] += float(w)\n return c\n\n\ndef simulate_simairr_dataset(\n n_pos: int,\n n_neg: int,\n n_background_per_rep: int,\n n_public_background: int,\n n_disease_public: int,\n public_common_prob: float,\n disease_prob_pos: float,\n disease_prob_neg: float,\n aa_probs: np.ndarray,\n length_source: np.ndarray,\n random_state: int = 42,\n):\n rng = np.random.RandomState(random_state)\n\n public_bg_pool = make_public_pool(n_public_background, aa_probs, length_source, rng)\n disease_pool = make_public_pool(n_disease_public, aa_probs, length_source, rng)\n\n records = []\n cache = {}\n\n def make_one(rep_id: str, label: int):\n seqs = []\n weights = []\n\n # background unique-ish sequences\n bg_lengths = random_lengths(length_source, n_background_per_rep, rng)\n for L in bg_lengths:\n s = random_seq(int(L), aa_probs, rng)\n seqs.append(s)\n weights.append(float(rng.lognormal(mean=0.4, sigma=0.8)))\n\n # realistic public sharing in both classes\n for s in public_bg_pool:\n if rng.rand() < public_common_prob:\n seqs.append(s)\n weights.append(float(rng.lognormal(mean=0.8, sigma=0.5)))\n\n # disease-enriched public signatures\n p = disease_prob_pos if label == 1 else disease_prob_neg\n for s in disease_pool:\n if rng.rand() < p:\n seqs.append(s)\n weights.append(float(rng.lognormal(mean=1.0, sigma=0.6)))\n\n seq_counter = build_counter_from_seq_list(seqs, weights)\n cache[rep_id] = {\n \"seq_counter\": seq_counter,\n \"total_weight\": float(sum(seq_counter.values())),\n }\n records.append({\n \"repertoire_id\": rep_id,\n \"dataset_name\": \"simairr_synth\",\n \"label_positive\": int(label),\n })\n\n for i in range(n_pos):\n make_one(f\"simairr_pos_{i:04d}\", 1)\n for i in range(n_neg):\n make_one(f\"simairr_neg_{i:04d}\", 0)\n\n meta = pd.DataFrame(records)\n truth = {\n \"public_background_pool\": public_bg_pool,\n \"disease_pool\": disease_pool,\n }\n return meta, cache, truth\n\n\ndef implant_motif_in_sequence(seq: str, motif: str, rng: np.random.RandomState) -> str:\n if len(seq) < len(motif):\n seq = seq + motif\n return seq[: max(len(motif), len(seq))]\n\n start_max = max(0, len(seq) - len(motif))\n pos = int(rng.randint(0, start_max + 1))\n out = seq[:pos] + motif + seq[pos + len(motif):]\n return out\n\n\ndef simulate_ligo_dataset(\n n_pos: int,\n n_neg: int,\n n_background_per_rep: int,\n implanted_motifs: List[str],\n implants_per_positive: int,\n implant_prob_positive: float,\n implant_prob_negative: float,\n aa_probs: np.ndarray,\n length_source: np.ndarray,\n random_state: int = 42,\n):\n rng = np.random.RandomState(random_state)\n\n records = []\n cache = {}\n truth_rows = []\n\n def make_one(rep_id: str, label: int):\n seqs = []\n weights = []\n implanted = []\n\n bg_lengths = random_lengths(length_source, n_background_per_rep, rng)\n bg_lengths = np.maximum(bg_lengths, max(len(m) for m in implanted_motifs) + 2)\n\n bg_sequences = [random_seq(int(L), aa_probs, rng) for L in bg_lengths]\n\n # choose implants\n local_sequences = list(bg_sequences)\n n_implants = implants_per_positive if (label == 1 and rng.rand() < implant_prob_positive) else 0\n if label == 0 and rng.rand() < implant_prob_negative:\n n_implants = 1\n\n chosen_positions = rng.choice(np.arange(len(local_sequences)), size=min(n_implants, len(local_sequences)), replace=False).tolist() if n_implants > 0 else []\n\n for pos_idx in chosen_positions:\n motif = implanted_motifs[pos_idx % len(implanted_motifs)]\n s = implant_motif_in_sequence(local_sequences[pos_idx], motif, rng)\n local_sequences[pos_idx] = s\n implanted.append(s)\n\n for s in local_sequences:\n seqs.append(s)\n weights.append(float(rng.lognormal(mean=0.45, sigma=0.85)))\n\n seq_counter = build_counter_from_seq_list(seqs, weights)\n cache[rep_id] = {\n \"seq_counter\": seq_counter,\n \"total_weight\": float(sum(seq_counter.values())),\n }\n records.append({\n \"repertoire_id\": rep_id,\n \"dataset_name\": \"ligo_synth\",\n \"label_positive\": int(label),\n })\n\n for s in implanted:\n truth_rows.append({\n \"repertoire_id\": rep_id,\n \"sequence\": s,\n \"is_implanted\": 1,\n \"label_positive\": int(label),\n })\n\n for i in range(n_pos):\n make_one(f\"ligo_pos_{i:04d}\", 1)\n for i in range(n_neg):\n make_one(f\"ligo_neg_{i:04d}\", 0)\n\n meta = pd.DataFrame(records)\n truth_df = pd.DataFrame(truth_rows)\n return meta, cache, truth_df\n\n\ndef sequence_scores_from_bundle(seq_counter: Counter, bundle: dict) -> pd.DataFrame:\n exact_catalog = bundle.get(\"exact_catalog\", {})\n cluster_catalog = bundle.get(\"cluster_catalog\", {})\n\n rows = []\n for seq, cw in seq_counter.items():\n public_score = 0.0\n cluster_score = 0.0\n\n if seq in exact_catalog:\n public_score = float(exact_catalog[seq][\"score\"])\n\n for key in approx_cluster_keys(seq):\n if key in cluster_catalog:\n cluster_score = max(cluster_score, float(cluster_catalog[key][\"score\"]))\n\n total_score = public_score + cluster_score\n rows.append({\n \"sequence\": seq,\n \"weight\": float(cw),\n \"public_score\": float(public_score),\n \"cluster_score\": float(cluster_score),\n \"score\": float(total_score),\n })\n\n out = pd.DataFrame(rows)\n if len(out):\n out = out.sort_values([\"score\", \"public_score\", \"cluster_score\", \"weight\"], ascending=[False, False, False, False]).reset_index(drop=True)\n out[\"rank\"] = np.arange(1, len(out) + 1)\n return out\n\"\"\")\n\nevaluation_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nfrom pathlib import Path\nfrom typing import List\n\nimport numpy as np\nimport pandas as pd\n\n\ndef brier_score(y_true, y_prob):\n y_true = np.asarray(y_true, dtype=float)\n y_prob = np.asarray(y_prob, dtype=float)\n valid = ~np.isnan(y_true) & ~np.isnan(y_prob)\n y_true = y_true[valid]\n y_prob = y_prob[valid]\n if len(y_true) == 0:\n return np.nan\n return float(np.mean((y_prob - y_true) ** 2))\n\n\ndef expected_calibration_error(y_true, y_prob, n_bins: int = 10):\n y_true = np.asarray(y_true, dtype=float)\n y_prob = np.asarray(y_prob, dtype=float)\n valid = ~np.isnan(y_true) & ~np.isnan(y_prob)\n y_true = y_true[valid]\n y_prob = y_prob[valid]\n\n if len(y_true) == 0:\n return np.nan\n\n bins = np.linspace(0, 1, n_bins + 1)\n ece = 0.0\n n = len(y_true)\n\n for i in range(n_bins):\n lo, hi = bins[i], bins[i + 1]\n if i == n_bins - 1:\n mask = (y_prob >= lo) & (y_prob <= hi)\n else:\n mask = (y_prob >= lo) & (y_prob < hi)\n\n if mask.sum() == 0:\n continue\n\n acc = y_true[mask].mean()\n conf = y_prob[mask].mean()\n ece += (mask.sum() / n) * abs(acc - conf)\n\n return float(ece)\n\n\ndef safe_auc(y_true, y_prob):\n from sklearn.metrics import roc_auc_score\n\n y_true = np.asarray(y_true)\n y_prob = np.asarray(y_prob)\n valid = ~pd.isna(y_true) & ~pd.isna(y_prob)\n y_true = y_true[valid]\n y_prob = y_prob[valid]\n\n if len(y_true) == 0 or len(np.unique(y_true)) < 2:\n return np.nan\n return float(roc_auc_score(y_true, y_prob))\n\n\ndef load_ranked_attribution_tables(phase6_dir: Path) -> pd.DataFrame:\n parts = []\n\n parquets = sorted(phase6_dir.glob(\"ranked_sequence_attribution_*.parquet\"))\n for p in parquets:\n try:\n parts.append(pd.read_parquet(p))\n except Exception:\n pass\n\n fallbacks = sorted(phase6_dir.glob(\"ranked_sequence_attribution_*_fallback.csv\"))\n for p in fallbacks:\n try:\n parts.append(pd.read_csv(p))\n except Exception:\n pass\n\n if len(parts) == 0:\n return pd.DataFrame()\n return pd.concat(parts, ignore_index=True)\n\n\ndef real_benchmark_table(\n fusion_oof: pd.DataFrame,\n canonical_meta: pd.DataFrame,\n ranked_attr_df: pd.DataFrame,\n topk_list: List[int],\n):\n rows = []\n\n meta_train = canonical_meta[canonical_meta[\"source\"] == \"train\"].copy()\n meta_train = meta_train.rename(columns={\"dataset_name\": \"dataset\"})\n keep_meta = [c for c in [\"dataset\", \"repertoire_id\", \"sex\", \"race\", \"age\"] if c in meta_train.columns]\n if \"repertoire_id\" in keep_meta:\n meta_use = meta_train[keep_meta].copy().rename(columns={\"repertoire_id\": \"ID\"})\n else:\n meta_use = pd.DataFrame(columns=[\"ID\", \"dataset\"])\n\n fx = fusion_oof.merge(meta_use, on=[\"ID\", \"dataset\"], how=\"left\")\n\n method_cols = [c for c in [\"SIMPLE_MEAN\", \"RANK_MEAN\", \"STACK_GLOBAL\", \"STACK_DATASET\"] if c in fx.columns]\n\n for ds_name, g in fx.groupby(\"dataset\"):\n y = g[\"label_positive\"].values\n for m in method_cols:\n rows.append({\n \"benchmark\": \"real\",\n \"scope\": \"dataset\",\n \"dataset\": ds_name,\n \"metric\": \"auc\",\n \"method\": m,\n \"value\": safe_auc(y, g[m].values),\n })\n rows.append({\n \"benchmark\": \"real\",\n \"scope\": \"dataset\",\n \"dataset\": ds_name,\n \"metric\": \"brier\",\n \"method\": m,\n \"value\": brier_score(y, g[m].values),\n })\n rows.append({\n \"benchmark\": \"real\",\n \"scope\": \"dataset\",\n \"dataset\": ds_name,\n \"metric\": \"ece\",\n \"method\": m,\n \"value\": expected_calibration_error(y, g[m].values),\n })\n\n y_all = fx[\"label_positive\"].values\n for m in method_cols:\n rows.append({\n \"benchmark\": \"real\",\n \"scope\": \"global\",\n \"dataset\": \"ALL\",\n \"metric\": \"auc\",\n \"method\": m,\n \"value\": safe_auc(y_all, fx[m].values),\n })\n rows.append({\n \"benchmark\": \"real\",\n \"scope\": \"global\",\n \"dataset\": \"ALL\",\n \"metric\": \"brier\",\n \"method\": m,\n \"value\": brier_score(y_all, fx[m].values),\n })\n rows.append({\n \"benchmark\": \"real\",\n \"scope\": \"global\",\n \"dataset\": \"ALL\",\n \"metric\": \"ece\",\n \"method\": m,\n \"value\": expected_calibration_error(y_all, fx[m].values),\n })\n\n # subgroup performance if metadata exists\n for subgroup_col in [\"sex\", \"race\"]:\n if subgroup_col not in fx.columns:\n continue\n tmp = fx[pd.notna(fx[subgroup_col])].copy()\n if len(tmp) == 0:\n continue\n for subgroup_val, g in tmp.groupby(subgroup_col):\n if len(g) < 12:\n continue\n for m in method_cols:\n rows.append({\n \"benchmark\": \"real\",\n \"scope\": f\"subgroup_{subgroup_col}\",\n \"dataset\": str(subgroup_val),\n \"metric\": \"auc\",\n \"method\": m,\n \"value\": safe_auc(g[\"label_positive\"].values, g[m].values),\n })\n\n # ranking proxy from phase6 attribution outputs\n if len(ranked_attr_df):\n if \"weak_target\" in ranked_attr_df.columns:\n z = ranked_attr_df.copy()\n z[\"weak_positive\"] = (pd.to_numeric(z[\"weak_target\"], errors=\"coerce\").fillna(0.0) > 0).astype(int)\n for ds_name, g in z.groupby(\"dataset\"):\n for k in topk_list:\n topk = g[g[\"rank\"] <= k].copy()\n if len(topk) == 0:\n continue\n rep_level = (\n topk.groupby(\"ID\", as_index=False)[\"weak_positive\"]\n .mean()\n .rename(columns={\"weak_positive\": \"support\"})\n )\n rows.append({\n \"benchmark\": \"real\",\n \"scope\": \"dataset\",\n \"dataset\": ds_name,\n \"metric\": f\"weak_support_at_{k}\",\n \"method\": \"ATTRIBUTION_PROXY\",\n \"value\": float(rep_level[\"support\"].mean()),\n })\n\n for k in topk_list:\n topk = z[z[\"rank\"] <= k].copy()\n if len(topk) == 0:\n continue\n rep_level = (\n topk.groupby(\"ID\", as_index=False)[\"weak_positive\"]\n .mean()\n .rename(columns={\"weak_positive\": \"support\"})\n )\n rows.append({\n \"benchmark\": \"real\",\n \"scope\": \"global\",\n \"dataset\": \"ALL\",\n \"metric\": f\"weak_support_at_{k}\",\n \"method\": \"ATTRIBUTION_PROXY\",\n \"value\": float(rep_level[\"support\"].mean()),\n })\n\n return pd.DataFrame(rows)\n\"\"\")\n\ntrain_benchmarks_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nimport sys\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport yaml\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import train_test_split\n\nPROJECT_ROOT = Path(__file__).resolve().parent\nif str(PROJECT_ROOT) not in sys.path:\n sys.path.insert(0, str(PROJECT_ROOT))\n\nfrom src.branch_b_public import (\n build_feature_table_from_meta,\n build_fold_signal_catalog,\n)\nfrom src.evaluation import real_benchmark_table, safe_auc\nfrom src.simulation import (\n estimate_empirical_background,\n sequence_scores_from_bundle,\n simulate_ligo_dataset,\n simulate_simairr_dataset,\n)\nfrom src.utils import ensure_dir, seed_everything\n\n\ndef fit_lr_scores(train_feat: pd.DataFrame, val_feat: pd.DataFrame, feature_cols):\n X_tr = train_feat[feature_cols].fillna(0.0).values\n y_tr = train_feat[\"label_positive\"].astype(int).values\n X_va = val_feat[feature_cols].fillna(0.0).values\n\n clf = LogisticRegression(max_iter=3000, solver=\"liblinear\", class_weight=\"balanced\", random_state=42)\n clf.fit(X_tr, y_tr)\n pred = clf.predict_proba(X_va)[:, 1]\n return pred\n\n\ndef build_synth_branch_b_features(meta_train, meta_val, cache, bundle):\n tr_feat = build_feature_table_from_meta(meta_train, cache, bundle)\n va_feat = build_feature_table_from_meta(meta_val, cache, bundle)\n return tr_feat, va_feat\n\n\ndef run_simairr_benchmark(cfg, aa_probs, length_source):\n rows = []\n\n sim_cfg = cfg[\"simairr\"]\n dict_cfg = cfg[\"dictionary\"]\n\n meta, cache, truth = simulate_simairr_dataset(\n n_pos=sim_cfg[\"n_pos\"],\n n_neg=sim_cfg[\"n_neg\"],\n n_background_per_rep=sim_cfg[\"n_background_per_rep\"],\n n_public_background=sim_cfg[\"n_public_background\"],\n n_disease_public=sim_cfg[\"n_disease_public\"],\n public_common_prob=sim_cfg[\"public_common_prob\"],\n disease_prob_pos=sim_cfg[\"disease_prob_pos\"],\n disease_prob_neg=sim_cfg[\"disease_prob_neg\"],\n aa_probs=aa_probs,\n length_source=length_source,\n random_state=sim_cfg[\"random_state\"],\n )\n\n tr_meta, va_meta = train_test_split(\n meta,\n test_size=0.30,\n stratify=meta[\"label_positive\"],\n random_state=sim_cfg[\"random_state\"],\n )\n\n bundle, exact_df, cluster_df = build_fold_signal_catalog(\n fold_train_meta=tr_meta,\n cache=cache,\n min_freq=dict_cfg[\"min_freq\"],\n enrichment=dict_cfg[\"enrichment\"],\n top_exact=dict_cfg[\"top_exact\"],\n top_clusters=dict_cfg[\"top_clusters\"],\n max_repertoires_per_class=dict_cfg[\"max_repertoires_per_class\"],\n random_state=sim_cfg[\"random_state\"],\n )\n\n tr_feat, va_feat = build_synth_branch_b_features(tr_meta, va_meta, cache, bundle)\n y_va = va_feat[\"label_positive\"].astype(int).values\n\n feature_cols = [c for c in tr_feat.columns if c not in [\"ID\", \"dataset\", \"label_positive\"]]\n\n pred_full = fit_lr_scores(tr_feat, va_feat, feature_cols)\n auc_full = safe_auc(y_va, pred_full)\n\n exact_signal = va_feat[\"public_seq_weighted_score\"].values if \"public_seq_weighted_score\" in va_feat.columns else np.zeros(len(va_feat))\n cluster_signal = va_feat[\"cluster_enrichment_sum\"].values if \"cluster_enrichment_sum\" in va_feat.columns else np.zeros(len(va_feat))\n combo_signal = exact_signal + cluster_signal\n\n auc_exact = safe_auc(y_va, exact_signal)\n auc_cluster = safe_auc(y_va, cluster_signal)\n auc_combo = safe_auc(y_va, combo_signal)\n\n neg_mask = y_va == 0\n pos_mask = y_va == 1\n\n neg_public_hit_rate = float((va_feat.loc[neg_mask, \"public_seq_count\"] > 0).mean()) if neg_mask.sum() > 0 else np.nan\n pos_public_hit_rate = float((va_feat.loc[pos_mask, \"public_seq_count\"] > 0).mean()) if pos_mask.sum() > 0 else np.nan\n neg_cluster_hit_rate = float((va_feat.loc[neg_mask, \"cluster_hit_count\"] > 0).mean()) if neg_mask.sum() > 0 else np.nan\n pos_cluster_hit_rate = float((va_feat.loc[pos_mask, \"cluster_hit_count\"] > 0).mean()) if pos_mask.sum() > 0 else np.nan\n\n metrics = {\n \"branch_b_full_auc\": auc_full,\n \"exact_only_auc\": auc_exact,\n \"cluster_only_auc\": auc_cluster,\n \"exact_plus_cluster_auc\": auc_combo,\n \"branch_b_advantage_over_exact\": auc_full - auc_exact if pd.notna(auc_full) and pd.notna(auc_exact) else np.nan,\n \"branch_b_advantage_over_combo\": auc_full - auc_combo if pd.notna(auc_full) and pd.notna(auc_combo) else np.nan,\n \"neg_public_hit_rate\": neg_public_hit_rate,\n \"pos_public_hit_rate\": pos_public_hit_rate,\n \"neg_cluster_hit_rate\": neg_cluster_hit_rate,\n \"pos_cluster_hit_rate\": pos_cluster_hit_rate,\n \"disease_catalog_size\": float(len(exact_df)),\n \"cluster_catalog_size\": float(len(cluster_df)),\n }\n\n for metric, value in metrics.items():\n rows.append({\n \"benchmark\": \"simairr\",\n \"scenario\": \"realistic_public_sharing\",\n \"metric\": metric,\n \"value\": value,\n })\n\n return pd.DataFrame(rows)\n\n\ndef run_ligo_benchmark(cfg, aa_probs, length_source):\n rows = []\n\n ligo_cfg = cfg[\"ligo\"]\n dict_cfg = cfg[\"dictionary\"]\n\n meta, cache, truth_df = simulate_ligo_dataset(\n n_pos=ligo_cfg[\"n_pos\"],\n n_neg=ligo_cfg[\"n_neg\"],\n n_background_per_rep=ligo_cfg[\"n_background_per_rep\"],\n implanted_motifs=ligo_cfg[\"implanted_motifs\"],\n implants_per_positive=ligo_cfg[\"implants_per_positive\"],\n implant_prob_positive=ligo_cfg[\"implant_prob_positive\"],\n implant_prob_negative=ligo_cfg[\"implant_prob_negative\"],\n aa_probs=aa_probs,\n length_source=length_source,\n random_state=ligo_cfg[\"random_state\"],\n )\n\n tr_meta, va_meta = train_test_split(\n meta,\n test_size=0.30,\n stratify=meta[\"label_positive\"],\n random_state=ligo_cfg[\"random_state\"],\n )\n\n # repeated catalogs for stability\n bundle_list = []\n seed_list = [42, 52, 62]\n for seed in seed_list:\n bundle, _, _ = build_fold_signal_catalog(\n fold_train_meta=tr_meta,\n cache=cache,\n min_freq=dict_cfg[\"min_freq\"],\n enrichment=dict_cfg[\"enrichment\"],\n top_exact=dict_cfg[\"top_exact\"],\n top_clusters=dict_cfg[\"top_clusters\"],\n max_repertoires_per_class=dict_cfg[\"max_repertoires_per_class\"],\n random_state=seed,\n )\n bundle_list.append((seed, bundle))\n\n # AUROC on first bundle with branch-B-style features\n main_bundle = bundle_list[0][1]\n tr_feat = build_feature_table_from_meta(tr_meta, cache, main_bundle)\n va_feat = build_feature_table_from_meta(va_meta, cache, main_bundle)\n\n feature_cols = [c for c in tr_feat.columns if c not in [\"ID\", \"dataset\", \"label_positive\"]]\n pred = fit_lr_scores(tr_feat, va_feat, feature_cols)\n auc = safe_auc(va_feat[\"label_positive\"].astype(int).values, pred)\n\n rows.append({\n \"benchmark\": \"ligo\",\n \"scenario\": \"low_witness_implant\",\n \"metric\": \"repertoire_auc\",\n \"value\": auc,\n })\n\n # sequence recovery@k\n pos_val_ids = set(va_meta.loc[va_meta[\"label_positive\"] == 1, \"repertoire_id\"].astype(str).tolist())\n truth_pos = truth_df[truth_df[\"repertoire_id\"].astype(str).isin(pos_val_ids)].copy()\n\n recovery_by_k = {k: [] for k in ligo_cfg[\"recovery_k_list\"]}\n topk_sets_by_seed = {seed: {} for seed in seed_list}\n\n for seed, bundle in bundle_list:\n for rep_id in sorted(pos_val_ids):\n seq_counter = cache[rep_id][\"seq_counter\"]\n score_df = sequence_scores_from_bundle(seq_counter, bundle)\n\n if len(score_df) == 0:\n continue\n\n true_implants = set(truth_pos.loc[truth_pos[\"repertoire_id\"] == rep_id, \"sequence\"].astype(str).tolist())\n if len(true_implants) == 0:\n continue\n\n top10 = set(score_df.head(10)[\"sequence\"].astype(str).tolist())\n topk_sets_by_seed[seed][rep_id] = top10\n\n for k in ligo_cfg[\"recovery_k_list\"]:\n pred_topk = set(score_df.head(k)[\"sequence\"].astype(str).tolist())\n rec = len(pred_topk & true_implants) / max(1, len(true_implants))\n recovery_by_k[k].append(rec)\n\n for k, vals in recovery_by_k.items():\n rows.append({\n \"benchmark\": \"ligo\",\n \"scenario\": \"low_witness_implant\",\n \"metric\": f\"recovery_at_{k}\",\n \"value\": float(np.mean(vals)) if len(vals) else np.nan,\n })\n\n # attribution stability: average pairwise jaccard of top10 across seeds\n jac_vals = []\n rep_ids_common = sorted(pos_val_ids)\n for rep_id in rep_ids_common:\n sets = []\n for seed in seed_list:\n if rep_id in topk_sets_by_seed[seed]:\n sets.append(topk_sets_by_seed[seed][rep_id])\n\n if len(sets) < 2:\n continue\n\n for i in range(len(sets)):\n for j in range(i + 1, len(sets)):\n a, b = sets[i], sets[j]\n denom = len(a | b)\n if denom == 0:\n continue\n jac_vals.append(len(a & b) / denom)\n\n rows.append({\n \"benchmark\": \"ligo\",\n \"scenario\": \"low_witness_implant\",\n \"metric\": \"attribution_stability_top10_jaccard\",\n \"value\": float(np.mean(jac_vals)) if len(jac_vals) else np.nan,\n })\n\n return pd.DataFrame(rows)\n\n\ndef main():\n cfg = yaml.safe_load((PROJECT_ROOT / \"configs\" / \"benchmark.yaml\").read_text())\n out_dir = ensure_dir(cfg[\"paths\"][\"output_root\"])\n\n seed_everything(42)\n\n canonical = pd.read_csv(cfg[\"paths\"][\"canonical_metadata\"])\n fusion_oof = pd.read_csv(cfg[\"paths\"][\"fusion_oof\"])\n phase6_dir = Path(cfg[\"paths\"][\"phase6_dir\"])\n\n from src.evaluation import load_ranked_attribution_tables\n ranked_attr = load_ranked_attribution_tables(phase6_dir)\n\n train_meta = canonical[canonical[\"source\"] == \"train\"].copy()\n train_root = Path(\"/kaggle/input/competitions/adaptive-immune-profiling-challenge-2025/train_datasets/train_datasets\")\n\n aa_probs, length_source = estimate_empirical_background(\n canonical_train_meta=train_meta,\n train_root=train_root,\n max_repertoires=16,\n max_sequences_per_file=4000,\n random_state=42,\n )\n\n # Benchmark 1: real data (OOF + attribution proxy)\n real_df = real_benchmark_table(\n fusion_oof=fusion_oof,\n canonical_meta=canonical,\n ranked_attr_df=ranked_attr,\n topk_list=cfg[\"real\"][\"topk_list\"],\n )\n real_df.to_csv(out_dir / \"benchmark_real.csv\", index=False)\n\n # Benchmark 2: simAIRR-style\n sim_df = run_simairr_benchmark(cfg, aa_probs=aa_probs, length_source=length_source)\n sim_df.to_csv(out_dir / \"benchmark_simairr.csv\", index=False)\n\n # Benchmark 3: LIgO-style implanted signal\n ligo_df = run_ligo_benchmark(cfg, aa_probs=aa_probs, length_source=length_source)\n ligo_df.to_csv(out_dir / \"benchmark_ligo.csv\", index=False)\n\n print(\"=\" * 80)\n print(\"PHASE 8: Benchmarking section\")\n print(\"=\" * 80)\n\n print(\"\\\\nReal benchmark:\")\n print(real_df.head(20).to_string(index=False))\n\n print(\"\\\\nsimAIRR benchmark:\")\n print(sim_df.to_string(index=False))\n\n print(\"\\\\nLIgO benchmark:\")\n print(ligo_df.to_string(index=False))\n\n print(\"\\\\nSaved:\")\n print(out_dir / \"benchmark_real.csv\")\n print(out_dir / \"benchmark_simairr.csv\")\n print(out_dir / \"benchmark_ligo.csv\")\n print(\"=\" * 80)\n\n\nif __name__ == \"__main__\":\n main()\n\"\"\")\n\n(PROJECT_ROOT / \"src\" / \"simulation.py\").write_text(simulation_py, encoding=\"utf-8\")\n(PROJECT_ROOT / \"src\" / \"evaluation.py\").write_text(evaluation_py, encoding=\"utf-8\")\n(PROJECT_ROOT / \"train_benchmarks.py\").write_text(train_benchmarks_py, encoding=\"utf-8\")\n\nprint(\"Written:\")\nprint(\"-\", PROJECT_ROOT / \"src\" / \"simulation.py\")\nprint(\"-\", PROJECT_ROOT / \"src\" / \"evaluation.py\")\nprint(\"-\", PROJECT_ROOT / \"train_benchmarks.py\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 3: Benchmark imports verify\n\nimport sys\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\nif str(PROJECT_ROOT) not in sys.path:\n sys.path.insert(0, str(PROJECT_ROOT))\n\nfrom src.simulation import (\n simulate_simairr_dataset,\n simulate_ligo_dataset,\n sequence_scores_from_bundle,\n)\nfrom src.evaluation import (\n real_benchmark_table,\n load_ranked_attribution_tables,\n)\n\nprint(\"Phase 8 imports OK\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 4: Run Phase 8 benchmarking\n\n!python /kaggle/working/project/train_benchmarks.py","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 5: Inspect benchmark outputs\n\nfrom pathlib import Path\nimport pandas as pd\n\nOUT_DIR = Path(\"/kaggle/working/project/artifacts/phase8_benchmarks\")\n\nprint(\"Files:\")\nfor p in sorted(OUT_DIR.glob(\"*\")):\n print(\"-\", p.name)\n\nprint(\"\\nbenchmark_real.csv\")\ndisplay(pd.read_csv(OUT_DIR / \"benchmark_real.csv\").head(30))\n\nprint(\"\\nbenchmark_simairr.csv\")\ndisplay(pd.read_csv(OUT_DIR / \"benchmark_simairr.csv\"))\n\nprint(\"\\nbenchmark_ligo.csv\")\ndisplay(pd.read_csv(OUT_DIR / \"benchmark_ligo.csv\"))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Phase 9 . ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 1: Phase 9 packaging config file write\n\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nphase9_yaml = \"\"\"\nenabled: true\n\npaths:\n output_root: /kaggle/working/project/artifacts/phase9_packaging\n phase1_root: /kaggle/working/project/artifacts/phase1\n phase2_root: /kaggle/working/project/artifacts/phase2_branch_a\n phase3_root: /kaggle/working/project/artifacts/phase3_branch_b\n phase4_root: /kaggle/working/project/artifacts/phase4_branch_c\n phase5_root: /kaggle/working/project/artifacts/phase5_branch_d\n phase6_root: /kaggle/working/project/artifacts/phase6_attribution\n phase7_root: /kaggle/working/project/artifacts/phase7_fusion\n phase8_root: /kaggle/working/project/artifacts/phase8_benchmarks\n\nruntime_flags:\n USE_BRANCH_A: true\n USE_BRANCH_B: true\n USE_BRANCH_C: true\n USE_BRANCH_D: true\n SPLIT_MODE: group_stratified\n N_SEEDS: 3\n RANDOM_STATE: 42\n\"\"\"\n\n(PROJECT_ROOT / \"configs\" / \"phase9.yaml\").write_text(phase9_yaml.strip() + \"\\n\", encoding=\"utf-8\")\nprint(\"Written:\", PROJECT_ROOT / \"configs\" / \"phase9.yaml\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 2: Write src/repro.py and run_phase9.py\n\nfrom pathlib import Path\nfrom textwrap import dedent\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nrepro_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nimport hashlib\nimport os\nimport platform\nimport shutil\nimport subprocess\nimport sys\nfrom pathlib import Path\nfrom typing import Dict, List\n\nimport numpy as np\nimport pandas as pd\n\nfrom .utils import ensure_dir\n\n\ndef safe_read_csv(path: Path) -> pd.DataFrame:\n try:\n if path.exists():\n return pd.read_csv(path)\n except Exception:\n pass\n return pd.DataFrame()\n\n\ndef safe_read_parquet(path: Path) -> pd.DataFrame:\n try:\n if path.exists():\n return pd.read_parquet(path)\n except Exception:\n pass\n return pd.DataFrame()\n\n\ndef concat_csvs(paths: List[Path]) -> pd.DataFrame:\n parts = []\n for p in paths:\n x = safe_read_csv(p)\n if len(x):\n parts.append(x)\n if len(parts) == 0:\n return pd.DataFrame()\n return pd.concat(parts, ignore_index=True)\n\n\ndef concat_parquet_or_csv(parquet_paths: List[Path], csv_paths: List[Path]) -> pd.DataFrame:\n parts = []\n for p in parquet_paths:\n x = safe_read_parquet(p)\n if len(x):\n parts.append(x)\n for p in csv_paths:\n x = safe_read_csv(p)\n if len(x):\n parts.append(x)\n if len(parts) == 0:\n return pd.DataFrame()\n return pd.concat(parts, ignore_index=True)\n\n\ndef sha256_small(path: Path, chunk_size: int = 1024 * 1024) -> str:\n try:\n h = hashlib.sha256()\n with open(path, \"rb\") as f:\n while True:\n chunk = f.read(chunk_size)\n if not chunk:\n break\n h.update(chunk)\n return h.hexdigest()\n except Exception:\n return \"\"\n\n\ndef write_environment_file(out_path: Path):\n lines = []\n lines.append(f\"python_version: {sys.version}\")\n lines.append(f\"platform: {platform.platform()}\")\n lines.append(f\"executable: {sys.executable}\")\n lines.append(\"\")\n\n try:\n res = subprocess.run([sys.executable, \"-m\", \"pip\", \"freeze\"], capture_output=True, text=True, timeout=60)\n lines.append(\"[pip_freeze]\")\n lines.append(res.stdout.strip())\n except Exception as e:\n lines.append(\"[pip_freeze]\")\n lines.append(f\"ERROR: {e}\")\n\n out_path.write_text(\"\\\\n\".join(lines) + \"\\\\n\", encoding=\"utf-8\")\n\n\ndef snapshot_configs(config_dir: Path, out_dir: Path):\n out_dir = ensure_dir(out_dir)\n for p in sorted(config_dir.glob(\"*.yaml\")):\n shutil.copy2(p, out_dir / p.name)\n\n\ndef build_manifest(project_root: Path, out_path: Path):\n rows = []\n for p in sorted(project_root.rglob(\"*\")):\n if p.is_file():\n try:\n rel = p.relative_to(project_root)\n except Exception:\n rel = p\n rows.append({\n \"relative_path\": str(rel),\n \"size_bytes\": int(p.stat().st_size),\n \"sha256\": sha256_small(p),\n })\n pd.DataFrame(rows).to_csv(out_path, index=False)\n\n\ndef write_runtime_flags_yaml(runtime_flags: dict, out_path: Path):\n lines = []\n for k, v in runtime_flags.items():\n if isinstance(v, bool):\n vv = \"true\" if v else \"false\"\n else:\n vv = v\n lines.append(f\"{k}: {vv}\")\n out_path.write_text(\"\\\\n\".join(lines) + \"\\\\n\", encoding=\"utf-8\")\n\n\ndef write_run_pipeline_skeleton(project_root: Path):\n script = f'''from __future__ import annotations\n\nfrom pathlib import Path\nimport subprocess\nimport sys\nimport yaml\n\nPROJECT_ROOT = Path(\"{project_root}\")\nCONFIG_PATH = PROJECT_ROOT / \"configs\" / \"phase9.yaml\"\n\n\ndef run_script(path: Path):\n print(f\"Running: {{path}}\")\n res = subprocess.run([sys.executable, str(path)])\n if res.returncode != 0:\n raise SystemExit(res.returncode)\n\n\ndef main():\n cfg = yaml.safe_load(CONFIG_PATH.read_text())\n flags = cfg[\"runtime_flags\"]\n\n # Phase 1\n run_script(PROJECT_ROOT / \"run_phase1.py\")\n\n # Branches\n if flags.get(\"USE_BRANCH_A\", True):\n run_script(PROJECT_ROOT / \"train_branch_a.py\")\n\n if flags.get(\"USE_BRANCH_B\", True):\n run_script(PROJECT_ROOT / \"train_branch_b.py\")\n\n if flags.get(\"USE_BRANCH_C\", False):\n run_script(PROJECT_ROOT / \"train_branch_c.py\")\n\n if flags.get(\"USE_BRANCH_D\", True):\n run_script(PROJECT_ROOT / \"train_branch_d.py\")\n\n # Fusion + Benchmarks + Packaging\n run_script(PROJECT_ROOT / \"train_fusion.py\")\n run_script(PROJECT_ROOT / \"train_benchmarks.py\")\n run_script(PROJECT_ROOT / \"run_phase9.py\")\n\n\nif __name__ == \"__main__\":\n main()\n'''\n (project_root / \"run_pipeline.py\").write_text(script, encoding=\"utf-8\")\n\n\ndef _best_rows(df: pd.DataFrame, metric_col: str, group_cols: List[str]) -> pd.DataFrame:\n if len(df) == 0:\n return pd.DataFrame()\n x = df.dropna(subset=[metric_col]).copy()\n if len(x) == 0:\n return pd.DataFrame()\n idx = x.groupby(group_cols)[metric_col].idxmax()\n return x.loc[idx].sort_values(group_cols).reset_index(drop=True)\n\n\ndef _df_to_text(df: pd.DataFrame, max_rows: int = 20) -> str:\n if df is None or len(df) == 0:\n return \"No rows found.\\\\n\"\n return df.head(max_rows).to_string(index=False) + \"\\\\n\"\n\n\ndef generate_branch_a_report(phase2_root: Path, out_path: Path):\n summary = safe_read_csv(phase2_root / \"branch_a_summary.csv\")\n ablation = safe_read_csv(phase2_root / \"branch_a_feature_ablation.csv\")\n selected_files = sorted(phase2_root.glob(\"selected_features_*.csv\"))\n\n txt = []\n txt.append(\"# report_branch_a\\\\n\")\n txt.append(\"## Files detected\\\\n\")\n txt.append(f\"- branch_a_summary.csv: {(phase2_root / 'branch_a_summary.csv').exists()}\\\\n\")\n txt.append(f\"- branch_a_feature_ablation.csv: {(phase2_root / 'branch_a_feature_ablation.csv').exists()}\\\\n\")\n txt.append(f\"- selected feature files: {len(selected_files)}\\\\n\")\n\n if len(summary):\n txt.append(\"\\\\n## Best Branch A models by dataset\\\\n\")\n best = _best_rows(summary, \"mean_auc\", [\"dataset\", \"seed\"])\n txt.append(_df_to_text(best[[\"dataset\", \"seed\", \"model\", \"mean_auc\", \"std_auc\", \"max_auc\"]], max_rows=50))\n\n if len(ablation):\n txt.append(\"\\\\n## Best ablation stage by dataset\\\\n\")\n best_ab = _best_rows(ablation, \"auc\", [\"dataset\"])\n txt.append(_df_to_text(best_ab[[\"dataset\", \"stage\", \"n_features_stage\", \"auc\"]], max_rows=50))\n\n out_path.write_text(\"\\\\n\".join(txt), encoding=\"utf-8\")\n\n\ndef generate_branch_b_report(phase3_root: Path, out_path: Path):\n summary = safe_read_csv(phase3_root / \"branch_b_summary.csv\")\n exact_files = sorted(phase3_root.glob(\"disease_enriched_sequences_*.csv\"))\n cluster_files = sorted(phase3_root.glob(\"cluster_catalog_*.csv\"))\n\n txt = []\n txt.append(\"# report_branch_b\\\\n\")\n txt.append(\"## Files detected\\\\n\")\n txt.append(f\"- branch_b_summary.csv: {(phase3_root / 'branch_b_summary.csv').exists()}\\\\n\")\n txt.append(f\"- exact catalog files: {len(exact_files)}\\\\n\")\n txt.append(f\"- cluster catalog files: {len(cluster_files)}\\\\n\")\n\n if len(summary):\n txt.append(\"\\\\n## Best Branch B models by dataset\\\\n\")\n best = _best_rows(summary, \"mean_auc\", [\"dataset\", \"seed\"])\n txt.append(_df_to_text(best[[\"dataset\", \"seed\", \"model\", \"mean_auc\", \"std_auc\", \"max_auc\"]], max_rows=50))\n\n if len(exact_files):\n sample_exact = safe_read_csv(exact_files[0])\n txt.append(\"\\\\n## Example exact disease-enriched catalog\\\\n\")\n txt.append(_df_to_text(sample_exact.head(10), max_rows=10))\n\n if len(cluster_files):\n sample_cluster = safe_read_csv(cluster_files[0])\n txt.append(\"\\\\n## Example approximate cluster catalog\\\\n\")\n txt.append(_df_to_text(sample_cluster.head(10), max_rows=10))\n\n out_path.write_text(\"\\\\n\".join(txt), encoding=\"utf-8\")\n\n\ndef generate_branch_c_report(phase4_root: Path, out_path: Path):\n summary = safe_read_csv(phase4_root / \"branch_c_summary.csv\")\n emb_parquet = phase4_root / \"embedding_repertoire_vectors.parquet\"\n emb_csv = phase4_root / \"embedding_repertoire_vectors_fallback.csv\"\n pca_plot = phase4_root / \"embedding_pca_plot.png\"\n\n txt = []\n txt.append(\"# report_branch_c\\\\n\")\n txt.append(\"## Files detected\\\\n\")\n txt.append(f\"- branch_c_summary.csv: {(phase4_root / 'branch_c_summary.csv').exists()}\\\\n\")\n txt.append(f\"- embedding_repertoire_vectors.parquet: {emb_parquet.exists()}\\\\n\")\n txt.append(f\"- embedding_repertoire_vectors_fallback.csv: {emb_csv.exists()}\\\\n\")\n txt.append(f\"- embedding_pca_plot.png: {pca_plot.exists()}\\\\n\")\n\n if len(summary):\n txt.append(\"\\\\n## Best Branch C models by dataset\\\\n\")\n best = _best_rows(summary, \"mean_auc\", [\"dataset\", \"seed\"])\n txt.append(_df_to_text(best[[\"dataset\", \"seed\", \"model\", \"mean_auc\", \"std_auc\", \"max_auc\"]], max_rows=50))\n\n emb_df = safe_read_parquet(emb_parquet)\n if len(emb_df) == 0:\n emb_df = safe_read_csv(emb_csv)\n\n if len(emb_df):\n meta_cols = [c for c in [\"dataset\", \"embed_route_used\", \"embed_model\", \"embed_dim\"] if c in emb_df.columns]\n meta_view = emb_df[meta_cols].drop_duplicates().head(20)\n txt.append(\"\\\\n## Embedding metadata\\\\n\")\n txt.append(_df_to_text(meta_view, max_rows=20))\n\n out_path.write_text(\"\\\\n\".join(txt), encoding=\"utf-8\")\n\n\ndef generate_branch_d_report(phase5_root: Path, out_path: Path):\n summary_files = sorted(phase5_root.glob(\"branch_d_summary_*.csv\"))\n summary = concat_csvs(summary_files)\n seq_files = sorted(phase5_root.glob(\"mil_sequence_scores_*.parquet\"))\n seq_fallback = sorted(phase5_root.glob(\"mil_sequence_scores_*_fallback.csv\"))\n attn_files = sorted(phase5_root.glob(\"mil_attention_maps_*.npz\"))\n\n txt = []\n txt.append(\"# report_branch_d\\\\n\")\n txt.append(\"## Files detected\\\\n\")\n txt.append(f\"- branch_d_oof.csv: {(phase5_root / 'branch_d_oof.csv').exists()}\\\\n\")\n txt.append(f\"- branch_d summary files: {len(summary_files)}\\\\n\")\n txt.append(f\"- mil sequence score files: {len(seq_files) + len(seq_fallback)}\\\\n\")\n txt.append(f\"- attention map files: {len(attn_files)}\\\\n\")\n\n if len(summary):\n txt.append(\"\\\\n## Best Branch D folds / seeds\\\\n\")\n best = _best_rows(summary[summary[\"fold\"] >= 0], \"auc\", [\"dataset\", \"seed\"])\n txt.append(_df_to_text(best[[\"dataset\", \"seed\", \"fold\", \"model\", \"auc\", \"best_epoch\"]], max_rows=50))\n\n seq_df = concat_parquet_or_csv(seq_files, seq_fallback)\n if len(seq_df):\n txt.append(\"\\\\n## Example ranked sequence attribution rows\\\\n\")\n keep_cols = [c for c in [\"ID\", \"dataset\", \"rank\", \"sequence\", \"attention_weight\", \"attr_prob\", \"final_seq_score\", \"weak_target\"] if c in seq_df.columns]\n txt.append(_df_to_text(seq_df[keep_cols].head(15), max_rows=15))\n\n out_path.write_text(\"\\\\n\".join(txt), encoding=\"utf-8\")\n\n\ndef generate_fusion_report(phase7_root: Path, phase8_root: Path, out_path: Path):\n fusion_summary = safe_read_csv(phase7_root / \"fusion_summary.csv\")\n fusion_weights = safe_read_csv(phase7_root / \"fusion_weights.csv\")\n bench_real = safe_read_csv(phase8_root / \"benchmark_real.csv\")\n bench_sim = safe_read_csv(phase8_root / \"benchmark_simairr.csv\")\n bench_ligo = safe_read_csv(phase8_root / \"benchmark_ligo.csv\")\n\n txt = []\n txt.append(\"# report_fusion\\\\n\")\n txt.append(\"## Files detected\\\\n\")\n txt.append(f\"- fusion_oof.csv: {(phase7_root / 'fusion_oof.csv').exists()}\\\\n\")\n txt.append(f\"- fusion_weights.csv: {(phase7_root / 'fusion_weights.csv').exists()}\\\\n\")\n txt.append(f\"- fusion_summary.csv: {(phase7_root / 'fusion_summary.csv').exists()}\\\\n\")\n txt.append(f\"- benchmark_real.csv: {(phase8_root / 'benchmark_real.csv').exists()}\\\\n\")\n txt.append(f\"- benchmark_simairr.csv: {(phase8_root / 'benchmark_simairr.csv').exists()}\\\\n\")\n txt.append(f\"- benchmark_ligo.csv: {(phase8_root / 'benchmark_ligo.csv').exists()}\\\\n\")\n\n if len(fusion_summary):\n txt.append(\"\\\\n## Best fusion method by dataset\\\\n\")\n best = _best_rows(fusion_summary, \"auc\", [\"dataset\"])\n txt.append(_df_to_text(best[[\"dataset\", \"method\", \"auc\", \"n_samples\"]], max_rows=50))\n\n if len(fusion_weights):\n txt.append(\"\\\\n## Fusion weights\\\\n\")\n txt.append(_df_to_text(fusion_weights.head(50), max_rows=50))\n\n if len(bench_real):\n txt.append(\"\\\\n## Real benchmark excerpt\\\\n\")\n txt.append(_df_to_text(bench_real.head(30), max_rows=30))\n\n if len(bench_sim):\n txt.append(\"\\\\n## simAIRR-style benchmark excerpt\\\\n\")\n txt.append(_df_to_text(bench_sim, max_rows=30))\n\n if len(bench_ligo):\n txt.append(\"\\\\n## LIgO-style benchmark excerpt\\\\n\")\n txt.append(_df_to_text(bench_ligo, max_rows=30))\n\n out_path.write_text(\"\\\\n\".join(txt), encoding=\"utf-8\")\n\n\ndef generate_benchmark_report(phase8_root: Path, out_path: Path):\n bench_real = safe_read_csv(phase8_root / \"benchmark_real.csv\")\n bench_sim = safe_read_csv(phase8_root / \"benchmark_simairr.csv\")\n bench_ligo = safe_read_csv(phase8_root / \"benchmark_ligo.csv\")\n\n txt = []\n txt.append(\"# report_benchmarks\\\\n\")\n\n txt.append(\"## benchmark_real.csv\\\\n\")\n txt.append(_df_to_text(bench_real.head(40), max_rows=40))\n\n txt.append(\"\\\\n## benchmark_simairr.csv\\\\n\")\n txt.append(_df_to_text(bench_sim, max_rows=40))\n\n txt.append(\"\\\\n## benchmark_ligo.csv\\\\n\")\n txt.append(_df_to_text(bench_ligo, max_rows=40))\n\n out_path.write_text(\"\\\\n\".join(txt), encoding=\"utf-8\")\n\n\ndef generate_old_code_modification_map(out_path: Path):\n txt = []\n txt.append(\"# exact_old_code_modification_map\\\\n\")\n\n txt.append(\"## Config split\\\\n\")\n txt.append(\"- Old: one constant-heavy `Config` class.\\\\n\")\n txt.append(\"- New: separate static paths and runtime YAML config.\\\\n\")\n txt.append(\"- Add runtime flags:\\\\n\")\n txt.append(\" - `USE_BRANCH_A = True`\\\\n\")\n txt.append(\" - `USE_BRANCH_B = True`\\\\n\")\n txt.append(\" - `USE_BRANCH_C = True` or `False` depending on checkpoint availability\\\\n\")\n txt.append(\" - `USE_BRANCH_D = True`\\\\n\")\n txt.append(\" - `SPLIT_MODE = \\\\\"group_stratified\\\\\"`\\\\n\")\n txt.append(\" - `N_SEEDS = 3`\\\\n\")\n\n txt.append(\"\\\\n## read_repertoire() changes\\\\n\")\n txt.append(\"- Keep function, but add optional output-support fields: `duplicate_count`, `normalized_templates`, `seq_len`.\\\\n\")\n\n txt.append(\"\\\\n## FeatureExtractor rename and split\\\\n\")\n txt.append(\"- Rename to `BranchAFeatureExtractor`.\\\\n\")\n txt.append(\"- Refactor into methods:\\\\n\")\n txt.append(\" - `extract_basic_stats()`\\\\n\")\n txt.append(\" - `extract_physchem()`\\\\n\")\n txt.append(\" - `extract_kmers()`\\\\n\")\n txt.append(\" - `extract_vj_usage()`\\\\n\")\n txt.append(\" - `extract_public_clone_features()`\\\\n\")\n txt.append(\" - `extract_overlap_features()`\\\\n\")\n\n txt.append(\"\\\\n## mine_public_clones() scope fix\\\\n\")\n txt.append(\"- Old: full-dataset public clone mining before CV.\\\\n\")\n txt.append(\"- New: fold-local mining only.\\\\n\")\n txt.append(\"- Required signature:\\\\n\")\n txt.append(\" `def mine_public_clones_from_fold(train_meta_fold, dataset_path, ...):`\\\\n\")\n\n txt.append(\"\\\\n## MultiModelTrainer rename and restriction\\\\n\")\n txt.append(\"- Rename to `BranchATrainer`.\\\\n\")\n txt.append(\"- Restrict responsibility to Branch A only.\\\\n\")\n txt.append(\"- Add:\\\\n\")\n txt.append(\" - group-aware fold loop\\\\n\")\n txt.append(\" - fold-local feature selection\\\\n\")\n txt.append(\" - multi-seed training\\\\n\")\n txt.append(\" - ablation mode\\\\n\")\n\n txt.append(\"\\\\n## main() rewrite\\\\n\")\n txt.append(\"- Old flow: load data -> extract features -> train one trainer -> predict\\\\n\")\n txt.append(\"- New orchestrated flow:\\\\n\")\n txt.append(\"```python\\\\n\")\n txt.append(\"def main():\\\\n\")\n txt.append(\" phase1_build_metadata()\\\\n\")\n txt.append(\" phase1_make_splits()\\\\n\")\n txt.append(\" phase1_run_drift()\\\\n\\\\n\")\n txt.append(\" if USE_BRANCH_A:\\\\n\")\n txt.append(\" run_branch_a()\\\\n\\\\n\")\n txt.append(\" if USE_BRANCH_B:\\\\n\")\n txt.append(\" run_branch_b()\\\\n\\\\n\")\n txt.append(\" if USE_BRANCH_C:\\\\n\")\n txt.append(\" run_branch_c()\\\\n\\\\n\")\n txt.append(\" if USE_BRANCH_D:\\\\n\")\n txt.append(\" run_branch_d()\\\\n\\\\n\")\n txt.append(\" run_fusion()\\\\n\")\n txt.append(\" run_benchmarks()\\\\n\")\n txt.append(\" make_submission()\\\\n\")\n txt.append(\"```\\\\n\")\n\n out_path.write_text(\"\\\\n\".join(txt), encoding=\"utf-8\")\n\"\"\")\n\nrun_phase9_py = dedent(\"\"\"\nfrom __future__ import annotations\n\nimport sys\nfrom pathlib import Path\n\nimport yaml\n\nPROJECT_ROOT = Path(__file__).resolve().parent\nif str(PROJECT_ROOT) not in sys.path:\n sys.path.insert(0, str(PROJECT_ROOT))\n\nfrom src.repro import (\n build_manifest,\n generate_benchmark_report,\n generate_branch_a_report,\n generate_branch_b_report,\n generate_branch_c_report,\n generate_branch_d_report,\n generate_fusion_report,\n generate_old_code_modification_map,\n snapshot_configs,\n write_environment_file,\n write_run_pipeline_skeleton,\n write_runtime_flags_yaml,\n)\nfrom src.utils import ensure_dir\n\n\ndef main():\n cfg = yaml.safe_load((PROJECT_ROOT / \"configs\" / \"phase9.yaml\").read_text())\n\n paths = cfg[\"paths\"]\n phase9_root = ensure_dir(paths[\"output_root\"])\n reports_dir = ensure_dir(phase9_root / \"reports\")\n config_snapshot_dir = ensure_dir(phase9_root / \"config_snapshot\")\n\n phase1_root = Path(paths[\"phase1_root\"])\n phase2_root = Path(paths[\"phase2_root\"])\n phase3_root = Path(paths[\"phase3_root\"])\n phase4_root = Path(paths[\"phase4_root\"])\n phase5_root = Path(paths[\"phase5_root\"])\n phase6_root = Path(paths[\"phase6_root\"])\n phase7_root = Path(paths[\"phase7_root\"])\n phase8_root = Path(paths[\"phase8_root\"])\n\n print(\"=\" * 80)\n print(\"PHASE 9: Reproducibility / packaging\")\n print(\"=\" * 80)\n\n # Snapshot configs\n snapshot_configs(PROJECT_ROOT / \"configs\", config_snapshot_dir)\n\n # Environment\n write_environment_file(phase9_root / \"environment.txt\")\n\n # Runtime flags\n write_runtime_flags_yaml(cfg[\"runtime_flags\"], phase9_root / \"runtime_flags.yaml\")\n\n # Reports\n generate_branch_a_report(phase2_root, reports_dir / \"report_branch_a.md\")\n generate_branch_b_report(phase3_root, reports_dir / \"report_branch_b.md\")\n generate_branch_c_report(phase4_root, reports_dir / \"report_branch_c.md\")\n generate_branch_d_report(phase5_root, reports_dir / \"report_branch_d.md\")\n generate_fusion_report(phase7_root, phase8_root, reports_dir / \"report_fusion.md\")\n generate_benchmark_report(phase8_root, reports_dir / \"report_benchmarks.md\")\n\n # Exact old-code modification map\n generate_old_code_modification_map(phase9_root / \"exact_old_code_modification_map.md\")\n\n # Orchestrator skeleton\n write_run_pipeline_skeleton(PROJECT_ROOT)\n\n # Manifest শেষের দিকে বানাও যাতে সব generated files ঢুকে যায়\n build_manifest(PROJECT_ROOT, phase9_root / \"project_file_manifest.csv\")\n\n print(\"Saved reports:\")\n for p in sorted(reports_dir.glob(\"*.md\")):\n print(\"-\", p)\n\n print(\"\\\\nSaved packaging files:\")\n print(\"-\", phase9_root / \"environment.txt\")\n print(\"-\", phase9_root / \"runtime_flags.yaml\")\n print(\"-\", phase9_root / \"project_file_manifest.csv\")\n print(\"-\", phase9_root / \"exact_old_code_modification_map.md\")\n print(\"-\", PROJECT_ROOT / \"run_pipeline.py\")\n print(\"=\" * 80)\n\n\nif __name__ == \"__main__\":\n main()\n\"\"\")\n\n(PROJECT_ROOT / \"src\" / \"repro.py\").write_text(repro_py, encoding=\"utf-8\")\n(PROJECT_ROOT / \"run_phase9.py\").write_text(run_phase9_py, encoding=\"utf-8\")\n\nprint(\"Written:\")\nprint(\"-\", PROJECT_ROOT / \"src\" / \"repro.py\")\nprint(\"-\", PROJECT_ROOT / \"run_phase9.py\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 3: Packaging imports verify\n\nimport sys\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\nif str(PROJECT_ROOT) not in sys.path:\n sys.path.insert(0, str(PROJECT_ROOT))\n\nfrom src.repro import (\n build_manifest,\n generate_branch_a_report,\n generate_branch_b_report,\n generate_branch_c_report,\n generate_branch_d_report,\n generate_fusion_report,\n generate_old_code_modification_map,\n write_environment_file,\n write_run_pipeline_skeleton,\n)\n\nprint(\"Phase 9 imports OK\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 4: Run Phase 9 reproducibility / packaging\n\n!python /kaggle/working/project/run_phase9.py","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 5: Inspect Phase 9 outputs\n\nfrom pathlib import Path\nimport pandas as pd\n\nOUT_DIR = Path(\"/kaggle/working/project/artifacts/phase9_packaging\")\nREPORTS_DIR = OUT_DIR / \"reports\"\n\nprint(\"Phase 9 files:\")\nfor p in sorted(OUT_DIR.glob(\"*\")):\n print(\"-\", p.name)\n\nprint(\"\\nReport files:\")\nfor p in sorted(REPORTS_DIR.glob(\"*.md\")):\n print(\"-\", p.name)\n\nmanifest_path = OUT_DIR / \"project_file_manifest.csv\"\nif manifest_path.exists():\n print(\"\\nproject_file_manifest.csv\")\n display(pd.read_csv(manifest_path).head())\n\nruntime_flags_path = OUT_DIR / \"runtime_flags.yaml\"\nif runtime_flags_path.exists():\n print(\"\\nruntime_flags.yaml\")\n print(runtime_flags_path.read_text())\n\nreport_a = REPORTS_DIR / \"report_branch_a.md\"\nif report_a.exists():\n print(\"\\nreport_branch_a.md (preview)\")\n print(report_a.read_text()[:2000])\n\nreport_fusion = REPORTS_DIR / \"report_fusion.md\"\nif report_fusion.exists():\n print(\"\\nreport_fusion.md (preview)\")\n print(report_fusion.read_text()[:2000])\n\nmod_map = OUT_DIR / \"exact_old_code_modification_map.md\"\nif mod_map.exists():\n print(\"\\nexact_old_code_modification_map.md (preview)\")\n print(mod_map.read_text()[:2500])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}