{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"066b8c05","cell_type":"markdown","source":"# Phase 1 — Standards-first Data Layer (MiAIRR-aware metadata, leakage-safe splits, drift, config-driven)\n\nBuilds the canonical MiAIRR-aware metadata table, leakage-safe train/val/test folds, per-dataset drift analysis, and config-driven (immuneML-style) experiment layout. All later phases consume this phase's artifacts/canonical_metadata.csv, all_folds.csv and drift_auc_by_dataset.csv.\n\n---\n\n## Kaggle inputs to add before running this notebook\n\nAdd the following as Kaggle inputs (via the \"Add Input\" button on the right\npanel of the notebook editor):\n- AIRR-ML competition dataset (adaptive-immune-profiling-challenge-2025)\n\n## Workflow\n\n1. Run the **SETUP** cell — it creates `/kaggle/working/project/` and auto-merges\n   any previous-phase notebook outputs found under `/kaggle/input/`.\n2. Run each subsequent cell in order. The **Run** cell executes the phase's\n   training script; the **Inspect** cell prints a quick summary of the outputs.\n3. When the run completes, click **Save Version → Save & Run All (Commit)** so\n   the next phase can pick this phase's outputs up via \"Add Input\".\n","metadata":{}},{"id":"894c654a","cell_type":"code","source":"# ============================================================\n# SETUP — Initialize project + merge previous-phase inputs\n# ============================================================\n# This cell:\n#   1. Creates /kaggle/working/project/ fresh (idempotent re-runs).\n#   2. Auto-detects previous-phase notebook outputs under /kaggle/input/.\n#   3. Merges their project/ contents (src/, artifacts/, configs/) into\n#      /kaggle/working/project/ so this phase can build on them.\n#\n# Kaggle \"Add Input\" workflow:\n#   - Phase 1: add the AIRR-ML competition dataset only.\n#   - Phase N (N>=2): add the AIRR-ML competition dataset AND the previous\n#     phase notebook output(s). For Phase 9 add ALL of Phases 1..8.\n# ============================================================\n\nimport os\nimport shutil\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\n# Reset working project dir (idempotent re-runs)\nif PROJECT_ROOT.exists():\n    shutil.rmtree(PROJECT_ROOT)\nPROJECT_ROOT.mkdir(parents=True, exist_ok=True)\n(PROJECT_ROOT / \"src\").mkdir(parents=True, exist_ok=True)\n(PROJECT_ROOT / \"src\" / \"__init__.py\").write_text(\"\", encoding=\"utf-8\")\n(PROJECT_ROOT / \"configs\").mkdir(parents=True, exist_ok=True)\n(PROJECT_ROOT / \"artifacts\").mkdir(parents=True, exist_ok=True)\n\n# Auto-detect and merge previous-phase inputs.\n# Each previous-phase notebook output should contain a top-level `project/`\n# directory (created by Phase 1 and propagated through every later phase).\nprev_inputs_found = []\ninput_root = Path(\"/kaggle/input\")\nif input_root.exists():\n    for entry in sorted(input_root.iterdir()):\n        if not entry.is_dir():\n            continue\n        prev_project = entry / \"project\"\n        if not prev_project.is_dir():\n            continue\n        prev_inputs_found.append(entry.name)\n        for sub in [\"src\", \"artifacts\", \"configs\"]:\n            src_dir = prev_project / sub\n            if not src_dir.exists():\n                continue\n            for path in src_dir.rglob(\"*\"):\n                if path.is_file():\n                    rel = path.relative_to(src_dir)\n                    target = PROJECT_ROOT / sub / rel\n                    target.parent.mkdir(parents=True, exist_ok=True)\n                    shutil.copy2(path, target)\n\nprint(\"PROJECT_ROOT :\", PROJECT_ROOT)\nif prev_inputs_found:\n    print(\"Previous-phase inputs merged:\")\n    for name in prev_inputs_found:\n        print(\"  -\", name)\nelse:\n    print(\"Previous-phase inputs: (none — running fresh)\")\nprint(\"\\nExisting artifacts:\")\nart_dir = PROJECT_ROOT / \"artifacts\"\nif art_dir.exists():\n    for p in sorted(art_dir.glob(\"*\")):\n        print(\"  -\", p.name)\nelse:\n    print(\"  (none)\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-20T14:55:25.069793Z","iopub.execute_input":"2026-07-20T14:55:25.070049Z","iopub.status.idle":"2026-07-20T14:55:25.087126Z","shell.execute_reply.started":"2026-07-20T14:55:25.070027Z","shell.execute_reply":"2026-07-20T14:55:25.086348Z"}},"outputs":[],"execution_count":null},{"id":"a2606815","cell_type":"code","source":"#Cell 2 — config files write করো\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nbase_yaml = \"\"\"\nproject_root: /kaggle/working/project\n\npaths:\n  data_root: /kaggle/input/competitions/adaptive-immune-profiling-challenge-2025\n  train_root: /kaggle/input/competitions/adaptive-immune-profiling-challenge-2025/train_datasets/train_datasets\n  test_root: /kaggle/input/competitions/adaptive-immune-profiling-challenge-2025/test_datasets/test_datasets\n  sample_submission: /kaggle/input/competitions/adaptive-immune-profiling-challenge-2025/sample_submissions.csv\n  output_root: /kaggle/working/project/artifacts/phase1\n\nruntime:\n  random_state: 42\n  n_jobs: 4\n\nphase1:\n  n_splits: 5\n  max_rows_profile: 20000\n\"\"\"\n\nbranch_a_yaml = \"enabled: false\\n\"\nbranch_b_yaml = \"enabled: false\\n\"\nbranch_c_yaml = \"enabled: false\\n\"\nbranch_d_yaml = \"enabled: false\\n\"\nfusion_yaml = \"enabled: false\\n\"\n\n(PROJECT_ROOT / \"configs\" / \"base.yaml\").write_text(base_yaml.strip() + \"\\n\", encoding=\"utf-8\")\n(PROJECT_ROOT / \"configs\" / \"branch_a.yaml\").write_text(branch_a_yaml, encoding=\"utf-8\")\n(PROJECT_ROOT / \"configs\" / \"branch_b.yaml\").write_text(branch_b_yaml, encoding=\"utf-8\")\n(PROJECT_ROOT / \"configs\" / \"branch_c.yaml\").write_text(branch_c_yaml, encoding=\"utf-8\")\n(PROJECT_ROOT / \"configs\" / \"branch_d.yaml\").write_text(branch_d_yaml, encoding=\"utf-8\")\n(PROJECT_ROOT / \"configs\" / \"fusion.yaml\").write_text(fusion_yaml, encoding=\"utf-8\")\n\nprint(\"Config files written.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-20T14:55:26.787409Z","iopub.execute_input":"2026-07-20T14:55:26.787747Z","iopub.status.idle":"2026-07-20T14:55:26.795727Z","shell.execute_reply.started":"2026-07-20T14:55:26.787721Z","shell.execute_reply":"2026-07-20T14:55:26.795088Z"}},"outputs":[],"execution_count":null},{"id":"48a42c2e","cell_type":"code","source":"#Cell 3 — Phase 1 module files write করো\nfrom pathlib import Path\nfrom textwrap import dedent\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\n\nfiles = {}\n\nfiles[\"src/utils.py\"] = dedent(\"\"\"\nfrom __future__ import annotations\n\nimport os\nimport re\nimport random\nimport hashlib\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\n\n\ndef seed_everything(seed: int = 42):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n\n\ndef ensure_dir(path):\n    path = Path(path)\n    path.mkdir(parents=True, exist_ok=True)\n    return path\n\n\ndef normalize_colname(name: str) -> str:\n    return re.sub(r\"[^a-z0-9]+\", \"_\", str(name).strip().lower()).strip(\"_\")\n\n\ndef dataset_id_from_name(name: str) -> int:\n    s = str(name)\n    m = re.search(r\"(?:train|test)_dataset_(\\\\d+)\", s)\n    if m:\n        return int(m.group(1))\n    nums = re.findall(r\"\\\\d+\", s)\n    return int(nums[0]) if nums else -1\n\n\ndef stable_hash(x, mod: int = 1000003) -> int:\n    if pd.isna(x):\n        x = \"NA\"\n    h = hashlib.md5(str(x).encode(\"utf-8\")).hexdigest()\n    return int(h[:12], 16) % mod\n\n\ndef candidate_lookup(columns):\n    return {normalize_colname(c): c for c in columns}\n\n\ndef first_existing(columns, candidates):\n    lookup = candidate_lookup(columns)\n    for cand in candidates:\n        key = normalize_colname(cand)\n        if key in lookup:\n            return lookup[key]\n    return None\n\n\ndef clean_string_series(series, default_value=pd.NA):\n    if series is None:\n        return None\n    s = series.astype(\"string\").str.strip()\n    s = s.replace({\"\": pd.NA, \"nan\": pd.NA, \"None\": pd.NA, \"NA\": pd.NA})\n    return s.fillna(default_value)\n\n\ndef coerce_binary_label(series):\n    if series is None:\n        return None\n    if pd.api.types.is_numeric_dtype(series):\n        return pd.to_numeric(series, errors=\"coerce\")\n\n    mapping = {\n        \"true\": 1,\n        \"false\": 0,\n        \"positive\": 1,\n        \"negative\": 0,\n        \"yes\": 1,\n        \"no\": 0,\n        \"1\": 1,\n        \"0\": 0,\n    }\n    s = series.astype(\"string\").str.strip().str.lower()\n    return s.map(mapping)\n\n\ndef safe_numeric_series(series):\n    if series is None:\n        return None\n    return pd.to_numeric(series, errors=\"coerce\")\n\"\"\")\n\nfiles[\"src/data_io.py\"] = dedent(\"\"\"\nfrom __future__ import annotations\n\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\n\nfrom .utils import dataset_id_from_name\n\nAA_SET = set(list(\"ARNDCQEGHILKMFPSTWYV\"))\n\n\ndef list_repertoire_files(ds_path: Path):\n    if not ds_path.exists():\n        return []\n    return sorted([p for p in ds_path.glob(\"*.tsv\") if p.is_file()])\n\n\ndef read_metadata_raw(ds_path: Path) -> pd.DataFrame:\n    meta_path = ds_path / \"metadata.csv\"\n    if meta_path.exists():\n        return pd.read_csv(meta_path)\n    files = list_repertoire_files(ds_path)\n    return pd.DataFrame({\"filename\": [f.name for f in files]})\n\n\ndef quick_profile_repertoire(tsv_path: Path, max_rows: int = 20000) -> dict:\n    default = {\n        \"n_rows_loaded\": 0.0,\n        \"unique_seq_count\": 0.0,\n        \"unique_ratio\": 0.0,\n        \"seq_len_mean\": 0.0,\n        \"seq_len_std\": 0.0,\n        \"seq_len_q10\": 0.0,\n        \"seq_len_q90\": 0.0,\n        \"templates_sum\": 0.0,\n        \"templates_mean\": 0.0,\n        \"templates_std\": 0.0,\n        \"top1_template_frac\": 0.0,\n        \"v_unique_count\": 0.0,\n        \"j_unique_count\": 0.0,\n        \"valid_aa_ratio\": 0.0,\n        \"file_size_bytes\": float(tsv_path.stat().st_size) if tsv_path.exists() else 0.0,\n    }\n\n    if not tsv_path.exists():\n        return default\n\n    cols = [\"junction_aa\", \"templates\", \"v_call\", \"j_call\"]\n    try:\n        header = pd.read_csv(tsv_path, sep=\"\\\\t\", nrows=0)\n        usecols = [c for c in cols if c in header.columns]\n        if len(usecols) == 0:\n            return default\n        df = pd.read_csv(tsv_path, sep=\"\\\\t\", usecols=usecols, nrows=max_rows)\n    except Exception:\n        return default\n\n    if \"junction_aa\" not in df.columns:\n        return default\n\n    df[\"junction_aa\"] = df[\"junction_aa\"].fillna(\"\").astype(str)\n    df = df[df[\"junction_aa\"] != \"\"].copy()\n\n    if len(df) == 0:\n        default[\"n_rows_loaded\"] = 0.0\n        return default\n\n    if \"templates\" not in df.columns:\n        df[\"templates\"] = 1.0\n    df[\"templates\"] = pd.to_numeric(df[\"templates\"], errors=\"coerce\").fillna(1.0).clip(lower=0.0)\n\n    seqs = df[\"junction_aa\"]\n    lens = seqs.str.len().astype(float)\n    templates = df[\"templates\"].astype(float)\n\n    total_chars = float(lens.sum())\n    valid_chars = 0.0\n    for s in seqs.tolist():\n        valid_chars += sum(ch in AA_SET for ch in s)\n\n    out = dict(default)\n    out[\"n_rows_loaded\"] = float(len(df))\n    out[\"unique_seq_count\"] = float(seqs.nunique())\n    out[\"unique_ratio\"] = float(seqs.nunique() / max(1, len(df)))\n    out[\"seq_len_mean\"] = float(lens.mean())\n    out[\"seq_len_std\"] = float(lens.std(ddof=0))\n    out[\"seq_len_q10\"] = float(lens.quantile(0.10))\n    out[\"seq_len_q90\"] = float(lens.quantile(0.90))\n    out[\"templates_sum\"] = float(templates.sum())\n    out[\"templates_mean\"] = float(templates.mean())\n    out[\"templates_std\"] = float(templates.std(ddof=0))\n    out[\"top1_template_frac\"] = float(templates.max() / max(1e-9, templates.sum()))\n    out[\"v_unique_count\"] = float(df[\"v_call\"].fillna(\"\").astype(str).nunique()) if \"v_call\" in df.columns else 0.0\n    out[\"j_unique_count\"] = float(df[\"j_call\"].fillna(\"\").astype(str).nunique()) if \"j_call\" in df.columns else 0.0\n    out[\"valid_aa_ratio\"] = float(valid_chars / max(1.0, total_chars))\n    out[\"file_size_bytes\"] = float(tsv_path.stat().st_size)\n\n    return out\n\n\ndef discover_dataset_dirs(train_root: Path, test_root: Path):\n    train_dirs = sorted([p for p in train_root.glob(\"train_dataset_*\") if p.is_dir()])\n    test_dirs = sorted([p for p in test_root.glob(\"test_dataset_*\") if p.is_dir()])\n    return train_dirs, test_dirs\n\"\"\")\n\nfiles[\"src/metadata.py\"] = dedent(\"\"\"\nfrom __future__ import annotations\n\nfrom pathlib import Path\n\nimport pandas as pd\n\nfrom .data_io import discover_dataset_dirs, list_repertoire_files, read_metadata_raw\nfrom .utils import (\n    clean_string_series,\n    coerce_binary_label,\n    dataset_id_from_name,\n    first_existing,\n    safe_numeric_series,\n    stable_hash,\n)\n\n\nclass MetadataBuilder:\n    def __init__(self):\n        self.required_columns = [\n            \"dataset_name\",\n            \"dataset_id\",\n            \"source\",\n            \"repertoire_id\",\n            \"filename\",\n            \"absolute_path\",\n            \"label_positive\",\n            \"subject_id\",\n            \"sample_id\",\n            \"sequencing_run_id\",\n            \"batch_id\",\n            \"center_id\",\n            \"locus\",\n            \"chain\",\n            \"receptor_type\",\n            \"sex\",\n            \"race\",\n            \"age\",\n            \"hla_A\",\n            \"hla_B\",\n            \"hla_C\",\n            \"hla_DRB1\",\n            \"split_group_key\",\n            \"processing_hash\",\n        ]\n\n    def _pick_series(self, df: pd.DataFrame, candidates, default=pd.NA):\n        col = first_existing(df.columns, candidates)\n        if col is None:\n            return pd.Series([default] * len(df), index=df.index)\n        return df[col]\n\n    def build_dataset_table(self, ds_path: Path, source: str | None = None) -> pd.DataFrame:\n        source = source or (\"train\" if \"train_datasets\" in str(ds_path) else \"test\")\n\n        raw = read_metadata_raw(ds_path).copy()\n        files = list_repertoire_files(ds_path)\n\n        file_df = pd.DataFrame({\"filename\": [f.name for f in files]})\n        file_df[\"absolute_path\"] = [str(f) for f in files]\n\n        if len(raw) == 0:\n            raw = file_df.copy()\n\n        filename_col = first_existing(raw.columns, [\"filename\", \"file_name\", \"file\"])\n        if filename_col is None:\n            raw[\"filename\"] = [Path(x).name for x in raw.index.astype(str)]\n        else:\n            raw[\"filename\"] = raw[filename_col].astype(str).map(lambda x: Path(x).name)\n\n        if len(file_df) > 0:\n            meta = file_df.merge(raw, on=\"filename\", how=\"left\")\n        else:\n            meta = raw.copy()\n            meta[\"absolute_path\"] = meta[\"filename\"].astype(str).map(lambda x: str(ds_path / Path(x).name))\n\n        meta[\"dataset_name\"] = ds_path.name\n        meta[\"dataset_id\"] = dataset_id_from_name(ds_path.name)\n        meta[\"source\"] = source\n\n        repertoire_series = self._pick_series(\n            meta,\n            [\"repertoire_id\", \"sample_id\", \"specimen_id\", \"id\", \"sample\"],\n            default=pd.NA,\n        )\n        repertoire_series = repertoire_series.astype(\"string\")\n        meta[\"repertoire_id\"] = repertoire_series.fillna(\n            meta[\"filename\"].astype(str).map(lambda x: Path(x).stem)\n        )\n\n        label_series = self._pick_series(meta, [\"label_positive\", \"label\", \"y\"], default=pd.NA)\n        label_series = coerce_binary_label(label_series)\n        if source == \"train\":\n            meta[\"label_positive\"] = label_series\n        else:\n            meta[\"label_positive\"] = pd.NA\n\n        subject_id = self._pick_series(\n            meta,\n            [\"subject_id\", \"subject\", \"participant_id\", \"donor_id\", \"patient_id\", \"individual_id\"],\n            default=pd.NA,\n        )\n        sample_id = self._pick_series(\n            meta,\n            [\"sample_id\", \"sample\", \"specimen_id\"],\n            default=pd.NA,\n        )\n        sequencing_run_id = self._pick_series(\n            meta,\n            [\"sequencing_run_id\", \"run_id\", \"run\", \"sequencing_run\", \"seq_run_id\"],\n            default=pd.NA,\n        )\n        batch_id = self._pick_series(\n            meta,\n            [\"batch_id\", \"batch\", \"library_batch\", \"processing_batch\"],\n            default=pd.NA,\n        )\n        center_id = self._pick_series(\n            meta,\n            [\"center_id\", \"center\", \"site\", \"lab\", \"institution\"],\n            default=pd.NA,\n        )\n        locus = self._pick_series(meta, [\"locus\", \"gene_locus\"], default=\"UNK\")\n        chain = self._pick_series(meta, [\"chain\", \"chain_type\"], default=\"UNK\")\n        receptor_type = self._pick_series(meta, [\"receptor_type\", \"receptor\", \"airr_type\"], default=\"UNK\")\n        sex = self._pick_series(meta, [\"sex\", \"gender\"], default=pd.NA)\n        race = self._pick_series(meta, [\"race\", \"ethnicity\"], default=pd.NA)\n        age = self._pick_series(meta, [\"age\", \"age_years\"], default=pd.NA)\n\n        hla_A = self._pick_series(meta, [\"hla_a\", \"hla-a\", \"a\"], default=pd.NA)\n        hla_B = self._pick_series(meta, [\"hla_b\", \"hla-b\", \"b\"], default=pd.NA)\n        hla_C = self._pick_series(meta, [\"hla_c\", \"hla-c\", \"c\"], default=pd.NA)\n        hla_DRB1 = self._pick_series(meta, [\"hla_drb1\", \"hla-drb1\", \"drb1\"], default=pd.NA)\n\n        meta[\"subject_id\"] = clean_string_series(subject_id).fillna(meta[\"repertoire_id\"])\n        meta[\"sample_id\"] = clean_string_series(sample_id).fillna(meta[\"repertoire_id\"])\n        meta[\"sequencing_run_id\"] = clean_string_series(sequencing_run_id)\n        meta[\"batch_id\"] = clean_string_series(batch_id)\n        meta[\"center_id\"] = clean_string_series(center_id)\n        meta[\"locus\"] = clean_string_series(locus, default_value=\"UNK\").fillna(\"UNK\")\n        meta[\"chain\"] = clean_string_series(chain, default_value=\"UNK\").fillna(\"UNK\")\n        meta[\"receptor_type\"] = clean_string_series(receptor_type, default_value=\"UNK\").fillna(\"UNK\")\n        meta[\"sex\"] = clean_string_series(sex)\n        meta[\"race\"] = clean_string_series(race)\n        meta[\"age\"] = safe_numeric_series(age)\n        meta[\"hla_A\"] = clean_string_series(hla_A)\n        meta[\"hla_B\"] = clean_string_series(hla_B)\n        meta[\"hla_C\"] = clean_string_series(hla_C)\n        meta[\"hla_DRB1\"] = clean_string_series(hla_DRB1)\n\n        meta[\"split_group_key\"] = meta[\"sequencing_run_id\"].fillna(meta[\"subject_id\"]).astype(\"string\")\n\n        meta[\"processing_hash\"] = meta.apply(\n            lambda row: stable_hash(\n                f\"{row['sequencing_run_id']}|{row['batch_id']}|{row['center_id']}|{row['dataset_name']}\"\n            ),\n            axis=1,\n        )\n\n        out = meta[self.required_columns].copy()\n        out[\"label_positive\"] = pd.to_numeric(out[\"label_positive\"], errors=\"coerce\")\n        return out\n\n    def build_all(self, train_root: Path, test_root: Path) -> pd.DataFrame:\n        train_dirs, test_dirs = discover_dataset_dirs(train_root, test_root)\n        all_tables = []\n\n        for ds in train_dirs:\n            all_tables.append(self.build_dataset_table(ds, source=\"train\"))\n\n        for ds in test_dirs:\n            all_tables.append(self.build_dataset_table(ds, source=\"test\"))\n\n        if len(all_tables) == 0:\n            return pd.DataFrame(columns=self.required_columns)\n\n        canonical = pd.concat(all_tables, ignore_index=True)\n        canonical = canonical.sort_values([\"source\", \"dataset_name\", \"repertoire_id\"]).reset_index(drop=True)\n        return canonical\n\"\"\")\n\nfiles[\"src/splits.py\"] = dedent(\"\"\"\nfrom __future__ import annotations\n\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\n\ntry:\n    from sklearn.model_selection import StratifiedGroupKFold\n    HAS_SGKF = True\nexcept Exception:\n    HAS_SGKF = False\n\nfrom .utils import ensure_dir\n\n\ndef _assign_dataset_folds(df: pd.DataFrame, n_splits: int = 5, random_state: int = 42) -> pd.DataFrame:\n    out = df.copy().reset_index(drop=True)\n    out[\"fold\"] = -1\n    out[\"splitter\"] = \"NA\"\n    out[\"n_splits_used\"] = 1\n\n    if len(out) == 0:\n        return out\n\n    y = pd.to_numeric(out[\"label_positive\"], errors=\"coerce\").fillna(0).astype(int).values\n    class_counts = pd.Series(y).value_counts()\n\n    if len(class_counts) < 2:\n        out[\"fold\"] = 0\n        out[\"splitter\"] = \"single_class\"\n        out[\"n_splits_used\"] = 1\n        return out\n\n    min_class = int(class_counts.min())\n    n_splits_used = min(n_splits, min_class)\n\n    if n_splits_used < 2:\n        out[\"fold\"] = 0\n        out[\"splitter\"] = \"not_enough_samples\"\n        out[\"n_splits_used\"] = 1\n        return out\n\n    groups = out[\"split_group_key\"].astype(\"string\").fillna(out[\"subject_id\"].astype(\"string\")).values\n    use_group = HAS_SGKF and (pd.Series(groups).nunique() >= n_splits_used)\n\n    if use_group:\n        splitter = StratifiedGroupKFold(n_splits=n_splits_used, shuffle=True, random_state=random_state)\n        split_iter = splitter.split(np.zeros(len(out)), y, groups=groups)\n        splitter_name = \"StratifiedGroupKFold\"\n    else:\n        splitter = StratifiedKFold(n_splits=n_splits_used, shuffle=True, random_state=random_state)\n        split_iter = splitter.split(np.zeros(len(out)), y)\n        splitter_name = \"StratifiedKFold\"\n\n    for fold, (_, va_idx) in enumerate(split_iter):\n        out.loc[va_idx, \"fold\"] = fold\n\n    out[\"splitter\"] = splitter_name\n    out[\"n_splits_used\"] = n_splits_used\n    return out\n\n\ndef build_folds_for_all(canonical_meta_df: pd.DataFrame, out_dir: Path, n_splits: int = 5, random_state: int = 42):\n    out_dir = ensure_dir(out_dir)\n    train_df = canonical_meta_df[canonical_meta_df[\"source\"] == \"train\"].copy()\n    train_df = train_df[pd.notna(train_df[\"label_positive\"])].copy()\n\n    all_fold_tables = []\n\n    for ds_name, g in train_df.groupby(\"dataset_name\", sort=True):\n        fold_df = _assign_dataset_folds(g, n_splits=n_splits, random_state=random_state)\n        fold_path = out_dir / f\"folds_{ds_name}.csv\"\n        fold_df.to_csv(fold_path, index=False)\n        all_fold_tables.append(fold_df)\n\n    if len(all_fold_tables) == 0:\n        return pd.DataFrame()\n\n    all_folds = pd.concat(all_fold_tables, ignore_index=True)\n    all_folds.to_csv(out_dir / \"all_folds.csv\", index=False)\n    return all_folds\n\"\"\")\n\nfiles[\"src/drift.py\"] = dedent(\"\"\"\nfrom __future__ import annotations\n\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nfrom joblib import Parallel, delayed\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold\n\ntry:\n    from xgboost import XGBClassifier\n    HAS_XGB = True\nexcept Exception:\n    HAS_XGB = False\n\ntry:\n    from lightgbm import LGBMClassifier\n    HAS_LGB = True\nexcept Exception:\n    HAS_LGB = False\n\nfrom .data_io import quick_profile_repertoire\nfrom .utils import ensure_dir\n\n\nclass DriftAnalyzer:\n    def __init__(self, max_rows: int = 20000, n_jobs: int = 4, random_state: int = 42):\n        self.max_rows = max_rows\n        self.n_jobs = n_jobs\n        self.random_state = random_state\n\n    def _profile_one(self, row_dict: dict):\n        feats = quick_profile_repertoire(Path(row_dict[\"absolute_path\"]), max_rows=self.max_rows)\n        return {\n            \"dataset_name\": row_dict[\"dataset_name\"],\n            \"dataset_id\": row_dict[\"dataset_id\"],\n            \"repertoire_id\": row_dict[\"repertoire_id\"],\n            **feats,\n        }\n\n    def build_profile_table(self, meta_df: pd.DataFrame) -> pd.DataFrame:\n        base = meta_df[\n            [\"dataset_name\", \"dataset_id\", \"repertoire_id\", \"source\", \"absolute_path\"]\n        ].drop_duplicates().reset_index(drop=True)\n\n        rows = base.to_dict(orient=\"records\")\n        prof = Parallel(n_jobs=self.n_jobs, backend=\"loky\")(\n            delayed(self._profile_one)(r) for r in rows\n        )\n        prof_df = pd.DataFrame(prof)\n\n        merged = base.merge(\n            prof_df,\n            on=[\"dataset_name\", \"dataset_id\", \"repertoire_id\"],\n            how=\"left\",\n        )\n        return merged\n\n    def _make_model(self):\n        if HAS_XGB:\n            return XGBClassifier(\n                n_estimators=300,\n                max_depth=4,\n                learning_rate=0.05,\n                subsample=0.9,\n                colsample_bytree=0.8,\n                objective=\"binary:logistic\",\n                eval_metric=\"auc\",\n                tree_method=\"hist\",\n                random_state=self.random_state,\n                n_jobs=1,\n            )\n        if HAS_LGB:\n            return LGBMClassifier(\n                n_estimators=300,\n                max_depth=-1,\n                learning_rate=0.05,\n                num_leaves=31,\n                subsample=0.9,\n                colsample_bytree=0.8,\n                random_state=self.random_state,\n            )\n        return LogisticRegression(max_iter=2000, random_state=self.random_state)\n\n    def _extract_importance(self, model, feature_cols):\n        if hasattr(model, \"feature_importances_\"):\n            imp = np.asarray(model.feature_importances_, dtype=float)\n            return pd.DataFrame({\"feature\": feature_cols, \"importance\": imp}).sort_values(\n                \"importance\", ascending=False\n            )\n        if hasattr(model, \"coef_\"):\n            coef = np.abs(np.asarray(model.coef_).ravel())\n            return pd.DataFrame({\"feature\": feature_cols, \"importance\": coef}).sort_values(\n                \"importance\", ascending=False\n            )\n        return pd.DataFrame({\"feature\": feature_cols, \"importance\": np.zeros(len(feature_cols))})\n\n    def analyze_pair(self, train_meta: pd.DataFrame, test_meta: pd.DataFrame, out_dir: Path):\n        pair_meta = pd.concat([train_meta, test_meta], ignore_index=True)\n        profile = self.build_profile_table(pair_meta)\n        profile[\"drift_target\"] = (profile[\"source\"] == \"test\").astype(int)\n\n        feature_cols = [\n            c for c in profile.columns\n            if c not in {\"dataset_name\", \"dataset_id\", \"repertoire_id\", \"source\", \"absolute_path\", \"drift_target\"}\n        ]\n        X_df = profile[feature_cols].apply(pd.to_numeric, errors=\"coerce\").fillna(0.0)\n        y = profile[\"drift_target\"].astype(int).values\n\n        class_counts = pd.Series(y).value_counts()\n        if len(class_counts) < 2:\n            auc_mean = np.nan\n            auc_std = np.nan\n            model = self._make_model()\n            model.fit(X_df, y)\n            imp_df = self._extract_importance(model, feature_cols)\n            return auc_mean, auc_std, imp_df, profile\n\n        min_class = int(class_counts.min())\n        n_splits = min(5, min_class)\n\n        if n_splits < 2:\n            auc_mean = np.nan\n            auc_std = np.nan\n            model = self._make_model()\n            model.fit(X_df, y)\n            imp_df = self._extract_importance(model, feature_cols)\n            return auc_mean, auc_std, imp_df, profile\n\n        skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=self.random_state)\n        scores = []\n\n        for tr_idx, va_idx in skf.split(X_df, y):\n            model = self._make_model()\n            model.fit(X_df.iloc[tr_idx], y[tr_idx])\n\n            if hasattr(model, \"predict_proba\"):\n                pred = model.predict_proba(X_df.iloc[va_idx])[:, 1]\n            else:\n                pred = model.decision_function(X_df.iloc[va_idx])\n\n            auc = roc_auc_score(y[va_idx], pred)\n            scores.append(float(auc))\n\n        final_model = self._make_model()\n        final_model.fit(X_df, y)\n        imp_df = self._extract_importance(final_model, feature_cols)\n\n        return float(np.mean(scores)), float(np.std(scores)), imp_df, profile\n\n    def run(self, canonical_meta_df: pd.DataFrame, out_dir: Path):\n        out_dir = ensure_dir(out_dir)\n\n        train_df = canonical_meta_df[canonical_meta_df[\"source\"] == \"train\"].copy()\n        test_df = canonical_meta_df[canonical_meta_df[\"source\"] == \"test\"].copy()\n\n        common_ids = sorted(set(train_df[\"dataset_id\"].unique()) & set(test_df[\"dataset_id\"].unique()))\n        summary_rows = []\n\n        for ds_id in common_ids:\n            tr = train_df[train_df[\"dataset_id\"] == ds_id].copy()\n            te = test_df[test_df[\"dataset_id\"] == ds_id].copy()\n\n            if len(tr) == 0 or len(te) == 0:\n                continue\n\n            auc_mean, auc_std, imp_df, profile_df = self.analyze_pair(tr, te, out_dir=out_dir)\n\n            train_names = \",\".join(sorted(tr[\"dataset_name\"].astype(str).unique()))\n            test_names = \",\".join(sorted(te[\"dataset_name\"].astype(str).unique()))\n\n            imp_path = out_dir / f\"drift_feature_importance_dataset_{ds_id}.csv\"\n            imp_df.to_csv(imp_path, index=False)\n\n            profile_path = out_dir / f\"drift_profile_dataset_{ds_id}.csv\"\n            profile_df.to_csv(profile_path, index=False)\n\n            summary_rows.append(\n                {\n                    \"dataset_id\": ds_id,\n                    \"train_dataset_names\": train_names,\n                    \"test_dataset_names\": test_names,\n                    \"n_train\": int(len(tr)),\n                    \"n_test\": int(len(te)),\n                    \"drift_auc_mean\": auc_mean,\n                    \"drift_auc_std\": auc_std,\n                    \"importance_file\": str(imp_path),\n                }\n            )\n\n        summary_df = pd.DataFrame(summary_rows)\n        summary_df.to_csv(out_dir / \"drift_auc_by_dataset.csv\", index=False)\n        return summary_df\n\"\"\")\n\nfiles[\"run_phase1.py\"] = dedent(\"\"\"\nfrom __future__ import annotations\n\nimport sys\nfrom pathlib import Path\n\nimport yaml\n\nPROJECT_ROOT = Path(__file__).resolve().parent\nif str(PROJECT_ROOT) not in sys.path:\n    sys.path.insert(0, str(PROJECT_ROOT))\n\nfrom src.utils import ensure_dir, seed_everything\nfrom src.metadata import MetadataBuilder\nfrom src.splits import build_folds_for_all\nfrom src.drift import DriftAnalyzer\n\n\ndef main():\n    cfg = yaml.safe_load((PROJECT_ROOT / \"configs\" / \"base.yaml\").read_text())\n\n    paths = cfg[\"paths\"]\n    runtime = cfg[\"runtime\"]\n    phase1 = cfg[\"phase1\"]\n\n    train_root = Path(paths[\"train_root\"])\n    test_root = Path(paths[\"test_root\"])\n    out_root = ensure_dir(paths[\"output_root\"])\n\n    seed_everything(runtime[\"random_state\"])\n\n    print(\"=\" * 80)\n    print(\"PHASE 1: Standards-first data layer\")\n    print(\"=\" * 80)\n    print(\"Train root:\", train_root)\n    print(\"Test root :\", test_root)\n    print(\"Output    :\", out_root)\n\n    builder = MetadataBuilder()\n    canonical_meta = builder.build_all(train_root=train_root, test_root=test_root)\n    canonical_path = out_root / \"canonical_metadata.csv\"\n    canonical_meta.to_csv(canonical_path, index=False)\n    print(f\"Saved canonical metadata -> {canonical_path}\")\n    print(\"Canonical shape:\", canonical_meta.shape)\n\n    folds_df = build_folds_for_all(\n        canonical_meta_df=canonical_meta,\n        out_dir=out_root,\n        n_splits=phase1[\"n_splits\"],\n        random_state=runtime[\"random_state\"],\n    )\n    print(\"Saved fold files.\")\n    if len(folds_df) > 0:\n        print(\"Fold table shape:\", folds_df.shape)\n\n    analyzer = DriftAnalyzer(\n        max_rows=phase1[\"max_rows_profile\"],\n        n_jobs=runtime[\"n_jobs\"],\n        random_state=runtime[\"random_state\"],\n    )\n    drift_df = analyzer.run(canonical_meta_df=canonical_meta, out_dir=out_root)\n    print(\"Saved drift outputs.\")\n    if len(drift_df) > 0:\n        print(drift_df.to_string(index=False))\n\n    print(\"=\" * 80)\n    print(\"PHASE 1 DONE\")\n    print(\"=\" * 80)\n\n\nif __name__ == \"__main__\":\n    main()\n\"\"\")\n\nfor rel_path, content in files.items():\n    path = PROJECT_ROOT / rel_path\n    path.parent.mkdir(parents=True, exist_ok=True)\n    path.write_text(content.strip() + \"\\n\", encoding=\"utf-8\")\n\nprint(\"Phase 1 module files written.\")\nprint(\"Files created:\")\nfor rel_path in sorted(files.keys()):\n    print(\"-\", rel_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-20T14:55:28.721550Z","iopub.execute_input":"2026-07-20T14:55:28.721946Z","iopub.status.idle":"2026-07-20T14:55:28.741731Z","shell.execute_reply.started":"2026-07-20T14:55:28.721918Z","shell.execute_reply":"2026-07-20T14:55:28.740723Z"}},"outputs":[],"execution_count":null},{"id":"8ed2da9b","cell_type":"code","source":"#Cell 4 — imports verify করো\nimport sys\nfrom pathlib import Path\n\nPROJECT_ROOT = Path(\"/kaggle/working/project\")\nif str(PROJECT_ROOT) not in sys.path:\n    sys.path.insert(0, str(PROJECT_ROOT))\n\nfrom src.metadata import MetadataBuilder\nfrom src.splits import build_folds_for_all\nfrom src.drift import DriftAnalyzer\n\nprint(\"Imports OK\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-20T14:55:29.337836Z","iopub.execute_input":"2026-07-20T14:55:29.338089Z","iopub.status.idle":"2026-07-20T14:55:35.455220Z","shell.execute_reply.started":"2026-07-20T14:55:29.338066Z","shell.execute_reply":"2026-07-20T14:55:35.454537Z"}},"outputs":[],"execution_count":null},{"id":"7ec995e8","cell_type":"code","source":"#Cell 5 — Phase 1 run করো\n!python /kaggle/working/project/run_phase1.py","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-20T14:55:35.456729Z","iopub.execute_input":"2026-07-20T14:55:35.457243Z"}},"outputs":[],"execution_count":null},{"id":"804ea732","cell_type":"code","source":"#Cell 6 — outputs inspect করো\nfrom pathlib import Path\nimport pandas as pd\n\nOUT_DIR = Path(\"/kaggle/working/project/artifacts/phase1\")\n\nprint(\"Output files:\")\nfor p in sorted(OUT_DIR.glob(\"*\")):\n    print(\"-\", p.name)\n\nprint(\"\\ncanonical_metadata.csv\")\ndisplay(pd.read_csv(OUT_DIR / \"canonical_metadata.csv\").head())\n\nprint(\"\\ndrift_auc_by_dataset.csv\")\ndrift_path = OUT_DIR / \"drift_auc_by_dataset.csv\"\nif drift_path.exists():\n    display(pd.read_csv(drift_path))\nelse:\n    print(\"drift_auc_by_dataset.csv not found\")\n\nprint(\"\\nall_folds.csv\")\nall_folds_path = OUT_DIR / \"all_folds.csv\"\nif all_folds_path.exists():\n    display(pd.read_csv(all_folds_path).head())\nelse:\n    print(\"all_folds.csv not found\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"92ff664f-d72b-4ccd-b661-62311bd9fc5b","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"f193c390-12e8-46ec-9dcf-cd045f14cab1","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}