{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":67356,"databundleVersionId":8006601},{"sourceType":"datasetVersion","sourceId":15710684,"datasetId":10064972,"databundleVersionId":16650631},{"sourceType":"modelInstanceVersion","sourceId":829972,"databundleVersionId":16650597,"modelInstanceId":631258,"modelId":643154},{"sourceType":"modelInstanceVersion","sourceId":838941,"databundleVersionId":16780923,"modelInstanceId":638158,"modelId":650143},{"sourceType":"modelInstanceVersion","sourceId":847000,"databundleVersionId":16901617,"modelInstanceId":644070,"modelId":656033},{"sourceType":"modelInstanceVersion","sourceId":847815,"databundleVersionId":16912868,"modelInstanceId":644669,"modelId":656621},{"sourceType":"modelInstanceVersion","sourceId":822944,"databundleVersionId":16521652,"modelInstanceId":625490,"modelId":637354},{"sourceType":"kernelVersion","sourceId":171884838},{"sourceType":"kernelVersion","sourceId":312874589},{"sourceType":"kernelVersion","sourceId":315000306}],"dockerImageVersionId":31328,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install transformers torch rdkit\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport torch\nimport polars as pl\nimport pandas as pd\nfrom tqdm import tqdm\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.amp import autocast\nfrom transformers import DataCollatorWithPadding\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport numpy as np\nfrom tqdm import tqdm\nfrom transformers import AutoTokenizer, AutoModel\nfrom sklearn.metrics import average_precision_score\nfrom sklearn.model_selection import train_test_split\nimport pyarrow.parquet as pq\nfrom torch.amp import GradScaler","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n# --- 1. 數據優化抽樣 (Optimized Sampling) ---\n# 參數設定\nINPUT_FILE = \"/kaggle/input/competitions/leash-BELKA/train.csv\"\nOUTPUT_FILE = \"optimized_train_data.csv\"\nCHUNK_SIZE = 3_000_000\n\ndef csv_to_parquet_optimized(input_csv, output_parquet):\n    print(\"正在將 CSV 轉換為高效能 Parquet 格式...\")\n    # 使用 Polars 讀取，並指定數據類型以節省空間\n    df = pl.read_csv(input_csv, dtypes={\n        \"molecule_smiles\": pl.Utf8,\n        \"protein_name\": pl.Categorical,\n        \"binds\": pl.Int8\n    })\n    df.write_parquet(output_parquet, compression=\"snappy\")\n    print(f\"✅ 轉換完成: {output_parquet}\")\n    del df\n    gc.collect()\n\ndef optimized_sampling_to_parquet(input_parquet, output_parquet, chunk_size=3_000_000, neg_to_pos_ratio=16):\n    print(\"Sampling...\")\n    \n    # 關鍵動作：建立 Parquet 檔案物件\n    parquet_file = pq.ParquetFile(input_parquet)\n    \n    # ==========================================\n    # 第一趟 (Pass 1)：母體普查\n    # ==========================================\n    print(\"\\n [Pass 1] 正在進行全資料庫極速普查...\")\n    total_pos_count = 0\n    total_neg_count = 0\n    \n    # 使用 iter_batches 來分批讀取，並轉成 Pandas DataFrame\n    for batch in tqdm(parquet_file.iter_batches(batch_size=chunk_size, columns=['binds']), desc=\"Counting\"):\n        chunk = batch.to_pandas()\n        total_pos_count += (chunk['binds'] == 1).sum()\n        total_neg_count += (chunk['binds'] == 0).sum()\n        \n    print(f\"母體總數 - 正樣本: {total_pos_count:,} | 負樣本: {total_neg_count:,}\")\n    \n    # 精算抽樣率\n    target_neg_count = total_pos_count * neg_to_pos_ratio\n    exact_sampling_rate = target_neg_count / total_neg_count\n    \n    print(f\" 目標負樣本數: {target_neg_count:,} (比例 15:1)\")\n    print(f\" 算出抽樣機率: {exact_sampling_rate:.8f}\")\n\n    # ==========================================\n    # 第二趟 (Pass 2)：精準資料提取\n    # ==========================================\n    print(\"\\n [Pass 2] 正在進行精準資料提取與抽樣...\")\n    use_cols = ['molecule_smiles', 'protein_name', 'binds', 'buildingblock1_smiles']\n    \n    pos_chunks = []\n    neg_chunks = []\n\n    for batch in tqdm(parquet_file.iter_batches(batch_size=chunk_size, columns=use_cols), desc=\"Extracting & Sampling\"):\n        chunk = batch.to_pandas()\n        \n        # 1. 抓出所有正樣本\n        pos_chunks.append(chunk[chunk['binds'] == 1])\n        \n        # 2. 抓出負樣本並使用機率抽樣\n        neg_subset = chunk[chunk['binds'] == 0]\n        neg_chunks.append(neg_subset.sample(frac=exact_sampling_rate, random_state=2))\n\n    print(\"\\n 讀取完畢！正在合併數據...\")\n    df_pos = pd.concat(pos_chunks)\n    df_neg = pd.concat(neg_chunks)\n    \n    # 合併並洗牌\n    df_final = pd.concat([df_pos, df_neg]).sample(frac=1, random_state=2).reset_index(drop=True)\n\n    print(f\"最終樣本數: {len(df_final):,} (正: {len(df_pos):,}, 負: {len(df_neg):,})\")\n    \n    # ==========================================\n    # 寫入與輸出\n    # ==========================================\n    print(\"正在轉換為 Polars 並寫入高效能 Parquet...\")\n    \n    pl_df = pl.from_pandas(df_final).with_columns([\n        pl.col(\"molecule_smiles\").cast(pl.Utf8),\n        pl.col(\"buildingblock1_smiles\").cast(pl.Utf8),  \n        pl.col(\"protein_name\").cast(pl.Categorical),    \n        pl.col(\"binds\").cast(pl.Int8)\n    ])\n    \n    pl_df.write_parquet(output_parquet, compression=\"snappy\")\n    \n    print(f\"完美轉換！檔案已儲存至 {output_parquet}\")\n    \n    del df_pos, df_neg, df_final, pl_df\n    gc.collect()\n\n\n\noptimized_sampling_to_parquet(\"/kaggle/input/competitions/leash-BELKA/train.parquet\", \"optimized_train_data.parquet\")\n\ngc.collect()\n\nif torch.cuda.is_available():\n    torch.cuda.empty_cache()\n'''","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nimport polars as pl\nimport gc\n\ndef merge_and_shuffle_data(main_parquet_path, external_csv_path, output_path, read_count_threshold=15):\n    print(\"[Phase 1] 啟動外部資料融合...\")\n    \n    # 1. 讀取主資料集\n    print(f\"讀取主資料集: {main_parquet_path}\")\n    df_main = pl.read_parquet(main_parquet_path)\n    \n    # 2. 讀取外部資料並進行嚴格篩選\n    print(f\"📥 讀取外部資料: {external_csv_path}\")\n    df_ext_raw = pl.read_csv(external_csv_path)\n    \n    # 過濾低頻雜訊 \n    initial_count = len(df_ext_raw)\n    df_ext_pos_filtered = df_ext_raw.filter(pl.col(\"read_count\") >= read_count_threshold)\n    print(f\"pos_filtered -> {len(df_ext_pos_filtered)} 筆)\")\n    df_ext_neg_filtered = df_ext_raw.filter(pl.col(\"read_count\") == 0)\n    print(f\"neg_filtered -> {len(df_ext_neg_filtered)} 筆)\")\n    \n    # 保留 [Dy] 並使用 iso 立體結構\n    df_positive = df_ext_pos_filtered.select([\n        pl.col(\"new_structure\").alias(\"molecule_smiles\"),         # 完美保留 [Dy]\n        pl.col(\"bb1_iso\").alias(\"buildingblock1_smiles\"),         # 完美保留立體化學特徵\n        pl.lit(\"sEH\").cast(pl.Categorical).alias(\"protein_name\"), \n        pl.lit(1).cast(pl.Int8).alias(\"binds\")                    # 因為經過高 read_count 篩選，現在可以安心給 1\n    ])\n\n    df_negative = df_ext_neg_filtered.select([\n        pl.col(\"new_structure\").alias(\"molecule_smiles\"),         # 完美保留 [Dy]\n        pl.col(\"bb1_iso\").alias(\"buildingblock1_smiles\"),         # 完美保留立體化學特徵\n        pl.lit(\"sEH\").cast(pl.Categorical).alias(\"protein_name\"), \n        pl.lit(0).cast(pl.Int8).alias(\"binds\")                    # 0\n    ])\n\n    df_ext = pl.concat([df_positive, df_negative])\n    \n    # 3. 合併與洗牌\n    print(\"正在合併與全局洗牌...\")\n    df_main_subset = df_main.select(df_ext.columns) \n    df_combined = pl.concat([df_main_subset, df_ext])\n    df_combined = df_combined.sample(fraction=1.0, shuffle=True, seed=42)\n\n    # 5. 寫出 Parquet\n    print(f\"正在寫入最終訓練檔至: {output_path}\")\n    df_combined.write_parquet(output_path, compression=\"snappy\")\n    \n    print(f\"總樣本數：{len(df_combined):,} (新增 {len(df_ext_pos_filtered):,} 筆強效正樣本)(新增 {len(df_ext_neg_filtered):,} 筆嚴格負樣本)\")\n    \n    del df_main, df_ext_raw, df_ext_pos_filtered, df_ext_neg_filtered, df_positive, df_negative, df_ext, df_combined, df_main_subset\n    gc.collect()\n\n\n\n# 執行合併 \nmerge_and_shuffle_data(\"optimized_train_data.parquet\", \"/kaggle/input/notebooks/chemdatafarmer/additional-seh-data/DNA_Labeled_Data.csv\", \"final_merged_train_data.parquet\", read_count_threshold=15)\n'''","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==================================================================\n# BELKA — Model 1 (Multi-Scale CNN + FiLM Peri-LN) 完整 Pipeline\n# ==================================================================\nimport os, gc, time, re\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.metrics import average_precision_score\nfrom transformers import AutoTokenizer, AutoModel\nfrom tqdm import tqdm\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==================================================================\n# 1. 資料載入 + BB-aware 切分 (StratifiedGroupKFold by BB1)\n# ==================================================================\ndef load_and_split_data(parquet_path, n_splits=5):\n    df = pl.read_parquet(parquet_path)\n    print(\"BB1 null 數:\", df['buildingblock1_smiles'].null_count())   # 確認外部資料有 BB1\n\n    sgkf = StratifiedGroupKFold(n_splits=n_splits, shuffle=True, random_state=42)\n    y      = df['binds'].to_numpy()\n    groups = df['buildingblock1_smiles'].to_numpy()\n    X      = np.zeros(len(y))\n\n    tr_idx, va_idx = next(iter(sgkf.split(X, y, groups)))\n    train_df = df[tr_idx.tolist()].to_pandas()\n    val_df   = df[va_idx.tolist()].to_pandas()\n\n    leak = set(train_df['buildingblock1_smiles']) & set(val_df['buildingblock1_smiles'])\n    print(f\"訓練: {len(train_df):,} | 驗證: {len(val_df):,} | BB1 洩漏: {len(leak)}\")\n    print(f\"val 正例比例: {val_df['binds'].mean():.4%}  (若 ~6% 代表已 RUS,AP 會虛高)\")\n\n    return train_df.drop(columns=['buildingblock1_smiles']), \\\n           val_df.drop(columns=['buildingblock1_smiles'])\n\nTRAIN_PARQUET = \"/kaggle/input/notebooks/t8101349/predict-new-medicines-with-belka-rusessemble-data3/final_merged_train_data.parquet\"   ###\ntrain_df, val_df = load_and_split_data(TRAIN_PARQUET, n_splits=5)\ngc.collect()\n\n# ==================================================================\n# 2. 蛋白質 Embedding 預快取 (ESM-2, mean pool) → 3 個固定向量\n# ==================================================================\nPROT_MODEL_NAME = \"facebook/esm2_t12_35M_UR50D\"\n\ndef precompute_prot_embeddings():\n    tok = AutoTokenizer.from_pretrained(PROT_MODEL_NAME)\n    mdl = AutoModel.from_pretrained(PROT_MODEL_NAME).to(device).eval()\n    seqs = {\n        'BRD4': \"MSAESGPGTRLRNLPVMGDGLETSQMSTTQAQAQPQPANAASTNPPPPETSNPNKPKRQTNQLQYLLRVVLKTLWKHQFAWPFQQPVDAVKLNLPDYYKIIKTPMDMGTIKKRLENNYYWNAQECIQDFNTMFTNCYIYNKPGDDIVLMAEALEKLFLQKINELPTEETEIMIVQAKGRGRGRKETGTAKPGVSTVPNTTQASTPPQTQTPQPNPPPVQATPHPFPAVTPDLIVQTPVMTVVPPQPLQTPPPVPPQPQPPPAPAPQPVQSHPPIIAATPQPVKTKKGVKRKADTTTPTTIDPIHEPPSLPPEPKTTKLGQRRESSRPVKPPKKDVPDSQQHPAPEKSSKVSEQLKCCSGILKEMFAKKHAAYAWPFYKPVDVEALGLHDYCDIIKHPMDMSTIKSKLEAREYRDAQEFGADVRLMFSNCYKYNPPDHEVVAMARKLQDVFEMRFAKMPDEPEEPVVAVSSPAVPPPTKVVAPPSSSDSSSDSSSDSDSSTDDSEEERAQRLAELQEQLKAVHEQLAALSQPQQNKPKKKKKKKKKKKK\",\n        'HSA':  \"MKWVTFISLLFLFSSAYSRGVFRRDAHKSEVAHRFKDLGEENFKALVLIAFAQYLQQCPFEDHVKLVNEVTEFAKTCVADESAENCDKSLHTLFGDKLCTVATLRETYGEMADCCAKQEPERNECFLQHKDDNPNLPRLVRPEVDVMCTAFHDNEETFLKKYLYEIARRHPYFYAPELLFFAKRYKAAFTECCQAADKAACLLPKLDELRDEGKASSAKQRLKCASLQKFGERAFKAWAVARLSQRFPKAEFAEVSKLVTDLTKVHTECCHGDLLECADDRADLAKYICENQDSISSKLKECCEKPLLEKSHCIAEVENDEMPADLPSLAADFVESKDVCKNYAEAKDVFLGMFLYEYARRHPDYSVVLLLRLAKTYETTLEKCCAAADPHECYAKVFDEFKPLVEEPQNLIKQNCELFEQLGEYKFQNALLVRYTKKVPQVSTPTLVEVSRNLGKVGSKCCKHPEAKRMPCAEDYLSVVLNQLCVLHEKTPVSDRVTKCCTESLVNRRPCFSALEVDETYVPKEFNAETFTFHADICTLSEKERQIKKQTALVELVKHKPKATKEQLKAVMDDFAAFVEKCCKADDKETCFAEEGKKLVAASQAALGL\",\n        'sEH':  \"MTLRAAVFDLDGVLALPAVFGVLGRTEEALALPRGLLNDAFQKGGPEGATTRLMKGEITLSQWIPLMEENCRKCSETAKVCLPKNFSIKEIFDKAISARKINRPMLQAALMLRKKGFTTAILTNTWLDDRAERDGLAQLMCELKMHFDFLIESCQVGMVKPEPQIYKFLLDTLKASPSEVVFLDDIGANLKPARDLGMVTILVQDTDTALKELEKVTGIQLLNTPAPLPTSCNPSDMSHGYVTVKPRVRLHFVELGSGPAVCLCHGFPESWYSWRYQIPALAQAGYRVLAMDMKGYGESSAPPEIEEYCMEVLCKEMVTFLDKLGLSQAVFIGHDWGGMLVWYMALFYPERVRAVASLNTPFIPANPNMSPLESIKANPVFDYQLYFQEPGVAEAELEQNLSRTFKSLFRASDESVLSMHKVCEAGGLFVNSPEEPSLSRMVTEEEIQFYVQQFKKSGFRGPLNWYRNMERNWKWACKSLGRKILIPALMVTAEKDFVLVPQMSQHMEDWIPHLKRGHIEDCGHWTQMDKPTEVNQILIKWLDSDARNPPVVSKM\"\n    }\n    out = {}\n    for name, seq in seqs.items():\n        enc = tok(seq, return_tensors=\"pt\", truncation=True, max_length=1024).to(device)\n        with torch.no_grad():\n            out[name] = mdl(**enc).last_hidden_state.mean(dim=1).cpu().numpy().squeeze()\n    del mdl, tok; torch.cuda.empty_cache()\n    return out\n\nprot_map = precompute_prot_embeddings()\nPROT_DIM = prot_map['BRD4'].shape[0]   # 480\nprint(f\"蛋白向量維度: {PROT_DIM}\")\n\n# ==================================================================\n# 3. 字元 Tokenizer \n# ==================================================================\nclass SmartSmilesTokenizer:\n    PATTERN = re.compile(r\"(\\[[^\\]]+\\]|Br|Cl|Si|Se|se|%\\d{2}|.)\")\n    def __init__(self):\n        specials  = ['<PAD>', '<UNK>']\n        organic   = ['C','N','O','S','P','F','I','B','Cl','Br','Si','Se']\n        aromatic  = ['c','n','o','s','p','se']\n        bonds     = ['-','=','#','/','\\\\','.',':']\n        rings     = list('0123456789') + [f'%{i:02d}' for i in range(10,100)]\n        branch    = ['(',')']\n        bracketed = ['[Dy]','[nH]','[N+]','[N-]','[O-]','[O+]','[NH+]','[NH2+]','[NH3+]',\n                     '[C@H]','[C@@H]','[C@]','[C@@]','[S+]','[s+]','[CH]','[CH2]','[CH-]',\n                     '[c-]','[n+]','[n-]','[se]','[B-]','[P+]','[P-]']\n        vocab = list(dict.fromkeys(specials+organic+aromatic+bonds+rings+branch+bracketed))\n        self.vocab = {t:i for i,t in enumerate(vocab)}\n        self.pad_id, self.unk_id = 0, 1\n        self.vocab_size = len(vocab)\n\n    def encode(self, s, max_length):\n        ids = [self.vocab.get(t, self.unk_id) for t in self.PATTERN.findall(s)]\n        return ids[:max_length]\n\n    def __call__(self, smiles_list, max_length=160):\n        batch = [self.encode(s, max_length) for s in smiles_list]\n        L = max(len(t) for t in batch)\n        ids  = torch.tensor([t + [self.pad_id]*(L-len(t)) for t in batch], dtype=torch.long)\n        mask = (ids != self.pad_id).long()\n        return {\"input_ids\": ids, \"attention_mask\": mask}\n\nmol_tokenizer = SmartSmilesTokenizer()\nprint(f\"Vocab size: {mol_tokenizer.vocab_size}\")\nMAX_LEN = 170   \n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==================================================================\n# 4. Dataset + Collate\n# ==================================================================\nclass BELKADynamicDataset(Dataset):\n    def __init__(self, df, prot_map):\n        self.smiles     = df['molecule_smiles'].values\n        self.prot_names = df['protein_name'].values\n        self.labels     = df['binds'].values\n        self.prot_map   = prot_map\n    def __len__(self): return len(self.labels)\n    def __getitem__(self, i):\n        return {\n            \"smiles\":   self.smiles[i],\n            \"prot_emb\": torch.tensor(self.prot_map[self.prot_names[i]], dtype=torch.float32),\n            \"label\":    torch.tensor([self.labels[i]], dtype=torch.float32),\n        }\n\ndef make_cnn_collate(tokenizer, max_length=MAX_LEN):\n    def collate(batch):\n        enc = tokenizer([b[\"smiles\"] for b in batch], max_length=max_length)\n        return {\n            \"input_ids\":      enc[\"input_ids\"],\n            \"attention_mask\": enc[\"attention_mask\"],\n            \"prot_emb\":       torch.stack([b[\"prot_emb\"] for b in batch]),\n            \"labels\":         torch.stack([b[\"label\"]    for b in batch]),\n        }\n    return collate\n\n# ==================================================================\n# 5. 模型:MultiScaleCNN + FiLM Peri-LN Gated + CLS readout\n# ==================================================================\nclass MultiScaleCNN(nn.Module):\n    \"\"\"並行 k=3/5/7 抓不同尺度官能基\"\"\"\n    def __init__(self, in_dim, out_dim):\n        super().__init__()\n        per, rem = out_dim // 3, out_dim - 2*(out_dim//3)\n        self.c3 = nn.Conv1d(in_dim, per, 3, padding=1)\n        self.c5 = nn.Conv1d(in_dim, per, 5, padding=2)\n        self.c7 = nn.Conv1d(in_dim, rem, 7, padding=3)\n        self.norm = nn.LayerNorm(out_dim); self.act = nn.GELU()\n    def forward(self, x):\n        x = x.transpose(1, 2)\n        x = torch.cat([self.c3(x), self.c5(x), self.c7(x)], dim=1).transpose(1, 2)\n        return self.norm(self.act(x))\n\nclass FiLMPeriLNGatedLayer(nn.Module):\n    \"\"\"Peri-LN + GEGLU + ReZero α + 蛋白質 FiLM 調變 FFN 輸出\"\"\"\n    def __init__(self, d_model, nhead, dim_ff, prot_dim, dropout=0.1):\n        super().__init__()\n        self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout, batch_first=True)\n        self.lin1 = nn.Linear(d_model, dim_ff*2)\n        self.lin2 = nn.Linear(dim_ff, d_model)\n        self.n_ai, self.n_ao = nn.LayerNorm(d_model), nn.LayerNorm(d_model)\n        self.n_fi, self.n_fo = nn.LayerNorm(d_model), nn.LayerNorm(d_model)\n        self.drop = nn.Dropout(dropout)\n        self.a_attn = nn.Parameter(torch.zeros(1))\n        self.a_ffn  = nn.Parameter(torch.zeros(1))\n        self.film = nn.Sequential(nn.Linear(prot_dim, d_model*2), nn.GELU(),\n                                  nn.Linear(d_model*2, d_model*2))\n        nn.init.zeros_(self.film[-1].weight); nn.init.zeros_(self.film[-1].bias)  # 初始 γ=β=0\n    def forward(self, src, prot_cond, pad_mask=None):\n        nx = self.n_ai(src)\n        a, _ = self.self_attn(nx, nx, nx, key_padding_mask=pad_mask, need_weights=False)\n        src = src + self.a_attn * self.drop(self.n_ao(a))\n        nx = self.n_fi(src)\n        h, g = self.lin1(nx).chunk(2, dim=-1)\n        ffn = self.n_fo(self.lin2(h * F.gelu(g)))\n        gamma, beta = self.film(prot_cond).chunk(2, dim=-1)\n        ffn = ffn * (1 + gamma.unsqueeze(1)) + beta.unsqueeze(1)\n        return src + self.a_ffn * self.drop(ffn)\n\nclass BELKAFiLMHybridModel(nn.Module):\n    def __init__(self, vocab_size, embed_dim=128, d_model=256, prot_dim=480,\n                 max_len=256, num_layers=4, nhead=8, dropout=0.1):\n        super().__init__()\n        self.embedding = nn.Embedding(vocab_size, embed_dim, padding_idx=0)\n        nn.init.trunc_normal_(self.embedding.weight, std=0.02)\n        self.cnn = MultiScaleCNN(embed_dim, d_model)\n        self.pos_emb = nn.Embedding(max_len + 1, d_model)\n        nn.init.trunc_normal_(self.pos_emb.weight, std=0.02)\n        self.prot_proj = nn.Sequential(nn.LayerNorm(prot_dim), nn.Linear(prot_dim, d_model),\n                                       nn.GELU(), nn.Linear(d_model, d_model))\n        self.cls = nn.Parameter(torch.randn(1, 1, d_model) * 0.02)\n        self.layers = nn.ModuleList([\n            FiLMPeriLNGatedLayer(d_model, nhead, d_model*4, d_model, dropout)\n            for _ in range(num_layers)])\n        self.final_norm = nn.LayerNorm(d_model)\n        self.head = nn.Sequential(\n            nn.Linear(d_model, 256), nn.LayerNorm(256), nn.GELU(), nn.Dropout(dropout*3),\n            nn.Linear(256, 128),     nn.LayerNorm(128), nn.GELU(), nn.Dropout(dropout*2),\n            nn.Linear(128, 1))\n\n    def forward(self, input_ids, attention_mask, prot_emb):\n        B, L = input_ids.size()\n        x = self.embedding(input_ids)\n        x = self.cnn(x)\n        x = x * attention_mask.unsqueeze(-1).float()          # 先把 PAD 位置歸零\n        cls = self.cls.expand(B, -1, -1)\n        x = torch.cat([cls, x], dim=1)                        # (B, L+1, D)\n        pos = torch.arange(L + 1, device=x.device).unsqueeze(0).expand(B, -1)\n        x = x + self.pos_emb(pos)\n        prot_cond = self.prot_proj(prot_emb)                  # (B, D)\n        cls_pad = torch.zeros(B, 1, dtype=torch.bool, device=x.device)\n        full_pad = torch.cat([cls_pad, attention_mask == 0], dim=1)\n        for layer in self.layers:\n            x = layer(x, prot_cond, pad_mask=full_pad)\n        x = self.final_norm(x)\n        return self.head(x[:, 0, :])                          # CLS readout","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==================================================================\n# 6. FocalLoss + EMA + 訓練迴圈\n# ==================================================================\nclass FocalLoss(nn.Module):\n    def __init__(self, alpha=0.25, gamma=2.0, label_smoothing=0.01):\n        super().__init__()\n        self.alpha, self.gamma, self.ls = alpha, gamma, label_smoothing\n        self.bce = nn.BCEWithLogitsLoss(reduction='none')\n    def forward(self, inputs, targets):\n        targets = targets.view(-1, 1).float(); inputs = inputs.view(-1, 1).float()\n        st = targets*(1-self.ls) + 0.5*self.ls if self.ls > 0 else targets\n        bce = self.bce(inputs, st)\n        p = torch.sigmoid(inputs)\n        p_t = p*targets + (1-p)*(1-targets)                   # 硬標籤\n        at  = self.alpha*targets + (1-self.alpha)*(1-targets) # 硬標籤\n        return (at * (1-p_t)**self.gamma * bce).mean()\n\nclass ModelEMA:\n    def __init__(self, model, decay=0.999):\n        self.decay = decay; self.n = 0\n        base = model.module if hasattr(model, \"module\") else model\n        self.shadow = {n: p.data.clone() for n, p in base.named_parameters() if p.requires_grad}\n    @torch.no_grad()\n    def update(self, model):\n        self.n += 1; d = min(self.decay, (1+self.n)/(10+self.n))\n        base = model.module if hasattr(model, \"module\") else model\n        for n, p in base.named_parameters():\n            if n in self.shadow: self.shadow[n].mul_(d).add_(p.data, alpha=1-d)\n    @torch.no_grad()\n    def swap_in(self, model):\n        base = model.module if hasattr(model, \"module\") else model\n        bk = {}\n        for n, p in base.named_parameters():\n            if n in self.shadow: bk[n] = p.data.clone(); p.data.copy_(self.shadow[n])\n        return bk\n    @torch.no_grad()\n    def swap_out(self, model, bk):\n        base = model.module if hasattr(model, \"module\") else model\n        for n, p in base.named_parameters():\n            if n in bk: p.data.copy_(bk[n])\n\ndef train_and_validate(model, train_loader, val_loader, optimizer, criterion, device,\n                       epochs=15, max_lr=5e-4, save_name=\"best_cnn.pth\",\n                       ema_decay=0.999, time_limit_hours=10.0):\n    scheduler = torch.optim.lr_scheduler.OneCycleLR(\n        optimizer, max_lr=max_lr, steps_per_epoch=len(train_loader),\n        epochs=epochs, pct_start=0.1, anneal_strategy='cos')\n    scaler = torch.amp.GradScaler('cuda')\n    ema = ModelEMA(model, decay=ema_decay)\n    best_ap, t0 = 0.0, time.time()\n\n    for epoch in range(epochs):\n        model.train(); tl = 0\n        pbar = tqdm(train_loader, desc=f\"Epoch {epoch+1}/{epochs} [Train]\")\n        for batch in pbar:\n            ids  = batch[\"input_ids\"].to(device, non_blocking=True)\n            mask = batch[\"attention_mask\"].to(device, non_blocking=True)\n            prot = batch[\"prot_emb\"].to(device, non_blocking=True)\n            lab  = batch[\"labels\"].to(device, non_blocking=True)\n            optimizer.zero_grad(set_to_none=True)\n            with torch.amp.autocast('cuda'):\n                logits = model(input_ids=ids, attention_mask=mask, prot_emb=prot)\n                loss = criterion(logits.float(), lab.view_as(logits).float())\n            scaler.scale(loss).backward()\n            scaler.unscale_(optimizer)\n            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n            scaler.step(optimizer); scaler.update(); scheduler.step()\n            ema.update(model)\n            tl += loss.item()\n            pbar.set_postfix(loss=f\"{loss.item():.4f}\", lr=f\"{scheduler.get_last_lr()[0]:.2e}\")\n\n        bk = ema.swap_in(model); model.eval()\n        preds, labs, vl = [], [], 0\n        with torch.no_grad():\n            for batch in tqdm(val_loader, desc=f\"Epoch {epoch+1} [Val-EMA]\"):\n                ids  = batch[\"input_ids\"].to(device); mask = batch[\"attention_mask\"].to(device)\n                prot = batch[\"prot_emb\"].to(device); lab = batch[\"labels\"].to(device)\n                with torch.amp.autocast('cuda'):\n                    logits = model(input_ids=ids, attention_mask=mask, prot_emb=prot)\n                    vl += criterion(logits.float(), lab.view_as(logits).float()).item()\n                preds.append(torch.sigmoid(logits.float()).cpu().numpy().ravel())\n                labs.append(lab.cpu().numpy().ravel())\n        ema.swap_out(model, bk)\n        ap = average_precision_score(np.concatenate(labs), np.concatenate(preds))\n        print(f\"\\nEpoch {epoch+1}: Train {tl/len(train_loader):.4f} | \"\n              f\"Val {vl/len(val_loader):.4f} | AP(EMA) {ap:.4f}\")\n\n        if ap > best_ap:\n            best_ap = ap\n            base = model.module if hasattr(model, \"module\") else model\n            bk2 = ema.swap_in(model)\n            torch.save(base.state_dict(), save_name)          # 全部存 (無凍結 backbone)\n            ema.swap_out(model, bk2)\n            print(f\"  ✓ saved {save_name} (AP={best_ap:.4f})\")\n        if (time.time()-t0)/3600 > time_limit_hours:\n            print(\"時間到,停止\"); break\n    return best_ap\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==================================================================\n# 7. 啟動訓練\n# ==================================================================\nBATCH_SIZE = 2048\ncollate = make_cnn_collate(mol_tokenizer, MAX_LEN)\ntrain_dataset = BELKADynamicDataset(train_df, prot_map)\nval_dataset   = BELKADynamicDataset(val_df,   prot_map)\ntrain_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True,\n                          num_workers=4, pin_memory=True, drop_last=True, collate_fn=collate)\nval_loader   = DataLoader(val_dataset,   batch_size=BATCH_SIZE, shuffle=False,\n                          num_workers=4, pin_memory=True, collate_fn=collate)\n\nmodel = BELKAFiLMHybridModel(vocab_size=mol_tokenizer.vocab_size, embed_dim=128,\n                             d_model=256, prot_dim=PROT_DIM, num_layers=4, nhead=8,\n                             dropout=0.1).to(device)\nmodel.load_state_dict(torch.load(\"/kaggle/input/models/t8101349/cnn-model/pytorch/default/1/best_cnn_film.pth\", map_location=device), strict=True)\n\nif torch.cuda.device_count() > 1:\n    print(f\"{torch.cuda.device_count()} GPU → DataParallel\")\n    model = nn.DataParallel(model)\nprint(f\"參數量: {sum(p.numel() for p in model.parameters())/1e6:.2f}M\")\n\noptimizer = torch.optim.AdamW(model.parameters(), lr=5e-4, weight_decay=1e-2)\ncriterion = FocalLoss(alpha=0.25, gamma=2.0, label_smoothing=0.01)\nbest_ap = train_and_validate(model, train_loader, val_loader, optimizer, criterion, device,\n                             epochs=15, max_lr=5e-4, save_name=\"best_cnn_film.pth\")\nprint(f\"\\n✅ 訓練完成,最佳 EMA AP: {best_ap:.4f}\")\n\n# ==================================================================\n# 8. submission \n# ==================================================================\nclass BELKATestDataset(Dataset):\n    def __init__(self, df, prot_map):\n        self.ids = df['id'].to_numpy(); self.smiles = df['molecule_smiles'].to_numpy()\n        self.prot_names = df['protein_name'].to_numpy(); self.prot_map = prot_map\n    def __len__(self): return len(self.smiles)\n    def __getitem__(self, i):\n        return {\"id\": self.ids[i], \"smiles\": self.smiles[i],\n                \"prot_emb\": torch.tensor(self.prot_map[self.prot_names[i]], dtype=torch.float32)}\n\ndef test_collate(batch):\n    enc = mol_tokenizer([b[\"smiles\"] for b in batch], max_length=MAX_LEN)\n    return {\"ids\": [b[\"id\"] for b in batch], \"input_ids\": enc[\"input_ids\"],\n            \"attention_mask\": enc[\"attention_mask\"],\n            \"prot_emb\": torch.stack([b[\"prot_emb\"] for b in batch])}\n\ndef generate_submission(model, test_file, prot_map, device, out=\"submission_cnn.csv\"):\n    model.eval()\n    test_df = pl.read_parquet(test_file, columns=[\"id\",\"molecule_smiles\",\"protein_name\"])\n    loader = DataLoader(BELKATestDataset(test_df, prot_map), batch_size=8192, shuffle=False,\n                        num_workers=4, pin_memory=True, collate_fn=test_collate)\n    ids, chunks = [], []\n    with torch.no_grad():\n        for batch in tqdm(loader, desc=\"Predicting\"):\n            with torch.amp.autocast('cuda'):\n                logits = model(input_ids=batch[\"input_ids\"].to(device),\n                               attention_mask=batch[\"attention_mask\"].to(device),\n                               prot_emb=batch[\"prot_emb\"].to(device))\n            chunks.append(torch.sigmoid(logits.float()).cpu().numpy().ravel())\n            ids.extend(batch[\"ids\"])\n    pd.DataFrame({\"id\": ids, \"binds\": np.concatenate(chunks)}).to_csv(out, index=False)\n    print(f\"✅ {out}: {len(ids):,} 筆\"); del test_df; gc.collect()\n\n\ninfer_model = BELKAFiLMHybridModel(vocab_size=mol_tokenizer.vocab_size, embed_dim=128,\n                                   d_model=256, prot_dim=PROT_DIM, num_layers=4, nhead=8,\n                                   dropout=0.1).to(device)\ninfer_model.load_state_dict(torch.load(\"best_cnn_film.pth\", map_location=device), strict=True)\nprint(\"✅ 權重載入 (strict=True 通過)\")\n\nTEST_FILE = \"/kaggle/input/competitions/leash-BELKA/test.parquet\"\nif os.path.exists(TEST_FILE):\n    generate_submission(infer_model, TEST_FILE, prot_map, device, out=\"submission_cnn.csv\")\n\n\n    sub = pl.read_csv(\"submission_cnn.csv\")\n    print(\"id 遞增:\", sub['id'].is_sorted(), \"| binds:\", \n          round(sub['binds'].min(),5), \"~\", round(sub['binds'].max(),5))\n    meta = pl.read_parquet(TEST_FILE, columns=[\"id\",\"protein_name\"])\n    print(sub.join(meta, on=\"id\").group_by(\"protein_name\").agg(pl.col(\"binds\").mean()))\nelse:\n    print(f\"❌ 找不到 {TEST_FILE}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\n\nSAMPLE_SUB_FILE = \"/kaggle/input/competitions/leash-BELKA/sample_submission.csv\"\nif os.path.exists(\"submission_cnn.csv\") and os.path.exists(SAMPLE_SUB_FILE):\n    sub_count = pl.scan_csv(\"submission_cnn.csv\").select(pl.len()).collect().item()\n    sample_count = pl.scan_csv(SAMPLE_SUB_FILE).select(pl.len()).collect().item()\n    \n    if sub_count == sample_count:\n        print(f\"✅ 行數檢查完美通過: {sub_count} 筆數據\")\n    else:\n        print(f\"❌ 警告：行數不符！提交檔: {sub_count}, 範本: {sample_count}\")\n\nprint(\"ID 是否嚴格遞增且無遺漏：\", sub['id'].is_sorted())\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}