{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":67356,"databundleVersionId":8006601},{"sourceType":"datasetVersion","sourceId":15710684,"datasetId":10064972,"databundleVersionId":16650631},{"sourceType":"modelInstanceVersion","sourceId":829972,"databundleVersionId":16650597,"modelInstanceId":631258,"modelId":643154},{"sourceType":"modelInstanceVersion","sourceId":838941,"databundleVersionId":16780923,"modelInstanceId":638158,"modelId":650143},{"sourceType":"modelInstanceVersion","sourceId":847000,"databundleVersionId":16901617,"modelInstanceId":644070,"modelId":656033},{"sourceType":"modelInstanceVersion","sourceId":847815,"databundleVersionId":16912868,"modelInstanceId":644669,"modelId":656621},{"sourceType":"modelInstanceVersion","sourceId":822944,"databundleVersionId":16521652,"modelInstanceId":625490,"modelId":637354},{"sourceType":"kernelVersion","sourceId":171884838},{"sourceType":"kernelVersion","sourceId":312874589},{"sourceType":"kernelVersion","sourceId":315000306}],"dockerImageVersionId":31328,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install transformers torch rdkit\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport torch\nimport polars as pl\nimport pandas as pd\nfrom tqdm import tqdm\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.amp import autocast\nfrom transformers import DataCollatorWithPadding\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport numpy as np\nfrom tqdm import tqdm\nfrom transformers import AutoTokenizer, AutoModel\nfrom sklearn.metrics import average_precision_score\nfrom sklearn.model_selection import train_test_split\nimport pyarrow.parquet as pq\nfrom torch.amp import GradScaler","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n# --- 1. 數據優化抽樣 (Optimized Sampling) ---\n# 參數設定\nINPUT_FILE = \"/kaggle/input/competitions/leash-BELKA/train.csv\"\nOUTPUT_FILE = \"optimized_train_data.csv\"\nCHUNK_SIZE = 3_000_000\n\ndef csv_to_parquet_optimized(input_csv, output_parquet):\n    print(\"正在將 CSV 轉換為高效能 Parquet 格式...\")\n    # 使用 Polars 讀取，並指定數據類型以節省空間\n    df = pl.read_csv(input_csv, dtypes={\n        \"molecule_smiles\": pl.Utf8,\n        \"protein_name\": pl.Categorical,\n        \"binds\": pl.Int8\n    })\n    df.write_parquet(output_parquet, compression=\"snappy\")\n    print(f\"✅ 轉換完成: {output_parquet}\")\n    del df\n    gc.collect()\n\ndef optimized_sampling_to_parquet(input_parquet, output_parquet, chunk_size=3_000_000, neg_to_pos_ratio=16):\n    print(\"Sampling...\")\n    \n    # 關鍵動作：建立 Parquet 檔案物件\n    parquet_file = pq.ParquetFile(input_parquet)\n    \n    # ==========================================\n    # 第一趟 (Pass 1)：母體普查\n    # ==========================================\n    print(\"\\n [Pass 1] 正在進行全資料庫極速普查...\")\n    total_pos_count = 0\n    total_neg_count = 0\n    \n    # 使用 iter_batches 來分批讀取，並轉成 Pandas DataFrame\n    for batch in tqdm(parquet_file.iter_batches(batch_size=chunk_size, columns=['binds']), desc=\"Counting\"):\n        chunk = batch.to_pandas()\n        total_pos_count += (chunk['binds'] == 1).sum()\n        total_neg_count += (chunk['binds'] == 0).sum()\n        \n    print(f\"母體總數 - 正樣本: {total_pos_count:,} | 負樣本: {total_neg_count:,}\")\n    \n    # 精算抽樣率\n    target_neg_count = total_pos_count * neg_to_pos_ratio\n    exact_sampling_rate = target_neg_count / total_neg_count\n    \n    print(f\" 目標負樣本數: {target_neg_count:,} (比例 15:1)\")\n    print(f\" 算出抽樣機率: {exact_sampling_rate:.8f}\")\n\n    # ==========================================\n    # 第二趟 (Pass 2)：精準資料提取\n    # ==========================================\n    print(\"\\n [Pass 2] 正在進行精準資料提取與抽樣...\")\n    use_cols = ['molecule_smiles', 'protein_name', 'binds', 'buildingblock1_smiles']\n    \n    pos_chunks = []\n    neg_chunks = []\n\n    for batch in tqdm(parquet_file.iter_batches(batch_size=chunk_size, columns=use_cols), desc=\"Extracting & Sampling\"):\n        chunk = batch.to_pandas()\n        \n        # 1. 抓出所有正樣本\n        pos_chunks.append(chunk[chunk['binds'] == 1])\n        \n        # 2. 抓出負樣本並使用機率抽樣\n        neg_subset = chunk[chunk['binds'] == 0]\n        neg_chunks.append(neg_subset.sample(frac=exact_sampling_rate, random_state=2))\n\n    print(\"\\n 讀取完畢！正在合併數據...\")\n    df_pos = pd.concat(pos_chunks)\n    df_neg = pd.concat(neg_chunks)\n    \n    # 合併並洗牌\n    df_final = pd.concat([df_pos, df_neg]).sample(frac=1, random_state=2).reset_index(drop=True)\n\n    print(f\"最終樣本數: {len(df_final):,} (正: {len(df_pos):,}, 負: {len(df_neg):,})\")\n    \n    # ==========================================\n    # 寫入與輸出\n    # ==========================================\n    print(\"正在轉換為 Polars 並寫入高效能 Parquet...\")\n    \n    pl_df = pl.from_pandas(df_final).with_columns([\n        pl.col(\"molecule_smiles\").cast(pl.Utf8),\n        pl.col(\"buildingblock1_smiles\").cast(pl.Utf8),  \n        pl.col(\"protein_name\").cast(pl.Categorical),    \n        pl.col(\"binds\").cast(pl.Int8)\n    ])\n    \n    pl_df.write_parquet(output_parquet, compression=\"snappy\")\n    \n    print(f\"完美轉換！檔案已儲存至 {output_parquet}\")\n    \n    del df_pos, df_neg, df_final, pl_df\n    gc.collect()\n\n\n\noptimized_sampling_to_parquet(\"/kaggle/input/competitions/leash-BELKA/train.parquet\", \"optimized_train_data.parquet\")\n\ngc.collect()\n\nif torch.cuda.is_available():\n    torch.cuda.empty_cache()\n'''","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nimport polars as pl\nimport gc\n\ndef merge_and_shuffle_data(main_parquet_path, external_csv_path, output_path, read_count_threshold=15):\n    print(\"[Phase 1] 啟動外部資料融合...\")\n    \n    # 1. 讀取主資料集\n    print(f\"讀取主資料集: {main_parquet_path}\")\n    df_main = pl.read_parquet(main_parquet_path)\n    \n    # 2. 讀取外部資料並進行嚴格篩選\n    print(f\"📥 讀取外部資料: {external_csv_path}\")\n    df_ext_raw = pl.read_csv(external_csv_path)\n    \n    # 過濾低頻雜訊 \n    initial_count = len(df_ext_raw)\n    df_ext_pos_filtered = df_ext_raw.filter(pl.col(\"read_count\") >= read_count_threshold)\n    print(f\"pos_filtered -> {len(df_ext_pos_filtered)} 筆)\")\n    df_ext_neg_filtered = df_ext_raw.filter(pl.col(\"read_count\") == 0)\n    print(f\"neg_filtered -> {len(df_ext_neg_filtered)} 筆)\")\n    \n    # 保留 [Dy] 並使用 iso 立體結構\n    df_positive = df_ext_pos_filtered.select([\n        pl.col(\"new_structure\").alias(\"molecule_smiles\"),         # 完美保留 [Dy]\n        pl.col(\"bb1_iso\").alias(\"buildingblock1_smiles\"),         # 完美保留立體化學特徵\n        pl.lit(\"sEH\").cast(pl.Categorical).alias(\"protein_name\"), \n        pl.lit(1).cast(pl.Int8).alias(\"binds\")                    # 因為經過高 read_count 篩選，現在可以安心給 1\n    ])\n\n    df_negative = df_ext_neg_filtered.select([\n        pl.col(\"new_structure\").alias(\"molecule_smiles\"),         # 完美保留 [Dy]\n        pl.col(\"bb1_iso\").alias(\"buildingblock1_smiles\"),         # 完美保留立體化學特徵\n        pl.lit(\"sEH\").cast(pl.Categorical).alias(\"protein_name\"), \n        pl.lit(0).cast(pl.Int8).alias(\"binds\")                    # 0\n    ])\n\n    df_ext = pl.concat([df_positive, df_negative])\n    \n    # 3. 合併與洗牌\n    print(\"正在合併與全局洗牌...\")\n    df_main_subset = df_main.select(df_ext.columns) \n    df_combined = pl.concat([df_main_subset, df_ext])\n    df_combined = df_combined.sample(fraction=1.0, shuffle=True, seed=42)\n\n    # 5. 寫出 Parquet\n    print(f\"正在寫入最終訓練檔至: {output_path}\")\n    df_combined.write_parquet(output_path, compression=\"snappy\")\n    \n    print(f\"總樣本數：{len(df_combined):,} (新增 {len(df_ext_pos_filtered):,} 筆強效正樣本)(新增 {len(df_ext_neg_filtered):,} 筆嚴格負樣本)\")\n    \n    del df_main, df_ext_raw, df_ext_pos_filtered, df_ext_neg_filtered, df_positive, df_negative, df_ext, df_combined, df_main_subset\n    gc.collect()\n\n\n\n# 執行合併 \nmerge_and_shuffle_data(\"optimized_train_data.parquet\", \"/kaggle/input/notebooks/chemdatafarmer/additional-seh-data/DNA_Labeled_Data.csv\", \"final_merged_train_data.parquet\", read_count_threshold=15)\n'''","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nfrom sklearn.model_selection import StratifiedGroupKFold\n\ndef load_and_split_data(parquet_path, n_splits=5):\n    print(\"正在透過 Polars 載入數據...\")\n    df = pl.read_parquet(parquet_path)\n    \n    print(\"啟動 StratifiedGroupKFold 嚴格切分 (BB1 隔離)...\")\n    sgkf = StratifiedGroupKFold(n_splits=n_splits, shuffle=True, random_state=42)\n    \n    y = df['binds'].to_numpy()\n    groups = df['buildingblock1_smiles'].to_numpy()\n    X = np.zeros(len(y)) \n    \n    for train_idx, val_idx in sgkf.split(X, y, groups):\n        train_df = df[train_idx].to_pandas()\n        val_df = df[val_idx].to_pandas()\n        break \n        \n    print(f\"分割完成 | 訓練集: {len(train_df)} | 驗證集: {len(val_df)}\")\n    \n    # 4. 嚴格洩漏檢查 (此時還需要用到 BB1)\n    train_groups = set(train_df['buildingblock1_smiles'].unique())\n    val_groups = set(val_df['buildingblock1_smiles'].unique())\n    leakage = train_groups.intersection(val_groups)\n    \n    if len(leakage) == 0:\n        print(\"洩漏檢查完美通過：訓練集與驗證集完全沒有重複的化學骨架 (BB1)！\")\n    else:\n        print(f\"警告：發現洩漏！重疊的 BB1 數量: {len(leakage)}\")\n    \n    # ==========================================\n    # 卸載 BB1 欄位\n    # ==========================================\n    print(\"🧹 正在清理暫存欄位，無縫對齊下游 Dataset...\")\n    train_df = train_df.drop(columns=['buildingblock1_smiles'])\n    val_df = val_df.drop(columns=['buildingblock1_smiles'])\n    \n    return train_df, val_df\n\n\nprint(\"開始執行Scaffold Split 數據載入...\")\ntrain_df, val_df = load_and_split_data(\"/kaggle/input/notebooks/t8101349/predict-new-medicines-with-belka-rusessemble-data3/final_merged_train_data.parquet\", n_splits=5) ###\n\nimport gc\ngc.collect()\n\nprint(f\"分割完成！訓練集大小: {len(train_df)}, 驗證集大小: {len(val_df)}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torchvision\nimport torch.nn as nn\n\nclass FocalLoss(nn.Module):\n    def __init__(self, alpha=0.25, gamma=3.0, label_smoothing=0.005):\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.label_smoothing = label_smoothing\n        self.bce = nn.BCEWithLogitsLoss(reduction='none')\n\n    def forward(self, inputs, targets):\n        targets = targets.view(-1, 1).float()\n        inputs = inputs.view(-1, 1).float()\n        \n        if self.label_smoothing > 0:\n            smoothed_targets = targets * (1.0 - self.label_smoothing) + 0.5 * self.label_smoothing\n        else:\n            smoothed_targets = targets\n\n        bce_loss = self.bce(inputs, smoothed_targets)\n        probas = torch.sigmoid(inputs)\n        p_t = probas * targets + (1 - probas) * (1 - targets)\n        focal_weight = (1 - p_t) ** self.gamma\n        alpha_t = self.alpha * targets + (1 - self.alpha) * (1 - targets)\n        \n        loss = alpha_t * focal_weight * bce_loss\n        return loss.mean()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nimport pandas as pd\nimport polars as pl\nimport numpy as np\nfrom tqdm import tqdm\nfrom transformers import AutoTokenizer, AutoModel\nfrom sklearn.metrics import average_precision_score\n\n# --- 1. 配置與蛋白質預快取 ---\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nPROT_MODEL_NAME = \"facebook/esm2_t12_35M_UR50D\"\n\n# 預先載入 Tokenizer\nprot_tokenizer = AutoTokenizer.from_pretrained(PROT_MODEL_NAME)\nprot_model = AutoModel.from_pretrained(PROT_MODEL_NAME).to(device).eval()\n\ndef precompute_prot_embeddings():\n    \"\"\"預先計算 BELKA 的三種蛋白質 Embedding\"\"\"\n    prot_sequences = {\n        'BRD4': \"MSAESGPGTRLRNLPVMGDGLETSQMSTTQAQAQPQPANAASTNPPPPETSNPNKPKRQTNQLQYLLRVVLKTLWKHQFAWPFQQPVDAVKLNLPDYYKIIKTPMDMGTIKKRLENNYYWNAQECIQDFNTMFTNCYIYNKPGDDIVLMAEALEKLFLQKINELPTEETEIMIVQAKGRGRGRKETGTAKPGVSTVPNTTQASTPPQTQTPQPNPPPVQATPHPFPAVTPDLIVQTPVMTVVPPQPLQTPPPVPPQPQPPPAPAPQPVQSHPPIIAATPQPVKTKKGVKRKADTTTPTTIDPIHEPPSLPPEPKTTKLGQRRESSRPVKPPKKDVPDSQQHPAPEKSSKVSEQLKCCSGILKEMFAKKHAAYAWPFYKPVDVEALGLHDYCDIIKHPMDMSTIKSKLEAREYRDAQEFGADVRLMFSNCYKYNPPDHEVVAMARKLQDVFEMRFAKMPDEPEEPVVAVSSPAVPPPTKVVAPPSSSDSSSDSSSDSDSSTDDSEEERAQRLAELQEQLKAVHEQLAALSQPQQNKPKKKKKKKKKKKK\", \n        'HSA': \"MKWVTFISLLFLFSSAYSRGVFRRDAHKSEVAHRFKDLGEENFKALVLIAFAQYLQQCPFEDHVKLVNEVTEFAKTCVADESAENCDKSLHTLFGDKLCTVATLRETYGEMADCCAKQEPERNECFLQHKDDNPNLPRLVRPEVDVMCTAFHDNEETFLKKYLYEIARRHPYFYAPELLFFAKRYKAAFTECCQAADKAACLLPKLDELRDEGKASSAKQRLKCASLQKFGERAFKAWAVARLSQRFPKAEFAEVSKLVTDLTKVHTECCHGDLLECADDRADLAKYICENQDSISSKLKECCEKPLLEKSHCIAEVENDEMPADLPSLAADFVESKDVCKNYAEAKDVFLGMFLYEYARRHPDYSVVLLLRLAKTYETTLEKCCAAADPHECYAKVFDEFKPLVEEPQNLIKQNCELFEQLGEYKFQNALLVRYTKKVPQVSTPTLVEVSRNLGKVGSKCCKHPEAKRMPCAEDYLSVVLNQLCVLHEKTPVSDRVTKCCTESLVNRRPCFSALEVDETYVPKEFNAETFTFHADICTLSEKERQIKKQTALVELVKHKPKATKEQLKAVMDDFAAFVEKCCKADDKETCFAEEGKKLVAASQAALGL\",\n        'sEH': \"MTLRAAVFDLDGVLALPAVFGVLGRTEEALALPRGLLNDAFQKGGPEGATTRLMKGEITLSQWIPLMEENCRKCSETAKVCLPKNFSIKEIFDKAISARKINRPMLQAALMLRKKGFTTAILTNTWLDDRAERDGLAQLMCELKMHFDFLIESCQVGMVKPEPQIYKFLLDTLKASPSEVVFLDDIGANLKPARDLGMVTILVQDTDTALKELEKVTGIQLLNTPAPLPTSCNPSDMSHGYVTVKPRVRLHFVELGSGPAVCLCHGFPESWYSWRYQIPALAQAGYRVLAMDMKGYGESSAPPEIEEYCMEVLCKEMVTFLDKLGLSQAVFIGHDWGGMLVWYMALFYPERVRAVASLNTPFIPANPNMSPLESIKANPVFDYQLYFQEPGVAEAELEQNLSRTFKSLFRASDESVLSMHKVCEAGGLFVNSPEEPSLSRMVTEEEIQFYVQQFKKSGFRGPLNWYRNMERNWKWACKSLGRKILIPALMVTAEKDFVLVPQMSQHMEDWIPHLKRGHIEDCGHWTQMDKPTEVNQILIKWLDSDARNPPVVSKM\"\n    }\n    prot_map = {}\n    for name, seq in prot_sequences.items():\n        inputs = prot_tokenizer(seq, return_tensors=\"pt\", truncation=True, max_length=1024).to(device)\n        with torch.no_grad():\n            outputs = prot_model(**inputs)\n            # 取 Mean Pooling\n            prot_map[name] = outputs.last_hidden_state.mean(dim=1).cpu().numpy().squeeze()\n    return prot_map\n\n# 預處理好的 map\nprot_map = precompute_prot_embeddings()\n\nclass SmilesCharTokenizer:\n    def __init__(self):\n        # BELKA 常見的 SMILES 字元庫 (包含我們保留的 Dy 標籤)\n        chars = ['<PAD>', 'C', 'N', 'O', 'F', 'S', 'c', 'n', 'o', 's', \n                 '(', ')', '[', ']', '=', '#', '@', '+', '-', '1', '2', '3', \n                 '4', '5', '6', '7', '8', '9', 'l', 'r', 'B', 'D', 'y', 'H']\n        \n        self.vocab = {c: i for i, c in enumerate(chars)}\n        self.vocab_size = len(self.vocab)\n        self.pad_token_id = 0\n\n    def __call__(self, smiles_list, max_length=128):\n        batch_ids = []\n        for smiles in smiles_list:\n            # 將字串轉換為 ID，遇到沒見過的字元統一當作 padding 或是忽略\n            token_ids = [self.vocab.get(c, 0) for c in smiles]\n            \n            # 截斷 (Truncation)\n            if len(token_ids) > max_length:\n                token_ids = token_ids[:max_length]\n            \n            batch_ids.append(token_ids)\n            \n        # 動態 Padding (Padding 到該 Batch 的最大長度)\n        max_len_in_batch = max(len(t) for t in batch_ids)\n        padded_ids = []\n        for t in batch_ids:\n            padded = t + [self.pad_token_id] * (max_len_in_batch - len(t))\n            padded_ids.append(padded)\n            \n        return {\"input_ids\": torch.tensor(padded_ids, dtype=torch.long)}\n\n# 實例化新的 Tokenizer\nmol_tokenizer = SmilesCharTokenizer()\n\n# --- 2. 數據集與模型定義 ---\n\nclass BELKADynamicDataset(Dataset):\n    def __init__(self, df, prot_map):\n        self.smiles = df['molecule_smiles'].values\n        self.prot_names = df['protein_name'].values\n        self.labels = df['binds'].values\n        self.prot_map = prot_map\n\n    def __len__(self):\n        return len(self.labels)\n\n    def __getitem__(self, idx):\n        # 1. 抓取蛋白質特徵\n        prot_vector = self.prot_map[self.prot_names[idx]]\n        if isinstance(prot_vector, torch.Tensor):\n            prot_vector = prot_vector.clone().detach().float()\n        else:\n            prot_vector = torch.tensor(prot_vector, dtype=torch.float32)\n\n        return {\n            \"smiles\": self.smiles[idx], \n            \"prot_emb\": prot_vector,\n            \"label\": torch.tensor([self.labels[idx]], dtype=torch.float32)\n        }\n\ndef create_cnn_collate_fn(tokenizer):\n    def collate_fn(batch):\n        smiles_list = [item['smiles'] for item in batch]\n        prot_embs = torch.stack([item['prot_emb'] for item in batch])\n        \n        labels = torch.stack([item['label'] for item in batch])\n        \n        tokenized = tokenizer(smiles_list, max_length=256)\n        mol_ids = tokenized[\"input_ids\"]\n        mol_mask = (mol_ids != 0).long()\n        \n        return {\n            \"mol_ids\": mol_ids,\n            \"mol_mask\": mol_mask,\n            \"prot_emb\": prot_embs,\n            \"labels\": labels  \n        }\n    return collate_fn\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BELKAHybridModel(nn.Module):\n    def __init__(self, vocab_size, embed_dim=128, cnn_dim=128, prot_dim=480, max_len=256):\n        super().__init__()\n        \n        # 1. 基礎字元 Embedding\n        # padding_idx=0 確保 <PAD> token 的初始向量為全 0\n        self.embedding = nn.Embedding(vocab_size, embed_dim, padding_idx=0)\n        \n        # 2. 局部特徵提取器：1D-CNN (Kernel=3)\n        # 模仿第五名：提取相鄰 3 個字元 (例如 'C', '=', 'O' 形成羰基) 的組合特徵\n        self.conv1d = nn.Conv1d(in_channels=embed_dim, out_channels=cnn_dim, kernel_size=3, padding=1)\n        self.cnn_activation = nn.GELU()\n        \n        # 3. 空間位置編碼 (Positional Encoding)\n        # 雖然 CNN 有局部順序，但 Transformer 看整個序列是無序的，必須加上位置編碼\n        self.pos_encoder = nn.Embedding(max_len, cnn_dim)\n        \n        # 4. 全局特徵編碼器：極度輕量的 Transformer Encoder (4 層, 4 頭)\n        # 模仿第一名：讓局部官能基互相產生 Attention，理解 2D 分子拓撲\n        encoder_layer = nn.TransformerEncoderLayer(\n            d_model=cnn_dim, \n            nhead=4, \n            dim_feedforward=cnn_dim * 4, \n            batch_first=True, \n            dropout=0.1,\n            activation='gelu'\n        )\n        self.transformer = nn.TransformerEncoder(encoder_layer, num_layers=4)\n        \n        # 5. 終極預測頭 (分類器)\n        self.head = nn.Sequential(\n            nn.Linear(cnn_dim + prot_dim, 512),\n            nn.BatchNorm1d(512),\n            nn.GELU(),\n            nn.Dropout(0.3),\n            nn.Linear(512, 128),\n            nn.BatchNorm1d(128),\n            nn.GELU(),\n            nn.Dropout(0.2),\n            nn.Linear(128, 1)\n        )\n\n    def forward(self, mol_ids, prot_emb):\n        \"\"\"\n        mol_ids: (Batch, Seq_Len) - 字元 token ID\n        prot_emb: (Batch, Prot_Dim) - 蛋白質特徵向量\n        \"\"\"\n        B, L = mol_ids.size()\n        \n        # --- Stage 1: Embedding ---\n        x = self.embedding(mol_ids)  # Shape: (B, L, Embed_Dim)\n        \n        # --- Stage 2: 1D-CNN 特徵提取 ---\n        # 輸入維度為 (Batch, Channels, Length)\n        x = x.permute(0, 2, 1)       # 轉換為 (B, Embed_Dim, L)\n        x = self.conv1d(x)           # 卷積提取 -> (B, CNN_Dim, L)\n        x = self.cnn_activation(x)\n        x = x.permute(0, 2, 1)       # 轉回 Transformer 需要的 (B, L, CNN_Dim)\n        \n        # --- Stage 3: 加入位置編碼 ---\n        positions = torch.arange(L, device=x.device).unsqueeze(0).expand(B, L)\n        x = x + self.pos_encoder(positions)\n        \n        # --- Stage 4: Transformer 處理 ---\n        # 創建 Padding Mask：mol_ids == 0 的地方為 True\n        pad_mask = (mol_ids == 0)    \n        \n        # src_key_padding_mask 負責擋掉無效的空白區域\n        x = self.transformer(x, src_key_padding_mask=pad_mask) # Shape: (B, L, CNN_Dim)\n        \n        # --- Stage 5: Masked Average Pooling ---\n        # 不能直接 .mean(dim=1) ,因為後面一大堆 pad token 是 0 ,會把平均值稀釋掉！\n        # 必須只計算「有效字元」的平均。\n        token_mask = (~pad_mask).unsqueeze(-1).float()          # 變成 1 (有效) 和 0 (無效), Shape: (B, L, 1)\n        sum_x = (x * token_mask).sum(dim=1)                     # 將有效字元的特徵相加, Shape: (B, CNN_Dim)\n        sum_mask = token_mask.sum(dim=1).clamp(min=1e-9)        # 計算有效字元的總數，clamp 防止除以 0\n        pooled_x = sum_x / sum_mask                             # 真正的有效平均特徵 Shape: (B, CNN_Dim)\n        \n        # --- Stage 6: 結合蛋白質特徵並預測 ---\n        combined = torch.cat([pooled_x, prot_emb], dim=1)       # Shape: (B, CNN_Dim + Prot_Dim)\n        logits = self.head(combined)                            # Shape: (B, 1)\n        \n        return logits","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import time\nimport torch\nfrom torch.amp import autocast, GradScaler\nfrom sklearn.metrics import average_precision_score\nfrom tqdm import tqdm\n\ndef train_and_validate_optimized(\n    model, train_loader, val_loader, optimizer, criterion, device, \n    epochs=10, time_limit_hours=7.0, \n    max_lr=5e-4,                           \n    save_name=\"best_belka_model.pth\",\n    scheduler_type='onecycle'  \n):\n    # --- 優化三：動態切換 Scheduler ---\n    if scheduler_type == 'onecycle':\n        # 有 Warmup 階段\n        scheduler = torch.optim.lr_scheduler.OneCycleLR(\n            optimizer, \n            max_lr=max_lr, \n            steps_per_epoch=len(train_loader), \n            epochs=epochs, \n            pct_start=0.3, \n            anneal_strategy='cos' \n        )\n        print(\"使用 OneCycleLR Scheduler (包含 Warmup)\")\n    elif scheduler_type == 'cosine':\n        # 無 Warmup，穩定遞減\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(\n            optimizer, \n            T_max=epochs * len(train_loader), \n            eta_min=1e-7\n        )\n        print(\"使用 CosineAnnealingLR Scheduler (適合極限微調)\")\n\n    scaler = GradScaler('cuda')\n    best_ap = 0.0\n    start_time = time.time()\n\n    for epoch in range(epochs):\n        # --- 訓練階段 ---\n        model.train()\n        train_loss = 0\n        pbar = tqdm(train_loader, desc=f\"Epoch {epoch+1}/{epochs} [Train]\")\n        \n        for batch in pbar:\n            ids = batch[\"mol_ids\"].to(device)\n            prot = batch[\"prot_emb\"].to(device)\n            labels = batch[\"labels\"].to(device) \n\n            optimizer.zero_grad()\n            \n            with torch.amp.autocast('cuda'):\n                outputs = model(mol_ids=ids, prot_emb=prot) \n                \n                labels = labels.view_as(outputs) \n                loss = criterion(outputs.float(), labels.float())\n            \n            scaler.scale(loss).backward()\n            scaler.unscale_(optimizer)\n            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n            \n            scaler.step(optimizer)\n            scaler.update()\n            scheduler.step()\n\n            train_loss += loss.item()\n            pbar.set_postfix({\"loss\": f\"{loss.item():.4f}\", \"lr\": f\"{scheduler.get_last_lr()[0]:.6e}\"})\n\n        # --- 驗證階段 ---\n        model.eval()\n        all_preds, all_labels = [], []\n        val_loss = 0\n\n        with torch.no_grad():\n            for batch in tqdm(val_loader, desc=f\"Epoch {epoch+1}/{epochs} [Val]\"):\n                ids = batch[\"mol_ids\"].to(device)\n                prot = batch[\"prot_emb\"].to(device)\n                labels = batch[\"labels\"].to(device) \n\n                with torch.amp.autocast('cuda'):\n                    outputs = model(mol_ids=ids, prot_emb=prot) \n                    labels = labels.view_as(outputs)\n                    v_loss = criterion(outputs.float(), labels.float())\n\n                val_loss += v_loss.item()\n                probs = torch.sigmoid(outputs.float()) \n                all_preds.extend(probs.cpu().numpy()) \n                all_labels.extend(labels.cpu().numpy())\n        \n        avg_train_loss = train_loss / len(train_loader)\n        avg_val_loss = val_loss / len(val_loader)\n        current_ap = average_precision_score(all_labels, all_preds)\n\n        print(f\"\\nEpoch {epoch+1} Summary:\")\n        print(f\"Train Loss: {avg_train_loss:.4f} | Val Loss: {avg_val_loss:.4f} | AP: {current_ap:.4f}\")\n\n        if current_ap > best_ap:\n            best_ap = current_ap\n            if isinstance(model, nn.DataParallel):\n                torch.save(model.module.state_dict(), save_name)\n            else:\n                torch.save(model.state_dict(), save_name)\n                \n            print(f\"新紀錄！最佳模型已儲存為: {save_name} (AP: {best_ap:.4f})\")\n        print(\"-\" * 30)\n        \n        # 時間控制\n        elapsed_hours = (time.time() - start_time) / 3600\n        if elapsed_hours > time_limit_hours:\n            print(f\"接近系統時間限制 ({time_limit_hours} 小時)，強制停止訓練以保留心血！\")\n            break\n\n    return best_ap","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport torch\nimport polars as pl\nfrom sklearn.model_selection import train_test_split # 應該從 sklearn 導入\nfrom torch.utils.data import DataLoader, Dataset      # torch 只負責 Data 加載\n\n\n# 設定設備\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")\n\nBATCH_SIZE = 3072\n\nprint(\"🛠️ Preparing DataLoaders...\")\ntrain_dataset = BELKADynamicDataset(train_df, prot_map)\nval_dataset = BELKADynamicDataset(val_df, prot_map)\n\ncustom_collate = create_cnn_collate_fn(mol_tokenizer)\n\ntrain_loader = DataLoader(\n    train_dataset, batch_size=BATCH_SIZE, shuffle=True, \n    num_workers=4, pin_memory=True, drop_last=True,\n    collate_fn=custom_collate\n)\nval_loader = DataLoader(\n    val_dataset, batch_size=BATCH_SIZE, shuffle=False, \n    num_workers=4, pin_memory=True,\n    collate_fn=custom_collate\n)\n\n\n# ==========================================\n# 🥇 第一階段：隨機初始化層的高速收斂\n# ==========================================\nprint(\"\\n\" + \"=\"*40)\nprint(\"啟動第一階段：訓練全新互動層\")\nprint(\"策略: OneCycleLR, Max LR=5e-4\")\nprint(\"=\"*40)\n\n# 1. 建立純淨模型\nprint(\"初始化基礎模型...\")\nmodel = BELKAHybridModel(\n    vocab_size=mol_tokenizer.vocab_size, \n    embed_dim=128, \n    cnn_dim=256, \n    prot_dim=480\n).to(device)\n\n\nif torch.cuda.device_count() > 1:\n    print(f\"偵測到 {torch.cuda.device_count()} 張 GPU！啟動 DataParallel 雙卡平行運算！\")\n    model = nn.DataParallel(model)\n\ncriterion_phase1 = FocalLoss(alpha=0.25, gamma=2.0, label_smoothing=0.05)\n\n# 只將 requires_grad=True 的參數放入 Optimizer \noptimizer_phase1 = torch.optim.AdamW(filter(lambda p: p.requires_grad, model.parameters()), lr=2e-4, weight_decay=1e-2)\n\nbest_ap_phase1 = train_and_validate_optimized(\n    model=model, train_loader=train_loader, val_loader=val_loader,\n    optimizer=optimizer_phase1, criterion=criterion_phase1, device=device,\n    epochs=8, \n    time_limit_hours=9.0,\n    max_lr=2e-4, \n    save_name=\"best_model_chemberta_1.pth\", \n    scheduler_type='onecycle'  \n)\n\n# ==========================================\n# 🥈 第二階段：極限微調與困難樣本挖掘\n# ==========================================\nprint(\"\\n\" + \"=\"*40)\nprint(\"啟動第二階段：極限微調與困難樣本挖掘\") \nprint(\"策略: OneCycleLR, Max LR=1e-3\")\nprint(\"=\"*40)\n\n# 1. 建立純淨模型\nprint(\"初始化基礎模型...\")\nmodel = BELKAHybridModel(\n    vocab_size=mol_tokenizer.vocab_size, \n    embed_dim=128, \n    cnn_dim=256, \n    prot_dim=480\n).to(device)\n\n# 2. 載入第一階段「熱身完畢」的最佳權重\nprint(\"載入 Phase 1 最佳權重...\")\nmodel.load_state_dict(torch.load(\"best_model_chemberta_1.pth\", map_location=device))\n\n\nif torch.cuda.device_count() > 1:\n    print(f\"偵測到 {torch.cuda.device_count()} 張 GPU！啟動 DataParallel 雙卡平行運算！\")\n    model = nn.DataParallel(model)\n    \ncriterion_phase2 = FocalLoss(alpha=0.3, gamma=3.0, label_smoothing=0.005)\n\n# 重新宣告 Optimizer\noptimizer_phase2 = torch.optim.AdamW(filter(lambda p: p.requires_grad, model.parameters()), lr=2e-5, weight_decay=1e-2)\n\nbest_ap_phase2 = train_and_validate_optimized(  \n    model=model, train_loader=train_loader, val_loader=val_loader,\n    optimizer=optimizer_phase2, criterion=criterion_phase2, device=device,\n    epochs=1, \n    time_limit_hours=4.0,\n    max_lr=2e-5,  \n    save_name=\"best_model_chemberta_2.pth\",\n    scheduler_type='onecycle' \n)\n\nprint(\"\\n🎉 三階段訓練全部完成！\")\nprint(f\"🥇 第一階段 AP: {best_ap_phase1:.4f}\")\nprint(f\"🥈 第二階段 AP: {best_ap_phase2:.4f}\")\n\nif best_ap_phase1 > best_ap_phase2:\n    best_model = \"best_model_chemberta_1.pth\"\nelse:\n    best_model = \"best_model_chemberta_2.pth\"\nprint(f\"📁 最終權重已安全儲存於: {best_model}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport torch\nimport polars as pl\nimport pandas as pd\nfrom tqdm import tqdm\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.amp import autocast\nfrom transformers import DataCollatorWithPadding\n\n\nclass BELKATestDataset(Dataset):\n    def __init__(self, df, prot_map):\n        self.ids = df['id'].to_numpy()\n        self.smiles = df['molecule_smiles'].to_numpy()\n        self.prot_names = df['protein_name'].to_numpy()\n        self.prot_map = prot_map\n\n    def __len__(self):\n        return len(self.smiles)\n\n    def __getitem__(self, idx):\n        # 1. 抓取蛋白質特徵\n        prot_vector = self.prot_map[self.prot_names[idx]]\n        if isinstance(prot_vector, torch.Tensor):\n            prot_vector = prot_vector.clone().detach().float()\n        else:\n            prot_vector = torch.tensor(prot_vector, dtype=torch.float32)\n\n        return {\n            \"id\": self.ids[idx],\n            \"smiles\": self.smiles[idx], \n            \"prot_emb\": prot_vector\n        }\n\n# ==========================================\n# 動態 Tokenize 的 Collate Function\n# ==========================================\ndef create_test_collate_fn(tokenizer):\n    def collate_fn(batch):\n        ids = [item['id'] for item in batch]\n        smiles_list = [item['smiles'] for item in batch]\n        prot_embs = torch.stack([item['prot_emb'] for item in batch])\n        \n        tokenized = tokenizer(smiles_list, max_length=256)\n        mol_ids = tokenized[\"input_ids\"]\n        \n        return {\n            \"ids\": ids, \n            \"mol_ids\": mol_ids,\n            \"prot_emb\": prot_embs\n        }\n    return collate_fn\n\n# ==========================================\n# generate_submission\n# ==========================================\ndef generate_submission_optimized(model, test_file, mol_tokenizer, prot_map, device, output_name=\"submission.csv\"):\n    model.eval()\n    \n    print(\"使用 Polars 讀取測試集中...\")\n    test_df = pl.read_parquet(test_file, columns=[\"id\", \"molecule_smiles\", \"protein_name\"])\n    \n    print(\"準備 DataLoader...\")\n    test_dataset = BELKATestDataset(test_df, prot_map)\n    test_collate = create_test_collate_fn(mol_tokenizer)\n    \n    test_loader = DataLoader(\n        test_dataset, \n        batch_size=4096, \n        shuffle=False, # 不洗牌！\n        num_workers=4, \n        pin_memory=True,\n        collate_fn=test_collate\n    )\n    \n    all_ids = []\n    all_preds = []\n\n    print(\"開始進行推論 (Inference)...\")\n    with torch.no_grad():\n        for batch in tqdm(test_loader, desc=\"Predicting\"):\n            batch_ids = batch[\"ids\"]\n            mol_ids = batch[\"mol_ids\"].to(device)\n            prot_emb = batch[\"prot_emb\"].to(device)\n            \n            # 使用 torch.amp.autocast 加速推論\n            with torch.amp.autocast('cuda'):\n                outputs = model(mol_ids=mol_ids, prot_emb=prot_emb)\n            \n            # 將 logits 轉成 0~1 的機率\n            probs = torch.sigmoid(outputs).cpu().numpy().flatten()\n            \n            all_ids.extend(batch_ids)\n            all_preds.extend(probs)\n\n            \n    print(f\"正在寫入 {output_name}...\")\n    # 使用 Pandas 快速寫出 CSV\n    sub_df = pd.DataFrame({\n        \"id\": all_ids,\n        \"binds\": all_preds\n    })\n    sub_df.to_csv(output_name, index=False)\n    print(\"✅ 提交檔案生成完畢！祝你 Leaderboard 暴衝！\")\n\n    del test_df, all_ids, all_preds, sub_df\n    gc.collect()\n\n# ==========================================\n# 執行區塊\n# ==========================================\n\nmodel = BELKAHybridModel(\n    vocab_size=mol_tokenizer.vocab_size, \n    embed_dim=128, \n    cnn_dim=256, \n    prot_dim=480\n).to(device)\n\nmodel.load_state_dict(torch.load(best_model, map_location=device))\n\nTEST_FILE = \"/kaggle/input/competitions/leash-BELKA/test.parquet\" \nSAMPLE_SUB_FILE = \"/kaggle/input/competitions/leash-BELKA/sample_submission.csv\"\n\nif os.path.exists(TEST_FILE):\n    generate_submission_optimized(model, TEST_FILE, mol_tokenizer, prot_map, device)\nelse:\n    print(f\"❌ 找不到測試檔: {TEST_FILE}，請檢查路徑。\")\n\n\nif os.path.exists(\"submission.csv\") and os.path.exists(SAMPLE_SUB_FILE):\n    # 用 Polars 掃描行數最快，不吃記憶體\n    sub_count = pl.scan_csv(\"submission.csv\").select(pl.len()).collect().item()\n    sample_count = pl.scan_csv(SAMPLE_SUB_FILE).select(pl.len()).collect().item()\n    \n    if sub_count == sample_count:\n        print(f\"✅ 行數檢查完美通過: {sub_count} 筆數據\")\n    else:\n        print(f\"❌ 警告：行數不符！提交檔: {sub_count}, 範本: {sample_count}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nsub = pl.read_csv(\"submission.csv\")\nprint(\"ID 是否嚴格遞增且無遺漏：\", sub['id'].is_sorted())\n\n# 把 submission 的結果跟 test.parquet 裡的 protein_name 對接起來看平均機率\ntest_df = pl.read_parquet(\"/kaggle/input/competitions/leash-BELKA/test.parquet\", columns=[\"id\", \"protein_name\"])\nsub_merged = sub.join(test_df, on=\"id\")\nprint(sub_merged.group_by(\"protein_name\").agg(pl.col(\"binds\").mean()))","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}