{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":67356,"databundleVersionId":8006601},{"sourceType":"datasetVersion","sourceId":15819856,"datasetId":10140715,"databundleVersionId":16768550},{"sourceType":"modelInstanceVersion","sourceId":822944,"databundleVersionId":16521652,"modelInstanceId":625490,"modelId":637354},{"sourceType":"kernelVersion","sourceId":171884838}],"dockerImageVersionId":31328,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install transformers torch rdkit","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport torch\nimport polars as pl\nimport pandas as pd\nfrom tqdm import tqdm\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.amp import autocast\nfrom transformers import DataCollatorWithPadding\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport numpy as np\nfrom tqdm import tqdm\nfrom transformers import AutoTokenizer, AutoModel\nfrom sklearn.metrics import average_precision_score\nfrom sklearn.model_selection import train_test_split\nimport pyarrow.parquet as pq","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 1. 數據優化抽樣 (Optimized Sampling) ---\n# 參數設定\nINPUT_FILE = \"/kaggle/input/competitions/leash-BELKA/train.csv\"\nOUTPUT_FILE = \"optimized_train_data.csv\"\nCHUNK_SIZE = 3_000_000\n\ndef csv_to_parquet_optimized(input_csv, output_parquet):\n    print(\"正在將 CSV 轉換為高效能 Parquet 格式...\")\n    # 使用 Polars 讀取，並指定數據類型以節省空間\n    df = pl.read_csv(input_csv, dtypes={\n        \"molecule_smiles\": pl.Utf8,\n        \"protein_name\": pl.Categorical,\n        \"binds\": pl.Int8\n    })\n    df.write_parquet(output_parquet, compression=\"snappy\")\n    print(f\"✅ 轉換完成: {output_parquet}\")\n    del df\n    gc.collect()\n\ndef optimized_sampling_to_parquet(input_parquet, output_parquet, chunk_size=3_000_000, neg_to_pos_ratio=15):\n    print(\"Sampling...\")\n    \n    # 關鍵動作：建立 Parquet 檔案物件\n    parquet_file = pq.ParquetFile(input_parquet)\n    \n    # ==========================================\n    # 第一趟 (Pass 1)：母體普查\n    # ==========================================\n    print(\"\\n [Pass 1] 正在進行全資料庫極速普查...\")\n    total_pos_count = 0\n    total_neg_count = 0\n    \n    # 使用 iter_batches 來分批讀取，並轉成 Pandas DataFrame\n    for batch in tqdm(parquet_file.iter_batches(batch_size=chunk_size, columns=['binds']), desc=\"Counting\"):\n        chunk = batch.to_pandas()\n        total_pos_count += (chunk['binds'] == 1).sum()\n        total_neg_count += (chunk['binds'] == 0).sum()\n        \n    print(f\"母體總數 - 正樣本: {total_pos_count:,} | 負樣本: {total_neg_count:,}\")\n    \n    # 精算抽樣率\n    target_neg_count = total_pos_count * neg_to_pos_ratio\n    exact_sampling_rate = target_neg_count / total_neg_count\n    \n    print(f\" 目標負樣本數: {target_neg_count:,} (比例 15:1)\")\n    print(f\" 算出抽樣機率: {exact_sampling_rate:.8f}\")\n\n    # ==========================================\n    # 第二趟 (Pass 2)：精準資料提取\n    # ==========================================\n    print(\"\\n [Pass 2] 正在進行精準資料提取與抽樣...\")\n    use_cols = ['molecule_smiles', 'protein_name', 'binds', 'buildingblock1_smiles']\n    \n    pos_chunks = []\n    neg_chunks = []\n\n    for batch in tqdm(parquet_file.iter_batches(batch_size=chunk_size, columns=use_cols), desc=\"Extracting & Sampling\"):\n        chunk = batch.to_pandas()\n        \n        # 1. 抓出所有正樣本\n        pos_chunks.append(chunk[chunk['binds'] == 1])\n        \n        # 2. 抓出負樣本並使用機率抽樣\n        neg_subset = chunk[chunk['binds'] == 0]\n        neg_chunks.append(neg_subset.sample(frac=exact_sampling_rate, random_state=111))\n\n    print(\"\\n 讀取完畢！正在合併數據...\")\n    df_pos = pd.concat(pos_chunks)\n    df_neg = pd.concat(neg_chunks)\n    \n    # 合併並洗牌\n    df_final = pd.concat([df_pos, df_neg]).sample(frac=1, random_state=111).reset_index(drop=True)\n\n    print(f\"最終樣本數: {len(df_final):,} (正: {len(df_pos):,}, 負: {len(df_neg):,})\")\n    \n    # ==========================================\n    # 寫入與輸出\n    # ==========================================\n    print(\"正在轉換為 Polars 並寫入高效能 Parquet...\")\n    \n    pl_df = pl.from_pandas(df_final).with_columns([\n        pl.col(\"molecule_smiles\").cast(pl.Utf8),\n        pl.col(\"buildingblock1_smiles\").cast(pl.Utf8),  \n        pl.col(\"protein_name\").cast(pl.Categorical),    \n        pl.col(\"binds\").cast(pl.Int8)\n    ])\n    \n    pl_df.write_parquet(output_parquet, compression=\"snappy\")\n    \n    print(f\"完美轉換！檔案已儲存至 {output_parquet}\")\n    \n    del df_pos, df_neg, df_final, pl_df\n    gc.collect()\n\n\n\noptimized_sampling_to_parquet(\"/kaggle/input/competitions/leash-BELKA/train.parquet\", \"optimized_train_data.parquet\")\n\ngc.collect()\n\nif torch.cuda.is_available():\n    torch.cuda.empty_cache()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nimport gc\n\ndef merge_and_shuffle_data(main_parquet_path, external_csv_path, output_path, read_count_threshold=15):\n    print(\"[Phase 1] 啟動外部資料融合...\")\n    \n    # 1. 讀取主資料集\n    print(f\"讀取主資料集: {main_parquet_path}\")\n    df_main = pl.read_parquet(main_parquet_path)\n    \n    # 2. 讀取外部資料並進行嚴格篩選\n    print(f\"📥 讀取外部資料: {external_csv_path}\")\n    df_ext_raw = pl.read_csv(external_csv_path)\n    \n    # 過濾低頻雜訊 \n    initial_count = len(df_ext_raw)\n    df_ext_pos_filtered = df_ext_raw.filter(pl.col(\"read_count\") >= read_count_threshold)\n    print(f\"pos_filtered -> {len(df_ext_pos_filtered)} 筆)\")\n\n    df_ext_neg_all = df_ext_raw.filter(pl.col(\"read_count\") == 0)\n    actual_neg_samples = min(len(df_ext_neg_all), 2100000)\n    df_ext_neg_filtered = df_ext_neg_all.sample(n=actual_neg_samples, seed=42)\n    print(f\"neg_filtered -> {len(df_ext_neg_filtered)} 筆)\")\n    \n    # 保留 [Dy] 並使用 iso 立體結構\n    df_positive = df_ext_pos_filtered.select([\n        pl.col(\"new_structure\").alias(\"molecule_smiles\"),         # 完美保留 [Dy]\n        pl.col(\"bb1_iso\").alias(\"buildingblock1_smiles\"),         # 完美保留立體化學特徵\n        pl.lit(\"sEH\").cast(pl.Categorical).alias(\"protein_name\"), \n        pl.lit(1).cast(pl.Int8).alias(\"binds\")                    # 因為經過高 read_count 篩選，現在可以安心給 1\n    ])\n\n    df_negative = df_ext_neg_filtered.select([\n        pl.col(\"new_structure\").alias(\"molecule_smiles\"),         # 完美保留 [Dy]\n        pl.col(\"bb1_iso\").alias(\"buildingblock1_smiles\"),         # 完美保留立體化學特徵\n        pl.lit(\"sEH\").cast(pl.Categorical).alias(\"protein_name\"), \n        pl.lit(0).cast(pl.Int8).alias(\"binds\")                    # 給 0\n    ])\n\n    df_ext = pl.concat([df_positive, df_negative])\n    \n    # 3. 合併與洗牌\n    print(\"正在合併與全局洗牌...\")\n    df_main_subset = df_main.select(df_ext.columns) \n    df_combined = pl.concat([df_main_subset, df_ext])\n    df_combined = df_combined.sample(fraction=1.0, shuffle=True, seed=42)\n\n    # 5. 寫出 Parquet\n    print(f\"正在寫入最終訓練檔至: {output_path}\")\n    df_combined.write_parquet(output_path, compression=\"snappy\")\n    \n    print(f\"總樣本數：{len(df_combined):,} (新增 {len(df_ext_pos_filtered):,} 筆強效正樣本)(新增 {len(df_ext_neg_filtered):,} 筆嚴格負樣本)\")\n    \n    del df_main, df_ext_raw, df_ext_pos_filtered, df_ext_neg_filtered, df_positive, df_negative, df_ext, df_combined, df_main_subset\n    gc.collect()\n\n\n\n# 執行合併 \nmerge_and_shuffle_data(\"optimized_train_data.parquet\", \"/kaggle/input/notebooks/chemdatafarmer/additional-seh-data/DNA_Labeled_Data.csv\", \"final_merged_train_data.parquet\", read_count_threshold=15)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nimport matplotlib.pyplot as plt\n\ndef data_pollution_check( test_parquet_path, external_csv_path):\n    print(\"🔍 啟動外部資料污染深度掃描...\\n\")\n    \n    # 讀取資料 (Lazy / 快速掃描)\n    print(\"📥 正在載入資料集...\")\n    df_test = pl.read_parquet(test_parquet_path, columns=[\"molecule_smiles\"])\n    df_ext = pl.read_csv(external_csv_path)\n    \n\n    # ==========================================\n    # 標籤雜訊分佈 (Label Noise)\n    # ==========================================\n    print(\"-\" * 40)\n    print(\"🚨 [檢查 3] 測序讀取次數 (Read Count) 分佈檢查\")\n    \n    # 取得 read_count 統計數據\n    rc_stats = df_ext[\"read_count\"].describe()\n    print(rc_stats)\n    \n    # 畫出分佈直方圖 (幫助你決定 Threshold)\n    # 為了避免極端值拉壞圖表，我們只看前 95% 的數據\n    p95 = df_ext[\"read_count\"].quantile(0.95)\n    filtered_rc = df_ext.filter(pl.col(\"read_count\") <= p95)[\"read_count\"].to_numpy()\n    \n    plt.figure(figsize=(10, 5))\n    plt.hist(filtered_rc, bins=50, color='skyblue', edgecolor='black')\n    plt.axvline(x=15, color='red', linestyle='--', label='Suggested Threshold (e.g., 15)')\n    plt.title(\"Read Count Distribution (Bottom 95%)\")\n    plt.xlabel(\"Read Count\")\n    plt.ylabel(\"Number of Molecules\")\n    plt.legend()\n    plt.show()\n    \n    print(\"💡 解決方案：請觀察上方圖表。通常左側會有一根極高的柱子（雜訊海），請把 threshold 畫在雜訊海右側開始平緩的地方。\")\n\n# 執行驗屍腳本 \ndata_pollution_check(\"/kaggle/input/competitions/leash-BELKA/test.parquet\", \"/kaggle/input/notebooks/chemdatafarmer/additional-seh-data/DNA_Labeled_Data.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-19T11:59:57.598628Z","iopub.execute_input":"2026-04-19T11:59:57.599726Z","iopub.status.idle":"2026-04-19T12:00:09.232463Z","shell.execute_reply.started":"2026-04-19T11:59:57.599633Z","shell.execute_reply":"2026-04-19T12:00:09.231513Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\n\nprint(\"🩺 啟動最終訓練集體檢...\")\ndf = pl.read_parquet(\"final_merged_train_data.parquet\")\n\nprint(\"\\n[檢查 1] 資料總量與空值 (Null) 檢查\")\nprint(f\"總筆數: {len(df):,}\")\n# 確保沒有因為型態轉換失敗而產生隱藏的 Null\nprint(df.null_count())\n\nprint(\"\\n[檢查 2] 蛋白質與正負樣本分佈 (Class Balance)\")\n# 看看加入 200 萬筆負樣本和 sEH 正樣本後，整體的分佈長怎樣\ndist = df.group_by([\"protein_name\", \"binds\"]).len().sort([\"protein_name\", \"binds\"])\nprint(dist)\n\nprint(\"\\n[檢查 3] [Dy] 標記存活確認\")\n# 確保我們有成功保留 [Dy] 標記\ndy_count = df.filter(pl.col(\"molecule_smiles\").str.contains(r\"\\[Dy\\]\")).height\nprint(f\"包含 [Dy] 的分子數量: {dy_count:,} 筆\")\n\nprint(\"\\n[檢查 4] 骨架多樣性 (Building Block 1)\")\n# 看看我們總共有幾種不同的 BB1，這攸關到模型泛化能力\nbb1_count = df.select(\"buildingblock1_smiles\").n_unique()\nprint(f\"獨立 BB1 骨架數量: {bb1_count:,} 種\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}