{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":67356,"databundleVersionId":8006601,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":10941293,"sourceType":"datasetVersion","datasetId":6804472},{"sourceId":10960150,"sourceType":"datasetVersion","datasetId":6818611},{"sourceId":10962304,"sourceType":"datasetVersion","datasetId":6820180},{"sourceId":10962319,"sourceType":"datasetVersion","datasetId":6820193},{"sourceId":10966483,"sourceType":"datasetVersion","datasetId":6823061},{"sourceId":10967151,"sourceType":"datasetVersion","datasetId":6823555},{"sourceId":10969209,"sourceType":"datasetVersion","datasetId":6825106},{"sourceId":10975496,"sourceType":"datasetVersion","datasetId":6829577},{"sourceId":11004654,"sourceType":"datasetVersion","datasetId":6850701},{"sourceId":11004688,"sourceType":"datasetVersion","datasetId":6850728},{"sourceId":11068904,"sourceType":"datasetVersion","datasetId":6897566},{"sourceId":277131,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":237341,"modelId":259025},{"sourceId":279238,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":239200,"modelId":260859},{"sourceId":279692,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":239594,"modelId":261245},{"sourceId":279993,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":239855,"modelId":261511},{"sourceId":280654,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":240459,"modelId":262105},{"sourceId":288962,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":247583,"modelId":269117},{"sourceId":289077,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":247679,"modelId":269210},{"sourceId":289156,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":247747,"modelId":269276},{"sourceId":289393,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":247944,"modelId":269457},{"sourceId":290273,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":248665,"modelId":270191},{"sourceId":290406,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":248785,"modelId":270315},{"sourceId":290593,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":248964,"modelId":270487},{"sourceId":290946,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":249288,"modelId":270803}],"dockerImageVersionId":30919,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:32:33.370051Z","iopub.execute_input":"2025-03-26T08:32:33.370341Z","iopub.status.idle":"2025-03-26T08:32:33.774187Z","shell.execute_reply.started":"2025-03-26T08:32:33.370321Z","shell.execute_reply":"2025-03-26T08:32:33.773494Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 1. 安裝 PyTorch\n!pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118\n\n\n# 2. 安裝 RDKit（用於處理分子）\n!pip install rdkit-pypi\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:32:33.775327Z","iopub.execute_input":"2025-03-26T08:32:33.775833Z","iopub.status.idle":"2025-03-26T08:32:44.367608Z","shell.execute_reply.started":"2025-03-26T08:32:33.775804Z","shell.execute_reply":"2025-03-26T08:32:44.366812Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile requirements.txt\neasydict\nfuture\nmatplotlib\nnumpy\nopencv-python\nscikit-image\nscipy\nclick\nrequests\ntqdm\npyspng\nninja\nimageio-ffmpeg==0.4.3\ntimm\npsutil\nscikit-learn","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:32:44.369128Z","iopub.execute_input":"2025-03-26T08:32:44.369373Z","iopub.status.idle":"2025-03-26T08:32:44.375048Z","shell.execute_reply.started":"2025-03-26T08:32:44.369351Z","shell.execute_reply":"2025-03-26T08:32:44.374403Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install -r requirements.txt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:32:44.376175Z","iopub.execute_input":"2025-03-26T08:32:44.376396Z","iopub.status.idle":"2025-03-26T08:32:54.414937Z","shell.execute_reply.started":"2025-03-26T08:32:44.376378Z","shell.execute_reply":"2025-03-26T08:32:54.414151Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 1. 升級 PyTorch 和 torchvision\n#!pip install torch==2.1.0 torchvision==0.15.2 torchaudio==2.0.2 torchdata==0.6.1\n!pip install torch torchvision torchaudio torchdata --index-url https://download.pytorch.org/whl/cu118\n!pip install pytorch-lightning\n\n\n# 3. 檢查安裝是否成功\nimport torch\nimport torchdata\n\n\nprint(\"Torch version:\", torch.__version__)\nprint(\"Torchdata version:\", torchdata.__version__)\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:32:54.415958Z","iopub.execute_input":"2025-03-26T08:32:54.416275Z","iopub.status.idle":"2025-03-26T08:33:06.109362Z","shell.execute_reply.started":"2025-03-26T08:32:54.416245Z","shell.execute_reply":"2025-03-26T08:33:06.108567Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!python --version\n!nvcc --version  # 如果已經安裝了CUDA\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:33:06.110115Z","iopub.execute_input":"2025-03-26T08:33:06.110432Z","iopub.status.idle":"2025-03-26T08:33:06.386919Z","shell.execute_reply.started":"2025-03-26T08:33:06.110412Z","shell.execute_reply":"2025-03-26T08:33:06.386123Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install duckdb","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:33:06.388108Z","iopub.execute_input":"2025-03-26T08:33:06.388438Z","iopub.status.idle":"2025-03-26T08:33:09.569985Z","shell.execute_reply.started":"2025-03-26T08:33:06.388402Z","shell.execute_reply":"2025-03-26T08:33:09.569112Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install pyarrow  # 或者你也可以選擇 fastparquet\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:33:09.573146Z","iopub.execute_input":"2025-03-26T08:33:09.573391Z","iopub.status.idle":"2025-03-26T08:33:12.805816Z","shell.execute_reply.started":"2025-03-26T08:33:09.573369Z","shell.execute_reply":"2025-03-26T08:33:12.804827Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nimport pyarrow.parquet as pq\nimport pandas as pd\nimport gc  # 引入垃圾回收模組\n\nfilename = \"/kaggle/input/leash-BELKA/train.parquet\"\ncolumns_to_read = [\"molecule_smiles\", \"protein_name\", \"binds\"]\n\nbatch_size = 60000    # 每次讀取 60,000 筆\ntarget_rows = 1200000  # 每輪儲存 1,200,000 筆\ntotal_batches = 15    # 總共執行 15 次\ntotal_rows = 0        # 記錄當前累積筆數\n\nparquet_file = pq.ParquetFile(filename)\n\n# 獲取總行組數量\nnum_row_groups = parquet_file.num_row_groups\nprint(f\"Total row groups in file: {num_row_groups}\")\n\n# 計算每次讀取多少行組\nrow_groups_per_batch = target_rows // batch_size\n\n\n# 開始進行批次處理\nfor i in range(total_batches):\n    chunks = []\n    current_rows = 0  # 每輪的計數器\n    batch_start_row_group = i * row_groups_per_batch  # 直接依序取\n    batch_end_row_group = min(batch_start_row_group + row_groups_per_batch, num_row_groups)\n\n    if batch_end_row_group >= num_row_groups:\n        batch_end_row_group = num_row_groups  # 避免超出行組範圍\n\n    print(f\"✅ 第 {i+1} 次處理：從行組 {batch_start_row_group} 到行組 {batch_end_row_group}\")\n\n    # 使用 pyarrow 的 ParquetFile 直接讀取指定範圍的行組\n    for row_group_idx in range(batch_start_row_group, batch_end_row_group):\n        try:\n            batch = parquet_file.read_row_groups([row_group_idx], columns=columns_to_read)\n            chunk = batch.to_pandas()\n\n            # 檢查是否有資料\n            if not chunk.empty:\n                chunks.append(chunk)\n                current_rows += len(chunk)\n                total_rows += len(chunk)\n\n            # 如果讀取到指定範圍的資料，就停止\n            if total_rows >= target_rows * (i + 1):\n                break  # 如果已經讀到該批次的結尾就停止\n\n        except Exception as e:\n            print(f\"⚠️ 讀取行組 {row_group_idx} 時發生錯誤: {e}\")\n\n    if chunks:  # 確保有資料才進行合併\n        # 合併 DataFrame\n        batch_df = pd.concat(chunks, ignore_index=True)\n\n        # **將每 3 列合併成 1 列**\n        batch_df[\"row_idx\"] = batch_df.index // 3  # 每 3 列分組\n        batch_pivot = batch_df.pivot(index=\"row_idx\", columns=\"protein_name\", values=\"binds\").reset_index()\n\n        # **合併 molecule_smiles**\n        smiles_df = batch_df.groupby(\"row_idx\")[\"molecule_smiles\"].first().reset_index()\n        final_df = smiles_df.merge(batch_pivot, on=\"row_idx\").drop(columns=[\"row_idx\"])\n\n        # 存成 parquet，每次都存不同的檔案\n        output_filename = f\"/kaggle/working/train_part{i+1}.parquet\"\n        final_df.to_parquet(output_filename, index=False)\n\n        print(f\"✅ 第 {i+1} 次存檔：{len(final_df)} 筆，已累積 {total_rows} 筆\")\n\n        # 清理無用的變數，釋放記憶體\n        del batch_df, batch_pivot, smiles_df, final_df\n        gc.collect()  # 執行垃圾回收\n    else:\n        print(f\"⚠️ 第 {i+1} 次處理未讀取到任何資料，跳過該批次。\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:33:12.807859Z","iopub.execute_input":"2025-03-26T08:33:12.808146Z","iopub.status.idle":"2025-03-26T08:33:12.815712Z","shell.execute_reply.started":"2025-03-26T08:33:12.808123Z","shell.execute_reply":"2025-03-26T08:33:12.814917Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 15 個 Parquet 檔案\nparquet_files = [f\"/kaggle/working/train_part{i+1}.parquet\" for i in range(0, 15)]\n\n# 初始化 DuckDB 連線\ncon = duckdb.connect()\n\n# 存放所有批次的 DataFrame\nall_samples = []\n\n# 逐個處理 15 個檔案\nfor i, file in enumerate(parquet_files):\n    print(f\"📂 正在處理檔案: {file}\")\n\n    df = con.query(f\"\"\"(SELECT * FROM parquet_scan('{file}')\n                            WHERE BRD4 = 0 and HSA = 0 and sEH = 0\n                            ORDER BY random()\n                            LIMIT 3000)\n                            UNION ALL\n                            (SELECT * FROM parquet_scan('{file}')\n                            WHERE BRD4 = 1 or HSA = 1 or sEH = 1\n                            ORDER BY random()\n                            LIMIT 13000)\"\"\").df()\n\n# 儲存該批次結果\n    output_filename = f\"/kaggle/working/sampled_test_part{i+1}.parquet\"\n    df.to_parquet(output_filename, index=False)\n    print(f\"✅ 已儲存抽樣結果: {output_filename}（共 {len(df)} 筆）\")\n\n    all_samples.append(df)\n\n# 合併所有結果\nfinal_test_df = pd.concat(all_samples, ignore_index=True)\n\n# 儲存總合併的 Parquet\nfinal_test_output = \"/kaggle/working/sampled_test_all.parquet\"\nfinal_test_df.to_parquet(final_test_output, index=False)\nprint(f\"🎯 全部 15 個檔案已處理完畢，最終合併檔案: {final_test_output}（共 {len(final_test_df)} 筆）\")\n\n# 關閉 DuckDB\ncon.close()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:33:12.816468Z","iopub.execute_input":"2025-03-26T08:33:12.816698Z","iopub.status.idle":"2025-03-26T08:33:12.835419Z","shell.execute_reply.started":"2025-03-26T08:33:12.816680Z","shell.execute_reply":"2025-03-26T08:33:12.834664Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n# 15 個 Parquet 檔案\nparquet_files = [f\"/kaggle/working/train_part{i+1}.parquet\" for i in range(0, 15)]\n\n# 初始化 DuckDB 連線\ncon = duckdb.connect()\n\n# 存放所有批次的 DataFrame\nall_samples = []\n\n# 逐個處理 15 個檔案\nfor i, file in enumerate(parquet_files):\n    print(f\"📂 正在處理檔案: {file}\")\n\n    df = con.query(f\"\"\"(SELECT * FROM parquet_scan('{file}')\n                            ORDER BY random()\n                            LIMIT 15000)\n                            UNION ALL\n                            (SELECT * FROM parquet_scan('{file}')\n                            WHERE BRD4 = 1 or HSA = 1 or sEH = 1\n                            ORDER BY random()\n                            LIMIT 5000)\"\"\").df()\n\n# 儲存該批次結果\n    output_filename = f\"/kaggle/working/sampled_test_part{i+1}.parquet\"\n    df.to_parquet(output_filename, index=False)\n    print(f\"✅ 已儲存抽樣結果: {output_filename}（共 {len(df)} 筆）\")\n\n    all_samples.append(df)\n\n# 合併所有結果\nfinal_test_df = pd.concat(all_samples, ignore_index=True)\n\n# 儲存總合併的 Parquet\nfinal_test_output = \"/kaggle/working/sampled_test_all.parquet\"\nfinal_test_df.to_parquet(final_test_output, index=False)\nprint(f\"🎯 全部 15 個檔案已處理完畢，最終合併檔案: {final_test_output}（共 {len(final_test_df)} 筆）\")\n\n# 關閉 DuckDB\ncon.close()\n'''\n#y = df[[\"BRD4\", \"HSA\", \"sEH\"]].values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:33:12.836307Z","iopub.execute_input":"2025-03-26T08:33:12.836605Z","iopub.status.idle":"2025-03-26T08:33:12.848380Z","shell.execute_reply.started":"2025-03-26T08:33:12.836584Z","shell.execute_reply":"2025-03-26T08:33:12.847783Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:33:12.849140Z","iopub.execute_input":"2025-03-26T08:33:12.849363Z","iopub.status.idle":"2025-03-26T08:33:12.863062Z","shell.execute_reply.started":"2025-03-26T08:33:12.849346Z","shell.execute_reply":"2025-03-26T08:33:12.862262Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\noutput_filename = f\"/kaggle/working/training_df.csv\"\ndf.to_csv(output_filename, index=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:33:12.864072Z","iopub.execute_input":"2025-03-26T08:33:12.864380Z","iopub.status.idle":"2025-03-26T08:33:12.876183Z","shell.execute_reply.started":"2025-03-26T08:33:12.864351Z","shell.execute_reply":"2025-03-26T08:33:12.875372Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install torch-sparse torch-cluster torch-spline-conv -f https://data.pyg.org/whl/torch-2.5.1+cu118.html\n\n!pip install torch-geometric\n!pip install torch-scatter -f https://data.pyg.org/whl/torch-2.5.0+${CUDA}.html","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:33:12.876943Z","iopub.execute_input":"2025-03-26T08:33:12.877143Z","iopub.status.idle":"2025-03-26T08:46:35.418134Z","shell.execute_reply.started":"2025-03-26T08:33:12.877125Z","shell.execute_reply":"2025-03-26T08:46:35.417233Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nprint(torch.cuda.is_available())  # True\nprint(torch.version.cuda)  # 應該顯示 11.8\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:46:35.419230Z","iopub.execute_input":"2025-03-26T08:46:35.419458Z","iopub.status.idle":"2025-03-26T08:46:35.442175Z","shell.execute_reply.started":"2025-03-26T08:46:35.419437Z","shell.execute_reply":"2025-03-26T08:46:35.441340Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch_scatter\nprint(\"torch-scatter 安裝成功！\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:46:35.442986Z","iopub.execute_input":"2025-03-26T08:46:35.443175Z","iopub.status.idle":"2025-03-26T08:46:35.469171Z","shell.execute_reply.started":"2025-03-26T08:46:35.443157Z","shell.execute_reply":"2025-03-26T08:46:35.468569Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\n\nimport rdkit\nfrom rdkit import Chem\n\nimport torch\nfrom torch_geometric.data import Data\nfrom torch_geometric.loader import DataLoader\n\nfrom torch_geometric.nn import MessagePassing, global_mean_pool\nfrom torch_scatter import scatter\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nprint('import ok!')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:46:35.469987Z","iopub.execute_input":"2025-03-26T08:46:35.470284Z","iopub.status.idle":"2025-03-26T08:46:39.519000Z","shell.execute_reply.started":"2025-03-26T08:46:35.470254Z","shell.execute_reply":"2025-03-26T08:46:39.518122Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# helper\n# torch version of np unpackbits\n#https://gist.github.com/vadimkantorov/30ea6d278bc492abf6ad328c6965613a\n\ndef tensor_dim_slice(tensor, dim, dim_slice):\n\treturn tensor[(dim if dim >= 0 else dim + tensor.dim()) * (slice(None),) + (dim_slice,)]\n\n# @torch.jit.script\ndef packshape(shape, dim: int = -1, mask: int = 0b00000001, dtype=torch.uint8, pack=True):\n\tdim = dim if dim >= 0 else dim + len(shape)\n\tbits, nibble = (\n\t\t8 if dtype is torch.uint8 else 16 if dtype is torch.int16 else 32 if dtype is torch.int32 else 64 if dtype is torch.int64 else 0), (\n\t\t1 if mask == 0b00000001 else 2 if mask == 0b00000011 else 4 if mask == 0b00001111 else 8 if mask == 0b11111111 else 0)\n\t# bits = torch.iinfo(dtype).bits # does not JIT compile\n\tassert nibble <= bits and bits % nibble == 0\n\tnibbles = bits // nibble\n\tshape = (shape[:dim] + (int(math.ceil(shape[dim] / nibbles)),) + shape[1 + dim:]) if pack else (\n\t\t\t\tshape[:dim] + (shape[dim] * nibbles,) + shape[1 + dim:])\n\treturn shape, nibbles, nibble\n\n# @torch.jit.script\ndef F_unpackbits(tensor, dim: int = -1, mask: int = 0b00000001, shape=None, out=None, dtype=torch.uint8):\n\tdim = dim if dim >= 0 else dim + tensor.dim()\n\tshape_, nibbles, nibble = packshape(tensor.shape, dim=dim, mask=mask, dtype=tensor.dtype, pack=False)\n\tshape = shape if shape is not None else shape_\n\tout = out if out is not None else torch.empty(shape, device=tensor.device, dtype=dtype)\n\tassert out.shape == shape\n\n\tif shape[dim] % nibbles == 0:\n\t\tshift = torch.arange((nibbles - 1) * nibble, -1, -nibble, dtype=torch.uint8, device=tensor.device)\n\t\tshift = shift.view(nibbles, *((1,) * (tensor.dim() - dim - 1)))\n\t\treturn torch.bitwise_and((tensor.unsqueeze(1 + dim) >> shift).view_as(out), mask, out=out)\n\n\telse:\n\t\tfor i in range(nibbles):\n\t\t\tshift = nibble * i\n\t\t\tsliced_output = tensor_dim_slice(out, dim, slice(i, None, nibbles))\n\t\t\tsliced_input = tensor.narrow(dim, 0, sliced_output.shape[dim])\n\t\t\ttorch.bitwise_and(sliced_input >> shift, mask, out=sliced_output)\n\treturn out\n\nclass dotdict(dict):\n\t__setattr__ = dict.__setitem__\n\t__delattr__ = dict.__delitem__\n\t\n\tdef __getattr__(self, name):\n\t\ttry:\n\t\t\treturn self[name]\n\t\texcept KeyError:\n\t\t\traise AttributeError(name)\n\n            \nprint('helper ok!')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:46:39.519912Z","iopub.execute_input":"2025-03-26T08:46:39.520362Z","iopub.status.idle":"2025-03-26T08:46:39.531707Z","shell.execute_reply.started":"2025-03-26T08:46:39.520338Z","shell.execute_reply":"2025-03-26T08:46:39.530864Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# mol to graph adopted from\n# from https://github.com/LiZhang30/GPCNDTA/blob/main/utils/DrugGraph.py\n\nPACK_NODE_DIM=9\nPACK_EDGE_DIM=1\nNODE_DIM=PACK_NODE_DIM*8\nEDGE_DIM=PACK_EDGE_DIM*8\n\ndef one_of_k_encoding(x, allowable_set, allow_unk=False):\n\tif x not in allowable_set:\n\t\tif allow_unk:\n\t\t\tx = allowable_set[-1]\n\t\telse:\n\t\t\traise Exception(f'input {x} not in allowable set{allowable_set}!!!')\n\treturn list(map(lambda s: x == s, allowable_set))\n\n\n#Get features of an atom (one-hot encoding:)\n'''\n\t1.atom element: 44+1 dimensions    \n\t2.the atom's hybridization: 5 dimensions\n\t3.degree of atom: 6 dimensions                        \n\t4.total number of H bound to atom: 6 dimensions\n\t5.number of implicit H bound to atom: 6 dimensions    \n\t6.whether the atom is on ring: 1 dimension\n\t7.whether the atom is aromatic: 1 dimension           \n\tTotal: 70 dimensions\n'''\n\nATOM_SYMBOL = [\n\t'C', 'N', 'O', 'S', 'F', 'Si', 'P', 'Cl', 'Br', 'Mg',\n\t'Na', 'Ca', 'Fe', 'As', 'Al', 'I', 'B', 'V', 'K', 'Tl',\n\t'Yb', 'Sb', 'Sn', 'Ag', 'Pd', 'Co', 'Se', 'Ti', 'Zn', 'H',\n\t'Li', 'Ge', 'Cu', 'Au', 'Ni', 'Cd', 'In', 'Mn', 'Zr', 'Cr',\n\t'Pt', 'Hg', 'Pb', 'Dy',\n\t#'Unknown'\n]\n#print('ATOM_SYMBOL', len(ATOM_SYMBOL))44\nHYBRIDIZATION_TYPE = [\n\tChem.rdchem.HybridizationType.S,\n\tChem.rdchem.HybridizationType.SP,\n\tChem.rdchem.HybridizationType.SP2,\n\tChem.rdchem.HybridizationType.SP3,\n\tChem.rdchem.HybridizationType.SP3D\n]\n\ndef get_atom_feature(atom):\n\tfeature = (\n\t\t one_of_k_encoding(atom.GetSymbol(), ATOM_SYMBOL)\n\t   + one_of_k_encoding(atom.GetHybridization(), HYBRIDIZATION_TYPE)\n\t   + one_of_k_encoding(atom.GetDegree(), [0, 1, 2, 3, 4, 5])\n\t   + one_of_k_encoding(atom.GetTotalNumHs(), [0, 1, 2, 3, 4, 5])\n\t   + one_of_k_encoding(atom.GetImplicitValence(), [0, 1, 2, 3, 4, 5])\n\t   + [atom.IsInRing()]\n\t   + [atom.GetIsAromatic()]\n\t)\n\t#feature = np.array(feature, dtype=np.uint8)\n\tfeature = np.packbits(feature)\n\treturn feature\n\n\n#Get features of an edge (one-hot encoding)\n'''\n\t1.single/double/triple/aromatic: 4 dimensions       \n\t2.the atom's hybridization: 1 dimensions\n\t3.whether the bond is on ring: 1 dimension          \n\tTotal: 6 dimensions\n'''\n\ndef get_bond_feature(bond):\n\tbond_type = bond.GetBondType()\n\tfeature = [\n\t\tbond_type == Chem.rdchem.BondType.SINGLE,\n\t\tbond_type == Chem.rdchem.BondType.DOUBLE,\n\t\tbond_type == Chem.rdchem.BondType.TRIPLE,\n\t\tbond_type == Chem.rdchem.BondType.AROMATIC,\n\t\tbond.GetIsConjugated(),\n\t\tbond.IsInRing()\n\t]\n\t#feature = np.array(feature, dtype=np.uint8)\n\tfeature = np.packbits(feature)\n\treturn feature\n\n\ndef smile_to_graph(smiles):\n\tmol = Chem.MolFromSmiles(smiles)\n\tN = mol.GetNumAtoms()\n\tnode_feature = []\n\tedge_feature = []\n\tedge = []\n\tfor i in range(mol.GetNumAtoms()):\n\t\tatom_i = mol.GetAtomWithIdx(i)\n\t\tatom_i_features = get_atom_feature(atom_i)\n\t\tnode_feature.append(atom_i_features)\n\n\t\tfor j in range(mol.GetNumAtoms()):\n\t\t\tbond_ij = mol.GetBondBetweenAtoms(i, j)\n\t\t\tif bond_ij is not None:\n\t\t\t\tedge.append([i, j])\n\t\t\t\tbond_features_ij = get_bond_feature(bond_ij)\n\t\t\t\tedge_feature.append(bond_features_ij)\n\tnode_feature=np.stack(node_feature)\n\tedge_feature=np.stack(edge_feature)\n\tedge = np.array(edge,dtype=np.uint8)\n\treturn N,edge,node_feature,edge_feature\n\ndef to_pyg_format(N,edge,node_feature,edge_feature):\n\tgraph = Data(\n\t\tidx=-1,\n\t\tedge_index = torch.from_numpy(edge.T).int(),\n\t\tx          = torch.from_numpy(node_feature).byte(),\n\t\tedge_attr  = torch.from_numpy(edge_feature).byte(),\n\t)\n\treturn graph\n\n#debug one example\ng = to_pyg_format(*smile_to_graph(smiles=\"C#CCOc1ccc(CNc2nc(NCc3cccc(Br)n3)nc(N[C@@H](CC#C)CC(=O)NC)n2)cc1\"))\nprint(g)\nprint('[Dy] is replaced by C !!')\nprint('smile_to_graph() ok!')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:46:39.532729Z","iopub.execute_input":"2025-03-26T08:46:39.533030Z","iopub.status.idle":"2025-03-26T08:46:39.571968Z","shell.execute_reply.started":"2025-03-26T08:46:39.532999Z","shell.execute_reply":"2025-03-26T08:46:39.571323Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#MODEL: simple MPNNModel\n#from https://github.com/chaitjo/geometric-gnn-dojo/blob/main/geometric_gnn_101.ipynb\n\n#DEVICE='cuda'\nDEVICE='cpu'\n\n# i have removed all comments here to jepp it clean. refer to orginal link for code comments\n# of MPNNModel\nclass MPNNLayer(MessagePassing):\n    def __init__(self, emb_dim=64, edge_dim=4, aggr='add'):\n        super().__init__(aggr=aggr)\n    \n        self.emb_dim = emb_dim\n        self.edge_dim = edge_dim\n        self.mlp_msg = nn.Sequential(\n            nn.Linear(2 * emb_dim + edge_dim, emb_dim), nn.BatchNorm1d(emb_dim), nn.ReLU(),\n            nn.Linear(emb_dim, emb_dim), nn.BatchNorm1d(emb_dim), nn.ReLU()\n        )\n        self.mlp_upd = nn.Sequential(\n            nn.Linear(2 * emb_dim, emb_dim), nn.BatchNorm1d(emb_dim), nn.ReLU(),\n            nn.Linear(emb_dim, emb_dim), nn.BatchNorm1d(emb_dim), nn.ReLU()\n        )\n    \n    def forward(self, h, edge_index, edge_attr):\n        out = self.propagate(edge_index, h=h, edge_attr=edge_attr)\n        return out\n    \n    def message(self, h_i, h_j, edge_attr):\n        msg = torch.cat([h_i, h_j, edge_attr], dim=-1)\n        return self.mlp_msg(msg)\n    \n    def aggregate(self, inputs, index):\n        return scatter(inputs, index, dim=self.node_dim, reduce=self.aggr)\n    \n    def update(self, aggr_out, h):\n        upd_out = torch.cat([h, aggr_out], dim=-1)\n        return self.mlp_upd(upd_out)\n    \n    def __repr__(self) -> str:\n        return (f'{self.__class__.__name__}(emb_dim={self.emb_dim}, aggr={self.aggr})')\n    \n\nclass MPNNModel(nn.Module):\n    def __init__(self, num_layers=4, emb_dim=64, in_dim=11, edge_dim=4, out_dim=1):\n        super().__init__()\n    \n        self.lin_in = nn.Linear(in_dim, emb_dim)\n    \n        # Stack of MPNN layers\n        self.convs = torch.nn.ModuleList()\n        for layer in range(num_layers):\n            self.convs.append(MPNNLayer(emb_dim, edge_dim, aggr='add'))\n    \n        self.pool = global_mean_pool\n    \n    def forward(self, data): #PyG.Data - batch of PyG graphs\n    \n        h = self.lin_in(F_unpackbits(data.x,-1).float())  \n    \n        for conv in self.convs:\n            h = h + conv(h, data.edge_index.long(), F_unpackbits(data.edge_attr,-1).float())  # (n, d) -> (n, d)\n    \n        h_graph = self.pool(h, data.batch)  \n        return h_graph\n\n# our prediction model here !!!!\nclass Net(nn.Module):\n    def __init__(self, ):\n        super().__init__()\n    \n        self.output_type = ['infer', 'loss']\n    \n        graph_dim=96\n        self.smile_encoder = MPNNModel(\n             in_dim=NODE_DIM, edge_dim=EDGE_DIM, emb_dim=graph_dim, num_layers=4,\n        )\n        self.bind = nn.Sequential(\n            nn.Linear(graph_dim, 1024),\n            #nn.BatchNorm1d(1024),\n            nn.ReLU(inplace=True),\n            nn.Dropout(0.1),\n            nn.Linear(1024, 1024),\n            #nn.BatchNorm1d(1024),\n            nn.ReLU(inplace=True),\n            nn.Dropout(0.1),\n            nn.Linear(1024, 512),\n            #nn.BatchNorm1d(512),\n            nn.ReLU(inplace=True),\n            nn.Dropout(0.1),\n            nn.Linear(512, 3),\n        )\n    \n    def forward(self, batch):\n        graph = batch['graph']\n        x = self.smile_encoder(graph) \n        bind = self.bind(x)\n    \n        # --------------------------\n        output = {}\n        if 'loss' in self.output_type:\n            target = batch['bind']\n            output['bce_loss'] = F.binary_cross_entropy_with_logits(bind.float(), target.float())\n            \"\"\"\n            # Debugging loss calculation新增\n            print(\"Bind Output (Sigmoid):\", torch.sigmoid(bind).detach().cpu().numpy())\n            print(\"Target:\", target.cpu().numpy())\n            print(\"Loss Value:\", output['bce_loss'].item())\n            \"\"\"\n    \n        if 'infer' in self.output_type:\n            output['bind'] = torch.sigmoid(bind)\n    \n        return output\n    \n#debug: make some dummy data and run\n'''\ndef run_check_net():\n    batch_size = 30\n    node_dim=NODE_DIM\n    edge_dim=EDGE_DIM\n    \n    data = []\n    for b in range(batch_size):\n        N = np.random.randint(5,10)\n        E = np.random.randint(3,N*(N-1))\n        edge_index = np.stack([\n            np.random.choice(N, E, replace=True),\n            np.random.choice(N, E, replace=True),\n        ]).T\n        edge_index = np.sort(edge_index)\n        edge_index = edge_index[edge_index[:, 0].argsort()]\n        edge_index[0] = [0,1] #default\n        edge_index = edge_index[edge_index[:,0]!=edge_index[:,1]]\n        edge_index = np.unique(edge_index, axis=0)\n    \n        E = len(edge_index)\n        edge_index = np.ascontiguousarray(edge_index.T)\n    \n        d = Data(\n            idx        = b,\n            edge_index = torch.from_numpy(edge_index).int(),\n            x          = torch.from_numpy(np.packbits(np.random.choice(2, (N, node_dim)),-1)).byte(),\n            edge_attr  = torch.from_numpy(np.packbits(np.random.choice(2, (E, edge_dim)),-1)).byte(),\n        )\n        data.append(d)\n    \n    \n    loader = DataLoader(data, batch_size=batch_size, shuffle=True)\n    epoch_indices = []  # 儲存 index 以檢查跳變\n    for batch in loader:\n        epoch_indices.extend(batch.idx.tolist())\n    \n    \n    # 📌 記錄當前 epoch 所有 batch index\n    print(f\"Epoch index range: {min(epoch_indices)} → {max(epoch_indices)}\")\n    \n    \n    # loader = DataLoader(data, batch_size=batch_size)\n    graph = next(iter(loader))\n    idx = graph.idx.tolist()  #use to index bind array\n    batch = dotdict( \n        graph = graph.to(DEVICE),\n        bind  = torch.from_numpy(np.random.choice(2, (batch_size, 3))).float().to(DEVICE),\n    )\n    zz=0\n    \n    net = Net().to(DEVICE)\n    #print(net)\n    \n    with torch.no_grad():\n        with torch.amp.autocast('cuda', enabled=True): # dtype=torch.float16):\n            output = net(batch)\n            #print(output['bind'])\n    \n    # ---\n    print('batch')\n    for k, v in batch.items():\n        if k=='idx':\n            print(f'{k:>32} : {len(v)} ')\n        elif k=='graph':\n            print(f'{k:>32} : {graph} ')\n        else:\n            print(f'{k:>32} : {v.shape} ')\n    \n    print('output')\n    for k, v in output.items():\n        if 'loss' not in k:\n            print(f'{k:>32} : {v.shape} ')\n    print('loss')\n    for k, v in output.items():\n        if 'loss' in k:\n            print(f'{k:>32} : {v.item()} ')\n    \n                \nrun_check_net()\n'''\nprint('model ok!')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:46:39.572702Z","iopub.execute_input":"2025-03-26T08:46:39.572916Z","iopub.status.idle":"2025-03-26T08:46:39.586033Z","shell.execute_reply.started":"2025-03-26T08:46:39.572887Z","shell.execute_reply":"2025-03-26T08:46:39.585324Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#example of parallel conversion of smiles to graph\n'''\nfrom multiprocessing import Pool\nfrom tqdm import tqdm\nimport gc\nfrom torch_geometric.loader import DataLoader as PyGDataLoader\n\ndef to_pyg_list(graph):\n\tL = len(graph)\n\tfor i in tqdm(range(L)):\n\t\tN, edge, node_feature, edge_feature = graph[i]\n\t\tgraph[i] = Data(\n\t\t\tidx=i,\n\t\t\tedge_index=torch.from_numpy(edge.T).int(),\n\t\t\tx=torch.from_numpy(node_feature).byte(),\n\t\t\tedge_attr=torch.from_numpy(edge_feature).byte(),\n\t\t)\n\treturn graph\n\n\ntrain_smiles=[ #replace [Dy] with C\n    \"C#CCOc1ccc(CNc2nc(NCc3cccc(Br)n3)nc(N[C@@H](CC#C)CC(=O)NC)n2)cc1\",\n    \"C#CCOc1ccc(CNc2nc(NCc3cccc(Br)n3)nc(N[C@@H](CC#C)CC(=O)NC)n2)cc1\",\n    \"C#CCOc1ccc(CNc2nc(NCc3cccc(Br)n3)nc(N[C@@H](CC#C)CC(=O)NC)n2)cc1\",\n    \"C#CCOc1ccc(CNc2nc(NCc3cccc(Br)n3)nc(N[C@@H](CC#C)CC(=O)NC)n2)cc1\",\n    \"C#CCOc1ccc(CNc2nc(NCc3cccc(Br)n3)nc(N[C@@H](CC#C)CC(=O)NC)n2)cc1\",\n    \"C#CCOc1ccc(CNc2nc(NCc3cccc(Br)n3)nc(N[C@@H](CC#C)CC(=O)NC)n2)cc1\",\n]\ntrain_bind =np.array([\n    [0,0,0],[1,0,0],[0,1,0],[0,0,1],[1,1,0],[0,0,0],\n])\nnum_train= len(train_smiles)\nwith Pool(processes=64) as pool:\n    train_graph = list(tqdm(pool.imap(smile_to_graph, train_smiles), total=num_train))\n\ntrain_graph = to_pyg_list(train_graph)\ntrain_loader = PyGDataLoader(train_graph, batch_size=3, shuffle=True)\n\n## example training loop\nscaler = torch.cuda.amp.GradScaler(enabled=True)\nnet = Net()\nnet.to(DEVICE)\n\noptimizer =\\\n\ttorch.optim.AdamW(filter(lambda p: p.requires_grad, net.parameters()), lr=0.001)\n\nnum_epoch=10\nepoch=0\niteration=0\nwhile epoch<num_epoch: \n\tfor t, graph_batch in enumerate(train_loader): \n\t\tindex = graph_batch.idx.tolist()\n\t\tB = len(index)\n\t\tbatch = dotdict(\n\t\t\tgraph  = graph_batch.to(DEVICE),\n\t\t\tbind   = torch.from_numpy(train_bind[index]).to(DEVICE),\n\t\t)\n\n\t\tnet.train()\n\t\tnet.output_type = ['loss', 'infer']\n\t\twith torch.cuda.amp.autocast(enabled=True):\n\t\t\toutput = net(batch)  #data_parallel(net,batch) #\n\t\t\tbce_loss = output['bce_loss']\n\n\t\toptimizer.zero_grad() \n\t\tscaler.scale(bce_loss).backward() \n\t\tscaler.step(optimizer)\n\t\tscaler.update()\n\t\t \n\t\ttorch.clear_autocast_cache()\n\t\tprint(epoch,iteration,bce_loss.item())\n\t\titeration +=  1\n        \n\tepoch += 1\n'''","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:46:39.586718Z","iopub.execute_input":"2025-03-26T08:46:39.586931Z","iopub.status.idle":"2025-03-26T08:46:39.601652Z","shell.execute_reply.started":"2025-03-26T08:46:39.586907Z","shell.execute_reply":"2025-03-26T08:46:39.600886Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nimport numpy as np\nimport torch\nfrom torch_geometric.data import Data\nfrom multiprocessing import Pool\nfrom tqdm import tqdm\nfrom torch_geometric.loader import DataLoader as PyGDataLoader\n\n\n# 轉換為 PyG 格式\ndef to_pyg_list(graph):\n\tL = len(graph)\n\tfor i in tqdm(range(L)):\n\t\tN, edge, node_feature, edge_feature = graph[i]\n\t\tgraph[i] = Data(\n\t\t\tidx=i,\n\t\t\tedge_index=torch.from_numpy(edge.T).int(),\n\t\t\tx=torch.from_numpy(node_feature).byte(),\n\t\t\tedge_attr=torch.from_numpy(edge_feature).byte(),\n\t\t)\n\treturn graph\n\n\ntrain_file = \"/kaggle/input/trainall0/sampled_train_all.parquet\"\ndf = pd.read_parquet(train_file)\n\n\n# train_loader\n# 從 df 讀取 SMILES\ntrain_smiles = df['molecule_smiles'].tolist()\n\n\ntrain_bind = df[[\"BRD4\", \"HSA\", \"sEH\"]].values\n#train_bind = np.array([[1, 0, 0] if b == 1 else [0, 0, 0] for b in train_bind])  # 假設 binds 是 0/1\n\nnum_train = len(train_smiles)\n\n# 平行處理 SMILES 轉 Graph\nwith Pool(processes=8) as pool:  # 設為 8 核心，避免記憶體爆炸\n    train_graph = list(tqdm(pool.imap(smile_to_graph, train_smiles), total=num_train))\n\n\ntrain_graph = to_pyg_list(train_graph)\n#train_loader = PyGDataLoader(train_graph, batch_size=3, shuffle=True)\ntrain_loader = PyGDataLoader(train_graph, batch_size=3, shuffle=True, worker_init_fn=np.random.seed(42))\n\n\n\n# val_loader\n\nval_file = \"/kaggle/working/sampled_train_all.parquet\"\nval_df = pd.read_parquet(val_file)\n\n# 從 df 讀取 SMILES\nval_smiles = val_df['molecule_smiles'].tolist()\n\n\nval_bind = val_df[[\"BRD4\", \"HSA\", \"sEH\"]].values\n\n\nnum_val = len(val_smiles)\n\n# 平行處理 SMILES 轉 Graph\nwith Pool(processes=8) as pool:  # 設為 8 核心，避免記憶體爆炸\n    val_graph = list(tqdm(pool.imap(smile_to_graph, val_smiles), total=num_val))\n\n\nval_graph = to_pyg_list(val_graph)\n\nnum_val_samples = 20000  # 取 20,000 筆\ntotal_val_samples = len(val_graph)\n\n# 隨機選取 20,000 個索引（如果數據少於 20,000，則全部選取）\nrandom_indices = np.random.choice(total_val_samples, min(num_val_samples, total_val_samples), replace=False)\n\n# 選取對應的 Graph 和 Binding Data\nval_graph_subset = [val_graph[i] for i in random_indices]\nval_bind_subset = val_bind[random_indices] \nval_bind =val_bind_subset\n\n\n# 建立 DataLoader\nval_loader = PyGDataLoader(val_graph_subset, batch_size=3, shuffle=True)\n\n\n\n## example training loop\nscaler = torch.cuda.amp.GradScaler(enabled=True)\n#net = Net()\nnet = torch.load(\"/kaggle/input/gnn/pytorch/default/1/gnn_model_finish.pth\")\nnet.to(DEVICE)\n\noptimizer =\\\n\ttorch.optim.AdamW(filter(lambda p: p.requires_grad, net.parameters()), lr=0.001)\n\nnum_epoch=5\nepoch=0\niteration=0\n\n\nwhile epoch<num_epoch: \n    for t, graph_batch in enumerate(train_loader): \n        index = graph_batch.idx.tolist()\n        B = len(index)\n        batch = dotdict(\n\t\t\tgraph  = graph_batch.to(DEVICE),\n\t\t\tbind   = torch.from_numpy(train_bind[index]).to(DEVICE),\n\t\t)\n\n        net.train()\n        net.output_type = ['loss', 'infer']\n        with torch.cuda.amp.autocast(enabled=True):\n            output = net(batch)  #data_parallel(net,batch) \n            bce_loss = output['bce_loss']\n\n        optimizer.zero_grad() \n        scaler.scale(bce_loss).backward() \n        scaler.step(optimizer)\n        scaler.update()\n\t\t \n        torch.clear_autocast_cache()\n        print(epoch,iteration,bce_loss.item())\n        iteration +=  1\n\n    # ===== Validation Step =====\n    net.eval()  # 切換為評估模式\n    val_loss = 0.0\n    num_batches = 0\n    num_samples = 0\n    \n    with torch.no_grad():\n        for batch_idx, val_graph_batch in enumerate(val_loader):  # 使用 enumerate 確保 batch 索引\n            if num_samples >= num_val_samples:\n                break  # 達到 50,000 筆後結束驗證\n    \n            # 這裡不能用 val_graph_batch.idx.tolist()，改成直接取 batch_idx\n            val_batch = dotdict(\n                graph = val_graph_batch.to(DEVICE),\n                bind = torch.from_numpy(val_bind_subset[batch_idx * 3 : (batch_idx + 1) * 3]).to(DEVICE),\n            )\n    \n            net.output_type = ['loss', 'infer']\n            val_output = net(val_batch)\n            val_loss += val_output['bce_loss'].item()\n            num_batches += 1\n            num_samples += len(val_batch.graph)\n    \n    avg_val_loss = val_loss / num_batches\n    print(f\"Validation Loss: {avg_val_loss:.4f}\")\n\n    epoch += 1\n\n'''","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:46:39.602489Z","iopub.execute_input":"2025-03-26T08:46:39.602752Z","iopub.status.idle":"2025-03-26T08:46:39.615927Z","shell.execute_reply.started":"2025-03-26T08:46:39.602721Z","shell.execute_reply":"2025-03-26T08:46:39.615113Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nimport numpy as np\nimport torch\nfrom torch_geometric.data import Data\nfrom multiprocessing import Pool\nfrom tqdm import tqdm\nfrom torch_geometric.loader import DataLoader as PyGDataLoader\nfrom sklearn.metrics import accuracy_score, f1_score\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\n\n# 轉換為 PyG 格式\ndef to_pyg_list(graph):\n\tL = len(graph)\n\tfor i in tqdm(range(L)):\n\t\tN, edge, node_feature, edge_feature = graph[i]\n\t\tgraph[i] = Data(\n\t\t\tidx=i,\n\t\t\tedge_index=torch.from_numpy(edge.T).int(),\n\t\t\tx=torch.from_numpy(node_feature).byte(),\n\t\t\tedge_attr=torch.from_numpy(edge_feature).byte(),\n\t\t)\n\treturn graph\n\n\ntrain_file = '/kaggle/input/trainall7/sampled_train_all_7.parquet'\ndf = pd.read_parquet(train_file)\n\n\n# train_loader\n# 從 df 讀取 SMILES\ntrain_smiles = df['molecule_smiles'].tolist()\n\n\ntrain_bind = df[[\"BRD4\", \"HSA\", \"sEH\"]].values\n#train_bind = np.array([[1, 0, 0] if b == 1 else [0, 0, 0] for b in train_bind])  # 假設 binds 是 0/1\n\nnum_train = len(train_smiles)\n\n# 平行處理 SMILES 轉 Graph\nwith Pool(processes=8) as pool:  # 設為 8 核心，避免記憶體爆炸\n    train_graph = list(tqdm(pool.imap(smile_to_graph, train_smiles), total=num_train))\n\n\ntrain_graph = to_pyg_list(train_graph)\n#train_loader = PyGDataLoader(train_graph, batch_size=3, shuffle=True)\n#train_loader = PyGDataLoader(train_graph, batch_size=3, shuffle=True, worker_init_fn=np.random.seed(42))\ntrain_loader = PyGDataLoader(train_graph, batch_size=9, shuffle=True, drop_last=False, worker_init_fn=np.random.seed(42))\n\n\n# val_loader\n\nval_file = \"/kaggle/input/trainall1/sampled_train_all_1.parquet\"\nval_df = pd.read_parquet(val_file)\n\n# 從 df 讀取 SMILES\nval_smiles = val_df['molecule_smiles'].tolist()\n\n\nval_bind = val_df[[\"BRD4\", \"HSA\", \"sEH\"]].values\n\n\nnum_val = len(val_smiles)\n\n# 平行處理 SMILES 轉 Graph\nwith Pool(processes=8) as pool:  # 設為 8 核心，避免記憶體爆炸\n    val_graph = list(tqdm(pool.imap(smile_to_graph, val_smiles), total=num_val))\n\n\nval_graph = to_pyg_list(val_graph)\n\nnum_val_samples = 20000  # 取 20,000 筆\ntotal_val_samples = len(val_graph)\n\n# 隨機選取 20,000 個索引（如果數據少於 20,000，則全部選取）\nrandom_indices = np.random.choice(total_val_samples, min(num_val_samples, total_val_samples), replace=False)\n\n# 選取對應的 Graph 和 Binding Data\nval_graph_subset = [val_graph[i] for i in random_indices]\nval_bind_subset = val_bind[random_indices] \nval_bind =val_bind_subset\n\n\n# 建立 DataLoader\nval_loader = PyGDataLoader(val_graph_subset, batch_size=9, shuffle=False, drop_last=False)\n\n\n\n\n## example training loop\nscaler = torch.cuda.amp.GradScaler(enabled=True)\n# scaler = torch.amp.GradScaler(enabled=True)\n\n#net = Net()\nnet = torch.load(\"/kaggle/input/gnn-v8-2/pytorch/default/1/gnn_model_finish8-2.pth\")\nnet.to(DEVICE)\n#net = torch.load(\"/kaggle/input/gnn-/pytorch/default/1/gnn_model_finish_4.pth\", map_location=torch.device('cpu'))\n\n\noptimizer = torch.optim.AdamW(filter(lambda p: p.requires_grad, net.parameters()), lr=0.0004)\nscheduler = ReduceLROnPlateau(optimizer, mode='min', factor=0.7, patience=3)\n\nnum_epoch=9\nepoch=0\niteration=0\n\n\nwhile epoch < num_epoch:\n    \n    epoch_loss = 0.0  # 紀錄整個 epoch 的 loss\n\n    for t, graph_batch in enumerate(train_loader): \n        epoch_indices = []  # 儲存 index 以檢查跳變\n        index = graph_batch.idx.tolist()\n        epoch_indices.extend(index)\n        B = len(index)\n\n        batch = dotdict(\n            graph = graph_batch.to(DEVICE),\n            bind = torch.from_numpy(train_bind[index]).to(DEVICE),\n        )\n\n        # 📌 記錄當前 epoch 所有 batch index\n        print(f\"Epoch {epoch+1}, Batch {t+1} - Index range: {min(epoch_indices)} → {max(epoch_indices)}\")\n\n        net.train()\n        net.output_type = ['loss', 'infer']\n        \n        with torch.amp.autocast(device_type='cuda', enabled=True):\n            output = net(batch)\n            bce_loss = output['bce_loss']\n\n        optimizer.zero_grad()\n        scaler.scale(bce_loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n\n        torch.cuda.empty_cache()  # 清除 CUDA 緩存\n\n        # 📌 更新 epoch 總 loss\n        epoch_loss += bce_loss.item()\n\n        print(f\"Epoch {epoch+1}, Iteration {iteration}, BCE Loss: {bce_loss.item()}\")\n        iteration += 1\n\n    # 計算 epoch 平均 loss 並更新學習率\n    epoch_loss /= len(train_loader)  # 計算 loss 平均值\n    scheduler.step(epoch_loss)  # 使用 ReduceLROnPlateau 更新 lr\n\n    print(f\"Epoch {epoch+1} 結束，平均 BCE Loss: {epoch_loss}, 當前學習率: {optimizer.param_groups[0]['lr']}\")\n    epoch += 1\n\n    \n    # ===== Validation Step =====\n    net.eval()  # 切換為評估模式\n    val_loss = 0.0\n    num_batches = 0\n    num_samples = 0\n    all_preds = []\n    all_labels = []\n    \n    with torch.no_grad():\n        for batch_idx, val_graph_batch in enumerate(val_loader):  # 使用 enumerate 確保 batch 索引\n            if num_samples >= num_val_samples:\n                break  # 達到 50,000 筆後結束驗證\n    \n            val_batch = dotdict(\n                graph = val_graph_batch.to(DEVICE),\n                bind = torch.from_numpy(val_bind_subset[batch_idx * 9 : (batch_idx + 1) * 9]).to(DEVICE),\n            )\n\n    \n            net.output_type = ['loss', 'infer']\n            val_output = net(val_batch)\n            val_loss += val_output['bce_loss'].item()\n            num_batches += 1\n            num_samples += len(val_batch.graph)\n    \n            # 取得預測值（通常為機率），轉為 0/1\n            probs = val_output['bind'].detach().cpu().numpy()  # 轉為 NumPy\n            preds = (probs > 0.5).astype(int)  # 設定閾值 0.5\n            labels = val_batch.bind.cpu().numpy()\n    \n            all_preds.append(preds)\n            all_labels.append(labels)\n    \n    # 計算平均 Loss\n    avg_val_loss = val_loss / num_batches\n    \n    # 將所有 batch 的預測值與標籤合併\n    all_preds = np.vstack(all_preds)\n    all_labels = np.vstack(all_labels)\n    \n    # 計算 Accuracy 和 F1 Score\n    accuracy = accuracy_score(all_labels, all_preds)\n    f1 = f1_score(all_labels, all_preds, average='macro')  # 'macro' 計算所有類別的平均 F1\n    \n    print(f\"Validation Loss: {avg_val_loss:.4f}\")\n    print(f\"Validation Accuracy: {accuracy:.4f}\")\n    print(f\"Validation F1 Score: {f1:.4f}\")\n\n    epoch += 1\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:46:39.619804Z","iopub.execute_input":"2025-03-26T08:46:39.619993Z","iopub.status.idle":"2025-03-26T08:46:39.635076Z","shell.execute_reply.started":"2025-03-26T08:46:39.619976Z","shell.execute_reply":"2025-03-26T08:46:39.634271Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nmodel_path = \"/kaggle/working/gnn_model_finish.pth\"\ntorch.save(net, model_path)\nprint(f\"模型已保存至 {model_path}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:46:39.636712Z","iopub.execute_input":"2025-03-26T08:46:39.636903Z","iopub.status.idle":"2025-03-26T08:46:39.649365Z","shell.execute_reply.started":"2025-03-26T08:46:39.636887Z","shell.execute_reply":"2025-03-26T08:46:39.648768Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nfrom sklearn.metrics import accuracy_score, f1_score, roc_auc_score\nfrom torch.utils.data import DataLoader\nfrom torch_geometric.loader import DataLoader as PyGDataLoader\nfrom multiprocessing import Pool\n\n# 設定設備 (GPU or CPU)\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# 載入模型並切換為推理模式\nnet = torch.load(\"/kaggle/input/gnn-v9-9-2/pytorch/default/1/gnn_model_finish9.pth\", map_location=DEVICE)\nnet.to(DEVICE)\nnet.eval()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:46:39.751079Z","iopub.status.idle":"2025-03-26T08:46:39.751307Z","shell.execute_reply":"2025-03-26T08:46:39.751212Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 設定讀取 Parquet 文件\nfilename = '/kaggle/input/test-110/sampled_test_all.parquet'\ndf = pd.read_parquet(filename)\n\n# 提取測試數據\ntest_smiles = df['molecule_smiles'].tolist()\ny_true = df[[\"BRD4\", \"HSA\", \"sEH\"]].values  # 轉為 NumPy (Shape: [N, 3])\n\nnum_test = len(test_smiles)\n\n# 轉換 SMILES → Graph（使用 8 核心加速）\nwith Pool(processes=8) as pool:\n    test_graph = list(tqdm(pool.imap(smile_to_graph, test_smiles), total=num_test))\n\n# 轉換為 PyG 格式\ntest_graph = to_pyg_list(test_graph)\n\n# 建立 PyG DataLoader\ntest_loader = PyGDataLoader(test_graph, batch_size=3, shuffle=False, drop_last=False, worker_init_fn=np.random.seed(42))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:46:39.751894Z","iopub.status.idle":"2025-03-26T08:46:39.752202Z","shell.execute_reply":"2025-03-26T08:46:39.752097Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n\ny_true_df = pd.DataFrame(y_true)  \ny_true_df = y_true_df.fillna(0.0)\ny_true_df.isnull().sum()\ny_true_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:46:39.752955Z","iopub.status.idle":"2025-03-26T08:46:39.753323Z","shell.execute_reply":"2025-03-26T08:46:39.753162Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ===== 進行推論 =====\nall_preds = []\nall_probs = []\nall_labels = []\n\nwith torch.no_grad():\n    for batch in tqdm(test_loader, desc=\"Testing\"):\n        batch = batch.to(DEVICE)  # 確保數據在正確設備上\n        \n        # 構造模型輸入\n        test_batch = {\"graph\": batch}\n\n        # 獲取模型輸出\n        net.output_type = ['infer']  # 設置為推理模式\n        test_output = net(test_batch)\n\n        # 獲取預測機率\n        probs = test_output[\"bind\"].detach().cpu().numpy()  # Shape: [batch_size, 3]\n        \n        # 轉換為 0/1 預測值 (閾值 0.5)\n        preds = (probs > 0.5).astype(int)\n\n        # 儲存結果\n        all_probs.append(probs)\n        all_preds.append(preds)\n\n# 合併所有 batch 的預測結果\nall_probs = np.vstack(all_probs)  # 預測機率\nall_preds = np.vstack(all_preds)  # 二元預測結果\nall_labels = y_true_df.to_numpy()\n\n# ===== 計算指標 =====\naccuracy = accuracy_score(all_labels, all_preds)\nf1 = f1_score(all_labels, all_preds, average='macro')  # 'macro' 計算所有類別的平均 F1\nauc = roc_auc_score(all_labels, all_probs, average='macro')  # 直接使用機率計算 AUC\n\n# ===== 輸出結果 =====\n\nprint(f\"Test F1 Score: {f1:.4f}\")\nprint(f\"Test AUC Score: {auc:.4f}\")\nprint(f\"Test Accuracy: {accuracy:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:46:39.754455Z","iopub.status.idle":"2025-03-26T08:46:39.754816Z","shell.execute_reply":"2025-03-26T08:46:39.754671Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"submission","metadata":{}},{"cell_type":"code","source":"import torch\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nfrom sklearn.metrics import accuracy_score, f1_score, roc_auc_score\nfrom torch.utils.data import DataLoader\nfrom torch_geometric.loader import DataLoader as PyGDataLoader\nfrom multiprocessing import Pool\n\n# 設定設備 (GPU or CPU)\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# 載入模型並切換為推理模式\nnet = torch.load(\"/kaggle/input/gnn-v9-9-2/pytorch/default/1/gnn_model_finish9.pth\", map_location=DEVICE)\nnet.to(DEVICE)\nnet.eval()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:49:45.179230Z","iopub.execute_input":"2025-03-26T08:49:45.179661Z","iopub.status.idle":"2025-03-26T08:49:46.185460Z","shell.execute_reply.started":"2025-03-26T08:49:45.179618Z","shell.execute_reply":"2025-03-26T08:49:46.184747Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import duckdb\nimport pandas as pd\nfrom tqdm import tqdm\nimport numpy as np # linear algebra","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:49:46.186726Z","iopub.execute_input":"2025-03-26T08:49:46.187251Z","iopub.status.idle":"2025-03-26T08:49:46.419060Z","shell.execute_reply.started":"2025-03-26T08:49:46.187228Z","shell.execute_reply":"2025-03-26T08:49:46.418392Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\n# Process the test.parquet file chunk by chunk\ntest_file = '/kaggle/input/leash-BELKA/test.csv'\noutput_file = 'submission_gnn.csv'  # Specify the path and filename for the output file\n\ntest = pd.read_csv(test_file)\ntest.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:49:46.420484Z","iopub.execute_input":"2025-03-26T08:49:46.420809Z","iopub.status.idle":"2025-03-26T08:49:52.152364Z","shell.execute_reply.started":"2025-03-26T08:49:46.420786Z","shell.execute_reply":"2025-03-26T08:49:52.151473Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"protein_mapping = {\n    \"BRD4\": 0,\n    \"HSA\": 1,\n    \"sEH\": 2\n}\n\ntest['protein_name'] = test['protein_name'].map(protein_mapping)\n\ntest","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:49:52.153748Z","iopub.execute_input":"2025-03-26T08:49:52.154025Z","iopub.status.idle":"2025-03-26T08:49:52.268543Z","shell.execute_reply.started":"2025-03-26T08:49:52.154004Z","shell.execute_reply":"2025-03-26T08:49:52.267850Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n# 提取測試數據\ntest_smiles = df['molecule_smiles'].tolist()\n\n\nnum_test = len(test_smiles)\n\n# 轉換 SMILES → Graph（使用 8 核心加速）\nwith Pool(processes=8) as pool:\n    test_graph = list(tqdm(pool.imap(smile_to_graph, test_smiles), total=num_test))\n\n# 轉換為 PyG 格式\ntest_graph = to_pyg_list(test_graph)\n\n# 建立 PyG DataLoader\ntest_loader = PyGDataLoader(test_graph, batch_size=16, shuffle=False, drop_last=False)\n\nall_preds = []\nwith torch.no_grad():\n    for batch in tqdm(test_loader, desc=\"Testing\"):\n\n        batch = batch.to(DEVICE)  # 確保數據在正確設備上\n                \n        # 構造模型輸入\n        test_batch = {\"graph\": batch}\n        \n        # 獲取模型輸出\n        net.output_type = ['infer']  # 設置為推理模式\n        test_output = net(test_batch)\n        \n        # 獲取預測機率\n        probs = test_output[\"bind\"].detach().cpu().numpy()  # Shape: [batch_size, 3]\n        probs = pd.DataFrame(probs.values.flatten()\n        \n        # 轉換為 0/1 預測值 (閾值 0.5)\n        preds = (probs > 0.5).astype(int)\n        \n        # 儲存結果\n        all_preds.append(preds)\n\n# Create a DataFrame with 'id' and 'probability' columns\noutput_df = pd.DataFrame({'id': test['id'], 'binds': all_preds})\n\n# Save the output DataFrame to a CSV file\noutput_df.to_csv(output_file, index=False, mode='a', header=not os.path.exists(output_file))\n'''","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:49:52.269314Z","iopub.execute_input":"2025-03-26T08:49:52.269535Z","iopub.status.idle":"2025-03-26T08:49:52.274653Z","shell.execute_reply.started":"2025-03-26T08:49:52.269501Z","shell.execute_reply":"2025-03-26T08:49:52.273883Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 轉換為 PyG 格式\ndef to_pyg_list(graph):\n\tL = len(graph)\n\tfor i in tqdm(range(L)):\n\t\tN, edge, node_feature, edge_feature = graph[i]\n\t\tgraph[i] = Data(\n\t\t\tidx=i,\n\t\t\tedge_index=torch.from_numpy(edge.T).int(),\n\t\t\tx=torch.from_numpy(node_feature).byte(),\n\t\t\tedge_attr=torch.from_numpy(edge_feature).byte(),\n\t\t)\n\treturn graph","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:49:52.275600Z","iopub.execute_input":"2025-03-26T08:49:52.275896Z","iopub.status.idle":"2025-03-26T08:49:52.293921Z","shell.execute_reply.started":"2025-03-26T08:49:52.275864Z","shell.execute_reply":"2025-03-26T08:49:52.292948Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport pandas as pd\nimport os\nfrom tqdm import tqdm\nfrom multiprocessing import Pool\n\nBATCH_SIZE = 3  # 減少 batch_size 來降低記憶體負擔\nCHUNK_SIZE = 50000  # 每次處理 50000 筆\nOUTPUT_FILE = \"output.csv\"\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n# **確保 CSV 不會重複寫入 header**\nif os.path.exists(OUTPUT_FILE):\n    os.remove(OUTPUT_FILE)  # 先刪除，避免重複 append\n\n# **逐批處理 SMILES**\nfor chunk_start in range(0, len(test), CHUNK_SIZE):\n    chunk_end = min(chunk_start + CHUNK_SIZE, len(test))\n    test_chunk = test.iloc[chunk_start:chunk_end]  # **取出一小部分**\n    \n    test_smiles = test_chunk['molecule_smiles'].tolist()\n    test_protein_names = test_chunk['protein_name'].tolist()\n    test_ids = test_chunk['id'].tolist()\n    \n    # **逐批轉換 SMILES → Graph**\n    with Pool(processes=8) as pool:\n        test_graph = list(tqdm(pool.imap(smile_to_graph, test_smiles), total=len(test_smiles)))\n\n    # **轉換為 PyG 格式**\n    test_graph = to_pyg_list(test_graph)\n\n    # **建立 DataLoader**\n    test_loader = PyGDataLoader(test_graph, batch_size=BATCH_SIZE, shuffle=False, drop_last=False)\n\n    all_preds = []  # **存放當前 chunk 的預測結果**\n    batch_start = 0  # **批次索引追蹤**\n\n    with torch.no_grad():\n        for batch in tqdm(test_loader, desc=f\"Testing {chunk_start}/{len(test)}\"):\n            batch_size = batch.num_graphs\n            batch = batch.to(DEVICE)\n\n            # **構造模型輸入**\n            test_batch = {\"graph\": batch}\n            net.output_type = ['infer']\n            test_output = net(test_batch)  # **輸出形狀 [batch_size, num_proteins]**\n\n            for i in range(batch_size):  \n                protein_name = test_protein_names[batch_start + i]\n\n                if protein_name >= test_output[\"bind\"].size(1):\n                    print(f\"Warning: protein_name {protein_name} out of bounds for batch {i}\")\n                    continue\n                \n                protein_pred = test_output[\"bind\"][i, protein_name]\n                all_preds.append(1 if protein_pred > 0.5 else 0)\n\n            batch_start += batch_size\n\n    # **寫入 CSV，減少記憶體佔用**\n    output_df = pd.DataFrame({'id': test_ids, 'binds': all_preds})\n    output_df.to_csv(OUTPUT_FILE, index=False, mode='a', header=not os.path.exists(OUTPUT_FILE))\n\n    del test_graph, test_loader, all_preds  # **釋放記憶體**\n    torch.cuda.empty_cache()  # **清理 GPU 記憶體**\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T08:49:52.311946Z","iopub.execute_input":"2025-03-26T08:49:52.312274Z","iopub.status.idle":"2025-03-26T10:17:53.315894Z","shell.execute_reply.started":"2025-03-26T08:49:52.312243Z","shell.execute_reply":"2025-03-26T10:17:53.315118Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\ninput_file = '/kaggle/working/output.csv'\noutput_file = '/kaggle/working/submission_gnn.csv'\n# 讀取已儲存的 CSV\noutput_df = pd.read_csv(input_file)\n\nprint(len(output_df))\n\n# 修改 id 欄位\noutput_df['id'] = range(295246830, 295246830 + len(output_df))\n\nprint(output_df.shape)\n\n# 將修改後的 DataFrame 儲存回 CSV\noutput_df.to_csv(output_file, index=False, mode='a', header=not os.path.exists(output_file))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-26T10:58:33.281845Z","iopub.execute_input":"2025-03-26T10:58:33.282172Z","iopub.status.idle":"2025-03-26T10:58:34.485952Z","shell.execute_reply.started":"2025-03-26T10:58:33.282145Z","shell.execute_reply":"2025-03-26T10:58:34.485063Z"}},"outputs":[],"execution_count":null}]}