{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":67356,"databundleVersionId":8006601,"sourceType":"competition"},{"sourceId":8275617,"sourceType":"datasetVersion","datasetId":4914065},{"sourceId":8539699,"sourceType":"datasetVersion","datasetId":5101479},{"sourceId":8539714,"sourceType":"datasetVersion","datasetId":5101491},{"sourceId":8539739,"sourceType":"datasetVersion","datasetId":5101512},{"sourceId":8639399,"sourceType":"datasetVersion","datasetId":5173862},{"sourceId":8640782,"sourceType":"datasetVersion","datasetId":5174868},{"sourceId":8640904,"sourceType":"datasetVersion","datasetId":5174962},{"sourceId":8646854,"sourceType":"datasetVersion","datasetId":5179000},{"sourceId":8687302,"sourceType":"datasetVersion","datasetId":5208627},{"sourceId":8709535,"sourceType":"datasetVersion","datasetId":5224533},{"sourceId":8716567,"sourceType":"datasetVersion","datasetId":5229729},{"sourceId":8716571,"sourceType":"datasetVersion","datasetId":5229731},{"sourceId":8719261,"sourceType":"datasetVersion","datasetId":5231739},{"sourceId":8721196,"sourceType":"datasetVersion","datasetId":5233272},{"sourceId":8782564,"sourceType":"datasetVersion","datasetId":5279362}],"dockerImageVersionId":30715,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport keras \nimport os\nimport pyarrow.parquet as pq\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-07-10T12:04:45.165738Z","iopub.execute_input":"2024-07-10T12:04:45.166208Z","iopub.status.idle":"2024-07-10T12:05:03.697476Z","shell.execute_reply.started":"2024-07-10T12:04:45.166172Z","shell.execute_reply":"2024-07-10T12:05:03.696044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install rdkit\n# !pip install py3Dmol","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install dask \n# !pip install duckdb","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import duckdb\n# import pandas as pd\n\n# train_path = '/kaggle/input/leash-BELKA/train.parquet'\n# # test_path = '/kaggle/input/leash-BELKA/test.parquet'\n\n# con = duckdb.connect()\n\n# train_df = con.query(f\"\"\"(SELECT *\n#                         FROM parquet_scan('{train_path}')\n#                         WHERE binds = 0\n#                         ORDER BY random()\n#                         LIMIT 100000)\n#                         UNION ALL\n#                         (SELECT *\n#                         FROM parquet_scan('{train_path}')\n#                         WHERE binds = 1\n#                         ORDER BY random()\n#                         LIMIT 100000)\"\"\").df()\n\n# con.close()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import duckdb\n# import pandas as pd\n\n# train_path = '/kaggle/input/leash-bio-train-subsets/train_df1.parquet'\n# # test_path = '/kaggle/input/leash-BELKA/test.parquet'\n\n# con = duckdb.connect()\n\n# train_df0 = con.query(f\"\"\"(SELECT *\n#                         FROM parquet_scan('{train_path}')\n#                         WHERE binds = 0\n#                         ORDER BY random()\n#                         LIMIT 100000)\n#                         UNION ALL\n#                         (SELECT *\n#                         FROM parquet_scan('{train_path}')\n#                         WHERE binds = 1\n#                         ORDER BY random()\n#                         LIMIT 100000)\"\"\").df()\n\n# con.close()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import duckdb\n# import pandas as pd\n\n# # train_path = '/kaggle/input/leash-bio-preprocess-train/preprocessed_train_df_3.parquet'\n# test_path = '/kaggle/input/leash-BELKA/test.parquet'\n\n# con = duckdb.connect()\n\n# # Query to select rows from 0 to 50\n# test_df = con.query(f\"\"\"\n#     SELECT * \n#     FROM parquet_scan('{test_path}')\n#     ORDER BY id\n#     LIMIT 100000 OFFSET 0\n# \"\"\").df()\n\n\n# con.close()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df1_maccs = pd.read_parquet('/kaggle/input/leash-bio-train/train_df1_maccs.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:05:03.699848Z","iopub.execute_input":"2024-07-10T12:05:03.700640Z","iopub.status.idle":"2024-07-10T12:05:05.317568Z","shell.execute_reply.started":"2024-07-10T12:05:03.700604Z","shell.execute_reply":"2024-07-10T12:05:05.316265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create new columns initialized to NaN\ntrain_df1_maccs['bind1'] = np.nan\ntrain_df1_maccs['bind2'] = np.nan\ntrain_df1_maccs['bind3'] = np.nan\n\n# Assign values to the new columns based on the protein_name condition\ntrain_df1_maccs.loc[train_df1_maccs['protein_name'] == 'BRD4', 'bind1'] = train_df1_maccs.loc[train_df1_maccs['protein_name'] == 'BRD4', 'binds'].values\ntrain_df1_maccs.loc[train_df1_maccs['protein_name'] == 'HSA', 'bind2'] = train_df1_maccs.loc[train_df1_maccs['protein_name'] == 'HSA', 'binds'].values\ntrain_df1_maccs.loc[train_df1_maccs['protein_name'] == 'sEH', 'bind3'] = train_df1_maccs.loc[train_df1_maccs['protein_name'] == 'sEH', 'binds'].values\n\ntrain_df1_maccs.fillna(0, inplace=True)\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:05:05.318969Z","iopub.execute_input":"2024-07-10T12:05:05.319355Z","iopub.status.idle":"2024-07-10T12:05:05.735828Z","shell.execute_reply.started":"2024-07-10T12:05:05.319323Z","shell.execute_reply":"2024-07-10T12:05:05.734589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_df1_maccs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df1_atoms = pd.read_parquet('/kaggle/input/leash-bio-torsions-atoms/train_df1_atoms.parquet')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df1_atoms","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df1_bbs = pd.read_parquet('/kaggle/input/leash-bio-bbs-maccs-train/train_df1_bb2_bb3_maccs.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:05:05.738976Z","iopub.execute_input":"2024-07-10T12:05:05.739770Z","iopub.status.idle":"2024-07-10T12:05:07.304251Z","shell.execute_reply.started":"2024-07-10T12:05:05.739724Z","shell.execute_reply":"2024-07-10T12:05:07.302875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df1_bbs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# len(train_df1_maccs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df1_maccs['binds'].dtype","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df2_maccs = pd.read_parquet('/kaggle/input/leash-bio-train/train_df2_maccs.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:05:07.305766Z","iopub.execute_input":"2024-07-10T12:05:07.306276Z","iopub.status.idle":"2024-07-10T12:05:08.452871Z","shell.execute_reply.started":"2024-07-10T12:05:07.306232Z","shell.execute_reply":"2024-07-10T12:05:08.451578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create new columns initialized to NaN\ntrain_df2_maccs['bind1'] = np.nan\ntrain_df2_maccs['bind2'] = np.nan\ntrain_df2_maccs['bind3'] = np.nan\n\n# Assign values to the new columns based on the protein_name condition\ntrain_df2_maccs.loc[train_df2_maccs['protein_name'] == 'BRD4', 'bind1'] = train_df2_maccs.loc[train_df2_maccs['protein_name'] == 'BRD4', 'binds'].values\ntrain_df2_maccs.loc[train_df2_maccs['protein_name'] == 'HSA', 'bind2'] = train_df2_maccs.loc[train_df2_maccs['protein_name'] == 'HSA', 'binds'].values\ntrain_df2_maccs.loc[train_df2_maccs['protein_name'] == 'sEH', 'bind3'] = train_df2_maccs.loc[train_df2_maccs['protein_name'] == 'sEH', 'binds'].values\n\ntrain_df2_maccs.fillna(0, inplace=True)\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:05:08.454624Z","iopub.execute_input":"2024-07-10T12:05:08.455115Z","iopub.status.idle":"2024-07-10T12:05:08.871708Z","shell.execute_reply.started":"2024-07-10T12:05:08.455079Z","shell.execute_reply":"2024-07-10T12:05:08.870310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_df2_maccs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df2_atoms","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df2_atoms = pd.read_parquet('/kaggle/input/leash-bio-torsions-atoms/train_df2_atoms.parquet')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df2_bbs = pd.read_parquet('/kaggle/input/leash-bio-bbs-maccs-train/train_df2_bb2_bb3_maccs.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:05:08.874149Z","iopub.execute_input":"2024-07-10T12:05:08.874544Z","iopub.status.idle":"2024-07-10T12:05:10.331679Z","shell.execute_reply.started":"2024-07-10T12:05:08.874511Z","shell.execute_reply":"2024-07-10T12:05:10.329781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df2_bbs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df2_atoms","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df3_maccs = pd.read_parquet('/kaggle/input/leash-bio-train/train_df3_maccs.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:05:10.333665Z","iopub.execute_input":"2024-07-10T12:05:10.334183Z","iopub.status.idle":"2024-07-10T12:05:11.626428Z","shell.execute_reply.started":"2024-07-10T12:05:10.334144Z","shell.execute_reply":"2024-07-10T12:05:11.624724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create new columns initialized to NaN\ntrain_df3_maccs['bind1'] = np.nan\ntrain_df3_maccs['bind2'] = np.nan\ntrain_df3_maccs['bind3'] = np.nan\n\n# Assign values to the new columns based on the protein_name condition\ntrain_df3_maccs.loc[train_df3_maccs['protein_name'] == 'BRD4', 'bind1'] = train_df3_maccs.loc[train_df3_maccs['protein_name'] == 'BRD4', 'binds'].values\ntrain_df3_maccs.loc[train_df3_maccs['protein_name'] == 'HSA', 'bind2'] = train_df3_maccs.loc[train_df3_maccs['protein_name'] == 'HSA', 'binds'].values\ntrain_df3_maccs.loc[train_df3_maccs['protein_name'] == 'sEH', 'bind3'] = train_df3_maccs.loc[train_df3_maccs['protein_name'] == 'sEH', 'binds'].values\n\ntrain_df3_maccs.fillna(0, inplace=True)\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:05:11.633000Z","iopub.execute_input":"2024-07-10T12:05:11.633881Z","iopub.status.idle":"2024-07-10T12:05:12.054260Z","shell.execute_reply.started":"2024-07-10T12:05:11.633817Z","shell.execute_reply":"2024-07-10T12:05:12.053038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df3_bbs = pd.read_parquet('/kaggle/input/leash-bio-bbs-maccs-train/train_df3_bb2_bb3_maccs.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:05:12.056401Z","iopub.execute_input":"2024-07-10T12:05:12.056824Z","iopub.status.idle":"2024-07-10T12:05:13.600974Z","shell.execute_reply.started":"2024-07-10T12:05:12.056791Z","shell.execute_reply":"2024-07-10T12:05:13.599272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df3_bbs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df3_atoms = pd.read_parquet('/kaggle/input/leash-bio-torsions-atoms/train_df3_atoms.parquet')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df3_atoms","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df1_morgan = pd.read_parquet('/kaggle/input/leash-bio-train/train_df1_morgan.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:05:14.991531Z","iopub.execute_input":"2024-07-10T12:05:14.992006Z","iopub.status.idle":"2024-07-10T12:05:28.282732Z","shell.execute_reply.started":"2024-07-10T12:05:14.991973Z","shell.execute_reply":"2024-07-10T12:05:28.281519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df1_morgan","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df2_morgan = pd.read_parquet('/kaggle/input/leash-bio-train/train_df2_morgan.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:05:28.285317Z","iopub.execute_input":"2024-07-10T12:05:28.285848Z","iopub.status.idle":"2024-07-10T12:05:41.232038Z","shell.execute_reply.started":"2024-07-10T12:05:28.285803Z","shell.execute_reply":"2024-07-10T12:05:41.230456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df2_morgan","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df3_morgan = pd.read_parquet('/kaggle/input/leash-bio-train/train_df3_morgan.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:05:41.234162Z","iopub.execute_input":"2024-07-10T12:05:41.234686Z","iopub.status.idle":"2024-07-10T12:05:54.038851Z","shell.execute_reply.started":"2024-07-10T12:05:41.234640Z","shell.execute_reply":"2024-07-10T12:05:54.037291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df3_morgan","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df1_maccs['binds'] = train_df1_maccs['binds'].astype('float32')\n# train_df2_maccs['binds'] = train_df2_maccs['binds'].astype('float32')\n# train_df2_maccs['binds'] = train_df2_maccs['binds'].astype('float32')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df1_maccs['bind1'] = train_df1_maccs['bind1'].astype('float32')\ntrain_df1_maccs['bind2'] = train_df1_maccs['bind2'].astype('float32')\ntrain_df1_maccs['bind3'] = train_df1_maccs['bind3'].astype('float32')\ntrain_df2_maccs['bind1'] = train_df2_maccs['bind1'].astype('float32')\ntrain_df2_maccs['bind2'] = train_df2_maccs['bind2'].astype('float32')\ntrain_df2_maccs['bind3'] = train_df2_maccs['bind3'].astype('float32')\ntrain_df3_maccs['bind1'] = train_df3_maccs['bind1'].astype('float32')\ntrain_df3_maccs['bind2'] = train_df3_maccs['bind2'].astype('float32')\ntrain_df3_maccs['bind3'] = train_df3_maccs['bind3'].astype('float32')\n\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:05:54.042712Z","iopub.execute_input":"2024-07-10T12:05:54.043740Z","iopub.status.idle":"2024-07-10T12:05:54.067464Z","shell.execute_reply.started":"2024-07-10T12:05:54.043683Z","shell.execute_reply":"2024-07-10T12:05:54.065765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# protein_num = train_df1_maccs['protein_num']\nmaccs_keys = train_df1_maccs['maccs_keys']\nmorgs = train_df1_morgan['mol_morgan']\nbbs = train_df1_bbs[['bb2_maccs', 'bb3_maccs']]\n# atoms = train_df1_atoms['atom_pair_fingerprint']\n# binds = train_df1_maccs['binds']\nbind1 = train_df1_maccs['bind1']\nbind2 = train_df1_maccs['bind2']\nbind3 = train_df1_maccs['bind3']","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:05:54.069618Z","iopub.execute_input":"2024-07-10T12:05:54.070206Z","iopub.status.idle":"2024-07-10T12:05:54.095494Z","shell.execute_reply.started":"2024-07-10T12:05:54.070154Z","shell.execute_reply":"2024-07-10T12:05:54.094042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df_1 = pd.concat([maccs_keys, morgs, bbs, bind1, bind2, bind3], axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:05:54.097558Z","iopub.execute_input":"2024-07-10T12:05:54.098033Z","iopub.status.idle":"2024-07-10T12:05:54.273719Z","shell.execute_reply.started":"2024-07-10T12:05:54.097994Z","shell.execute_reply":"2024-07-10T12:05:54.272400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_df_1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df_1 = pd.concat([protein_num, maccs_keys, atoms, binds], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df_1 = pd.concat([protein_num, maccs_keys, mol_morgan, binds], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# protein_num = train_df2_maccs['protein_num']\nmaccs_keys = train_df2_maccs['maccs_keys']\nmorgs = train_df2_morgan['mol_morgan']\nbbs = train_df2_bbs[['bb2_maccs', 'bb3_maccs']]\n# atoms = train_df2_atoms['atom_pair_fingerprint']\n# binds = train_df2_maccs['binds']\nbind1 = train_df2_maccs['bind1']\nbind2 = train_df2_maccs['bind2']\nbind3 = train_df2_maccs['bind3']","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:05:54.275596Z","iopub.execute_input":"2024-07-10T12:05:54.276137Z","iopub.status.idle":"2024-07-10T12:05:54.302330Z","shell.execute_reply.started":"2024-07-10T12:05:54.276089Z","shell.execute_reply":"2024-07-10T12:05:54.300926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df_2 = pd.concat([maccs_keys, morgs, bbs, bind1, bind2, bind3], axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:05:54.341922Z","iopub.execute_input":"2024-07-10T12:05:54.342351Z","iopub.status.idle":"2024-07-10T12:05:54.456789Z","shell.execute_reply.started":"2024-07-10T12:05:54.342320Z","shell.execute_reply":"2024-07-10T12:05:54.455503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_df_2.tail(15)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_df_2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df_2 = pd.concat([protein_num, maccs_keys, atoms, binds], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df_2 = pd.concat([protein_num, maccs_keys, mol_morgan, binds], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# protein_num = train_df3_maccs['protein_num']\nmaccs_keys = train_df3_maccs['maccs_keys']\nmorgs = train_df3_morgan['mol_morgan']\nbbs = train_df3_bbs[['bb2_maccs', 'bb3_maccs']]\n# atoms = train_df3_atoms['atom_pair_fingerprint']\n# binds = train_df3_maccs['binds']\nbind1 = train_df3_maccs['bind1']\nbind2 = train_df3_maccs['bind2']\nbind3 = train_df3_maccs['bind3']","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:05:54.461058Z","iopub.execute_input":"2024-07-10T12:05:54.461503Z","iopub.status.idle":"2024-07-10T12:05:54.484566Z","shell.execute_reply.started":"2024-07-10T12:05:54.461467Z","shell.execute_reply":"2024-07-10T12:05:54.482892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df_3 = pd.concat([maccs_keys, morgs, bbs, bind1, bind2, bind3], axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:05:54.486766Z","iopub.execute_input":"2024-07-10T12:05:54.487271Z","iopub.status.idle":"2024-07-10T12:05:54.520437Z","shell.execute_reply.started":"2024-07-10T12:05:54.487222Z","shell.execute_reply":"2024-07-10T12:05:54.518690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df_3","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del morgs, atoms, maccs_keys","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_df_3.tail(15)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_df_3","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df_3 = pd.concat([protein_num, maccs_keys, atoms, binds], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df_3 = pd.concat([protein_num, maccs_keys, binds, mol_morgan], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df_maccs = pd.read_parquet('/kaggle/input/leash-bio-test/test_df.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:05:54.522765Z","iopub.execute_input":"2024-07-10T12:05:54.523243Z","iopub.status.idle":"2024-07-10T12:06:02.556308Z","shell.execute_reply.started":"2024-07-10T12:05:54.523206Z","shell.execute_reply":"2024-07-10T12:06:02.554833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test_df_maccs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df_bbs = pd.read_parquet('/kaggle/input/leash-bio-test-bbs-maccs/test_df_bb2_bb3_maccs.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:06:02.558600Z","iopub.execute_input":"2024-07-10T12:06:02.559181Z","iopub.status.idle":"2024-07-10T12:06:14.080815Z","shell.execute_reply.started":"2024-07-10T12:06:02.559133Z","shell.execute_reply":"2024-07-10T12:06:14.079521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df_bbs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df_morgs = pd.read_parquet('/kaggle/input/leash-bio-test/test_df_morgan.parquet')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test_df_atoms = pd.read_parquet('/kaggle/input/leash-bio-test-atoms/test_df_atoms.parquet')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport pyarrow.parquet as pq\nimport gc\n\n# def process(chunk_df):\n#     # Example processing function\n#     print(chunk_df.head())\n\n# # Define the chunk size (number of rows per chunk)\n# chunk_size = 100000\n\n# # Open the Parquet file using PyArrow\n# parquet_file = pq.ParquetFile('/kaggle/input/leash-bio-test-atoms/test_df_atoms.parquet')\n\n# # List to store DataFrame chunks\n# chunks = []\n\n# # Read the file in batches\n# for batch in parquet_file.iter_batches(batch_size=chunk_size):\n#     # Convert each batch to a pandas DataFrame\n#     chunk_df = batch.to_pandas()\n#     # Optional: Process the chunk immediately or store for later processing\n#     process(chunk_df)  # Example processing step\n#     chunks.append(chunk_df)\n    \n#     gc.collect()\n# # Combine all chunks into a single DataFrame\n# test_df_atoms = pd.concat(chunks, ignore_index=True)\n\n# # Verify the result\n# print(test_df_atoms.info())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df_atoms","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:06:14.083861Z","iopub.execute_input":"2024-07-10T12:06:14.084301Z","iopub.status.idle":"2024-07-10T12:06:14.430519Z","shell.execute_reply.started":"2024-07-10T12:06:14.084267Z","shell.execute_reply":"2024-07-10T12:06:14.428797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport pyarrow.parquet as pq\nimport gc\n\ndef process(chunk_df):\n    # Example processing function\n    print(chunk_df.head())\n\n# Define the chunk size (number of rows per chunk)\nchunk_size = 100000\n\n# Open the Parquet file using PyArrow\nparquet_file = pq.ParquetFile('/kaggle/input/leash-bio-test-morgans/train_df_morg.parquet')\n\n# List to store DataFrame chunks\nchunks = []\n\n# Read the file in batches\nfor batch in parquet_file.iter_batches(batch_size=chunk_size):\n    # Convert each batch to a pandas DataFrame\n    chunk_df = batch.to_pandas()\n    # Optional: Process the chunk immediately or store for later processing\n    process(chunk_df)  # Example processing step\n    chunks.append(chunk_df)\n    gc.collect()\n\n# Combine all chunks into a single DataFrame\ntest_df_morgs = pd.concat(chunks, ignore_index=True)\n\n# Verify the result\nprint(test_df_morgs.info())","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:06:14.433012Z","iopub.execute_input":"2024-07-10T12:06:14.433610Z","iopub.status.idle":"2024-07-10T12:07:13.492456Z","shell.execute_reply.started":"2024-07-10T12:06:14.433559Z","shell.execute_reply":"2024-07-10T12:07:13.491196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test_df_morgs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# len(test_df_maccs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:07:13.494566Z","iopub.execute_input":"2024-07-10T12:07:13.494976Z","iopub.status.idle":"2024-07-10T12:07:13.772843Z","shell.execute_reply.started":"2024-07-10T12:07:13.494915Z","shell.execute_reply":"2024-07-10T12:07:13.771424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df_morgan = pd.read_parquet('/kaggle/input/leash-bio-test-morgans/train_df_morg.parquet')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1. Pyarrow as pq.ParquetFile\n2. Dask delayed and dask compute","metadata":{}},{"cell_type":"code","source":"# import pyarrow.parquet as pq\n# test_df_morgs = []\n# parquet_file = pq.ParquetFile('/kaggle/input/leash-bio-test-morgans/train_df_morg.parquet')\n# for i in parquet_file.iter_batches(batch_size=1000):\n# #     print(\"RecordBatch\")\n# #     print(i.to_pandas())\n#     test_df_morgs.append(i)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import pyarrow.parquet as pq\n# import pyarrow as pa\n\n# # Concatenate all RecordBatches into a single table\n# combined_table = pa.Table.from_batches(test_df_morgs)\n\n# # Convert the table to a pandas DataFrame (if needed)\n# test_df_morgs_df = combined_table.to_pandas()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df_morgs_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import dask\n# import pandas as pd\n# from dask import delayed\n# import pyarrow.parquet as pq\n\n# # Define a function to read and compute a specific chunk of rows from the Parquet file\n# @delayed\n# def read_and_compute_chunk(file_path, start_row, chunk_size):\n#     df = pd.read_parquet(file_path, engine='pyarrow')\n#     return df.iloc[start_row:start_row + chunk_size]\n\n# # Specify the path to your Parquet file and the chunk size\n# file_path = '/kaggle/input/leash-bio-test-morgans/train_df_morg.parquet'\n# chunk_size = 10000\n\n# # Get the total number of rows in the Parquet file\n# total_rows = pq.ParquetFile(file_path).metadata.num_rows\n\n# # Create a list to hold the computed chunks\n# computed_chunks = []\n\n# # Loop through the file in increments of the chunk size\n# for start_row in range(0, total_rows, chunk_size):\n#     delayed_task = read_and_compute_chunk(file_path, start_row, chunk_size)\n#     computed_chunk = delayed_task.compute()  # Compute each chunk\n#     computed_chunks.append(computed_chunk)\n\n# # Concatenate all computed chunks\n# final_df = pd.concat(computed_chunks)\n\n# print(final_df)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df_morgan","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Define a dictionary to map categories to float values\n# category_to_float = {'HSA': 1.0, 'BRD4': 2.0, 'sEH': 3.0}\n\n\n# test_df_maccs['protein_num'] = test_df_maccs['protein_name'].map(category_to_float)\n# # test_df_morgs_df['protein_num'] = test_df_morgs_df['protein_name'].map(category_to_float)\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# protein_num = test_df_maccs['protein_num']\ntest_maccs_keys = test_df_maccs['maccs_keys']\ntest_morgs = test_df_morgs['mol_morgan']\ntest_bbs = test_df_bbs[['bb2_maccs', 'bb3_maccs']]\n# test_atoms = test_df_atoms['atom_pair_fingerprint']\n# # id = test_df_maccs['id']","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:07:13.776547Z","iopub.execute_input":"2024-07-10T12:07:13.777026Z","iopub.status.idle":"2024-07-10T12:07:13.851088Z","shell.execute_reply.started":"2024-07-10T12:07:13.776989Z","shell.execute_reply":"2024-07-10T12:07:13.849492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.concat([test_maccs_keys, test_morgs, test_bbs], axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:07:13.852601Z","iopub.execute_input":"2024-07-10T12:07:13.853022Z","iopub.status.idle":"2024-07-10T12:07:14.002800Z","shell.execute_reply.started":"2024-07-10T12:07:13.852983Z","shell.execute_reply":"2024-07-10T12:07:14.001419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df = pd.concat([protein_num, maccs_keys, atoms, id], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df = pd.concat([protein_num, maccs_keys, mol_morgan , id], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import rdkit\n# from rdkit import Chem \n# from rdkit.Chem import Draw\n# import py3Dmol\n# from rdkit.Chem import AllChem\n# import dask.dataframe as dd \n# import duckdb\n# from rdkit.Chem import MACCSkeys\n# from joblib import Parallel, delayed\n# from tqdm import tqdm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras import layers\nimport pandas as pd\nfrom keras.models import Sequential\nfrom keras.layers import Dense\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dropout\nfrom tensorflow.keras.layers import Input, Dense\nfrom tensorflow.keras.models import Model","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:07:14.004465Z","iopub.execute_input":"2024-07-10T12:07:14.004886Z","iopub.status.idle":"2024-07-10T12:07:14.055161Z","shell.execute_reply.started":"2024-07-10T12:07:14.004854Z","shell.execute_reply":"2024-07-10T12:07:14.053875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def generate_maccs_keys(smiles):\n#     mol = Chem.MolFromSmiles(smiles)\n#     if mol is not None:\n#         keys = MACCSkeys.GenMACCSKeys(mol)\n#         return np.array(list(map(int, keys.ToBitString())), dtype=np.int8) # MACCS keys feature \n#     else:\n#         return None\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Define a function to generate Morgan fingerprints for a given SMILES string\n# def generate_morgan_fingerprints(smiles, radius=2, n_bits=2048):\n#     mol = Chem.MolFromSmiles(smiles)\n#     if mol is not None:\n#         fingerprints = AllChem.GetMorganFingerprintAsBitVect(mol, radius, nBits=n_bits)\n#         return np.array(list(map(int, fingerprints.ToBitString())), dtype=np.int8) # Morgan fingerprints feature\n#     else:\n#         return None","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df['maccs_keys'] = Parallel(n_jobs=500)(delayed(generate_maccs_keys)(smiles) for smiles in tqdm(train_df['molecule_smiles']))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df.to_parquet('train_df_100k.parquet', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df['maccs_keys'] = Parallel(n_jobs=700)(delayed(generate_maccs_keys)(smiles) for smiles in tqdm(test_df['molecule_smiles']))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Save the DataFrame to a Parquet file\n# test_df.to_parquet('test_df.parquet', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = pd.read_parquet('/kaggle/working/test_df.parquet')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_list = [train_df_1, train_df_2, train_df_3]","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:07:14.056750Z","iopub.execute_input":"2024-07-10T12:07:14.057125Z","iopub.status.idle":"2024-07-10T12:07:14.062961Z","shell.execute_reply.started":"2024-07-10T12:07:14.057093Z","shell.execute_reply":"2024-07-10T12:07:14.061455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for train_df in train_list:\n\n#     XP = np.array(train_df['protein_num'].values.reshape(-1, 1))\n    X1 = np.array(train_df['maccs_keys'].tolist())\n    X2 = np.array(train_df['mol_morgan'].tolist())\n    X3 = np.array(train_df['bb2_maccs'].tolist())\n    X4 = np.array(train_df['bb3_maccs'].tolist())\n\n    X = np.hstack((X1, X2, X3, X4))\n    y = np.array(train_df[['bind1', 'bind2', 'bind3']])\n\n    print(\"Shapes of arrays:\")\n#     print('XP:', XP.shape)\n    print(\"X1:\", X1.shape)\n    print(\"X2:\", X2.shape)\n    print(\"X3:\", X3.shape)\n    print(\"X4:\", X4.shape)\n    print(\"X:\", X.shape)\n    print('y', y.shape)","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:07:14.065016Z","iopub.execute_input":"2024-07-10T12:07:14.065507Z","iopub.status.idle":"2024-07-10T12:07:17.618061Z","shell.execute_reply.started":"2024-07-10T12:07:14.065465Z","shell.execute_reply":"2024-07-10T12:07:17.616499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X1, X2, X3","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:07:17.619541Z","iopub.execute_input":"2024-07-10T12:07:17.619916Z","iopub.status.idle":"2024-07-10T12:07:17.628671Z","shell.execute_reply.started":"2024-07-10T12:07:17.619886Z","shell.execute_reply":"2024-07-10T12:07:17.627125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:07:17.630607Z","iopub.execute_input":"2024-07-10T12:07:17.631101Z","iopub.status.idle":"2024-07-10T12:07:17.962632Z","shell.execute_reply.started":"2024-07-10T12:07:17.631067Z","shell.execute_reply":"2024-07-10T12:07:17.961323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import numpy as np\n\n# # Determine batch size\n# batch_size = 100\n\n# # Split the 'mol_morgan' column into batches\n# batches = np.array_split(test_df_morgs_df['mol_morgan'].tolist(), len(test_df_morgs_df) // batch_size + 1)\n\n# # Initialize list to hold batch arrays\n# XT2 = []\n\n# # Convert each batch to a numpy array and append to XT2\n# for batch in batches:\n#     batch_array = np.array(batch)\n#     XT2.append(batch_array)\n\n# # Concatenate all arrays in XT2 into a single array\n# XT2_combined = np.concatenate(XT2, axis=0)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# XT2_combined.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# XT2 = np.hstack(XT2_combined)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# XTP = np.array(test_df['protein_num'].values.reshape(-1, 1))\nXT1 = np.array(test_df['maccs_keys'].tolist())\nXT2 = np.array(test_df['mol_morgan'].tolist())\nXT3 = np.array(test_df['bb2_maccs'].tolist())\nXT4 = np.array(test_df['bb3_maccs'].tolist())\n# XT3 = np.array(test_df['atom_pair_fingerprint'].tolist())\nXT = np.hstack((XT1, XT2, XT3, XT4))\n\n\n\n               \n# print(\"Shapes of arrays:\")\n# print(\"XTP:\", XTP.shape)\nprint(\"XT1:\", XT1.shape)\nprint(\"XT2:\", XT2.shape)\nprint(\"XT3:\", XT3.shape)\nprint(\"XT4:\", XT4.shape)\nprint(\"XT:\", XT.shape)","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:07:17.967322Z","iopub.execute_input":"2024-07-10T12:07:17.968612Z","iopub.status.idle":"2024-07-10T12:07:28.773129Z","shell.execute_reply.started":"2024-07-10T12:07:17.968561Z","shell.execute_reply":"2024-07-10T12:07:28.771839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del XT1, XT2, XT3, XT4","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:07:28.775158Z","iopub.execute_input":"2024-07-10T12:07:28.775533Z","iopub.status.idle":"2024-07-10T12:07:28.793815Z","shell.execute_reply.started":"2024-07-10T12:07:28.775501Z","shell.execute_reply":"2024-07-10T12:07:28.792393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import numpy as np\n\n# # Assuming test_df is already defined and loaded with the data\n# XT1 = np.array(test_df['maccs_keys'].tolist())\n# XT2 = np.array(test_df['mol_morgan'].tolist())\n# XT3 = np.array(test_df['atom_pair_fingerprint'].tolist())\n\n# # Determine the total number of rows\n# total_rows = XT1.shape[0]\n# chunk_size = 10000\n\n# # Initialize an empty list to store the chunks\n# chunks = []\n\n# # Process each chunk\n# for start in range(0, total_rows, chunk_size):\n#     end = min(start + chunk_size, total_rows)\n#     chunk_XT1 = XT1[start:end]\n#     chunk_XT2 = XT2[start:end]\n#     chunk_XT3 = XT3[start:end]\n    \n#     # Stack the current chunk\n#     chunk = np.hstack((chunk_XT1, chunk_XT2, chunk_XT3))\n    \n#     # Append the chunk to the list\n#     chunks.append(chunk)\n#     gc.collect()\n# # Concatenate all chunks into a final array\n# XT = np.vstack(chunks)\n\n# # XT now contains the concatenated result of all chunks\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"XT.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from tensorflow.keras.layers import Input, Dense, Dropout, BatchNormalization, Reshape, Conv1D\n# from tensorflow.keras.models import Model\n# from tensorflow.keras.models import Model\n\n# # Create the base model\n# def create_model(input_shape):\n#     inputs = Input(shape=(input_shape,))\n#     x = Dense(1024, activation='relu')(inputs)\n#     x = BatchNormalization()(x)\n#     x = Dropout(0.2)(x)\n#     x = Dense(512, activation='relu')(x)\n#     x = Dropout(0.1)(x)\n#     x = Dense(512, activation='relu')(x)\n#     x = Dropout(0.1)(x)\n#     x = Dense(512, activation='relu')(x)\n#     x = BatchNormalization()(x)\n#     x = Dropout(0.1)(x)\n#     x = Dense(512, activation='relu')(x)\n#     x = Dropout(0.1)(x)\n#     x = Dense(512, activation='relu')(x)\n#     x = Dropout(0.1)(x)\n#     x = Dense(512, activation='relu')(x)\n#     x = Dropout(0.1)(x)\n#     x = Dense(256, activation='relu')(x)\n#     x = BatchNormalization()(x)\n#     x = Dropout(0.1)(x)\n#     x = Dense(128, activation='relu')(x)\n#     x = Dropout(0.1)(x)\n#     x = Dense(64, activation='relu')(x)\n#     x = Dropout(0.1)(x)\n#     outputs = Dense(3, activation='sigmoid')(x)\n#     model = Model(inputs=inputs, outputs=outputs)\n#     return model\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.layers import Input, Dense, Dropout, BatchNormalization\nfrom tensorflow.keras.models import Model\n\n# Create the base model\ndef create_model(input_shape):\n    inputs = Input(shape=(input_shape,))\n    x = Dense(1024, activation='swish')(inputs)  # Add inputs to the first Dense layer\n    x = BatchNormalization()(x)\n    x = Dropout(0.2)(x)\n    x = Dense(512, activation='swish')(x)\n    x = Dropout(0.1)(x)\n    x = Dense(512, activation='swish')(x)\n    x = Dropout(0.1)(x)\n    x = Dense(512, activation='swish')(x)\n    x = BatchNormalization()(x)\n    x = Dropout(0.1)(x)\n    x = Dense(512, activation='swish')(x)\n    x = Dropout(0.1)(x)\n    x = Dense(512, activation='swish')(x)\n    x = Dropout(0.1)(x)\n    x = Dense(512, activation='swish')(x)\n    x = Dropout(0.1)(x)\n    x = Dense(256, activation='swish')(x)\n    x = BatchNormalization()(x)\n    x = Dropout(0.1)(x)\n    x = Dense(128, activation='swish')(x)\n    x = Dropout(0.1)(x)\n    x = Dense(64, activation='swish')(x)\n    x = Dropout(0.1)(x)\n    outputs = Dense(3, activation='sigmoid')(x)\n    model = Model(inputs=inputs, outputs=outputs)\n    return model\n\n# Example usage\n# input_shape = 100  # Replace with the actual input shape\n# model = create_model(input_shape)\n# model.summary()\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:07:28.795595Z","iopub.execute_input":"2024-07-10T12:07:28.796066Z","iopub.status.idle":"2024-07-10T12:07:28.811562Z","shell.execute_reply.started":"2024-07-10T12:07:28.796031Z","shell.execute_reply":"2024-07-10T12:07:28.810014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from tensorflow.keras.models import Model\n# from tensorflow.keras.layers import Input, Dense, Dropout, BatchNormalization\n# from tensorflow.keras.regularizers import l2\n\n# def create_model(input_shape):\n#     inputs = Input(shape=(input_shape,))\n    \n#     x = Dense(1024, activation='relu', kernel_regularizer=l2(0.01))(inputs)\n#     x = BatchNormalization()(x)\n#     x = Dropout(0.1)(x)\n    \n    \n#     x = Dense(512, activation='relu', kernel_regularizer=l2(0.01))(inputs)\n#     x = BatchNormalization()(x)\n#     x = Dropout(0.2)(x)\n    \n#     x = Dense(256, activation='relu', kernel_regularizer=l2(0.01))(x)\n#     x = BatchNormalization()(x)\n#     x = Dropout(0.3)(x)\n    \n#     x = Dense(128, activation='relu', kernel_regularizer=l2(0.01))(x)\n#     x = BatchNormalization()(x)\n#     x = Dropout(0.3)(x)\n    \n#     x = Dense(64, activation='relu', kernel_regularizer=l2(0.01))(x)\n#     x = BatchNormalization()(x)\n#     x = Dropout(0.3)(x)\n    \n#     outputs = Dense(3, activation='sigmoid')(x)\n    \n#     model = Model(inputs=inputs, outputs=outputs)\n#     return model\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import average_precision_score\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:07:28.813143Z","iopub.execute_input":"2024-07-10T12:07:28.813511Z","iopub.status.idle":"2024-07-10T12:07:29.429042Z","shell.execute_reply.started":"2024-07-10T12:07:28.813480Z","shell.execute_reply":"2024-07-10T12:07:29.427678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint, LearningRateScheduler\n\n# def lr_schedule(epoch):\n#     initial_lr = 0.005\n#     decay_factor = 0.9\n#     step_size = 10\n    \n#     # Adjust the learning rate every 'step_size' epochs\n#     if epoch < 3:\n#         return initial_lr\n#     else:\n#         return initial_lr * (decay_factor)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:07:29.430862Z","iopub.execute_input":"2024-07-10T12:07:29.431379Z","iopub.status.idle":"2024-07-10T12:07:29.437364Z","shell.execute_reply.started":"2024-07-10T12:07:29.431330Z","shell.execute_reply":"2024-07-10T12:07:29.436136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom keras.models import Model, clone_model\nfrom keras.layers import Input, Dense\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint, LearningRateScheduler\nimport tensorflow as tf\nfrom sklearn.model_selection import train_test_split\n\n\n# Train and evaluate the model on each subset\nfor i, train_df in enumerate(train_list):\n    print(f\"Processing DataFrame {i + 1}/{len(train_list)}\")\n    # Define a dictionary to map categories to float values\n    \n\n    # Create the base model\n    base_model = create_model(X.shape[1])\n    \n    # Create a copy of the base model for each iteration\n    model = clone_model(base_model)\n    model.set_weights(base_model.get_weights())  # Load weights from the base model\n    \n    # Prepare data\n    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n    \n    #Define callbacks\n    early_stopping = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True, mode='min')\n    reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.05, patience=3, min_lr=0.0001, mode='min')\n    checkpoint = ModelCheckpoint('best_model.keras', monitor='val_acc', save_best_only=True, mode='max')\n#     lr_scheduler = LearningRateScheduler(lr_schedule)\n\n    # Compile and fit the model\n    model.compile(loss='binary_crossentropy', optimizer='AdamW', metrics=[tf.keras.metrics.AUC(curve='PR', name='avg_precision')])\n    model.fit(X_train, y_train, \n              validation_data=(X_test, y_test), \n              epochs=2, \n              batch_size=128,\n              callbacks=[reduce_lr, early_stopping, checkpoint])\n    \n    # Model evaluation\n    y_pred = model.predict(XT)\n    preds.append(y_pred)","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:07:29.439107Z","iopub.execute_input":"2024-07-10T12:07:29.439591Z","iopub.status.idle":"2024-07-10T12:09:48.353101Z","shell.execute_reply.started":"2024-07-10T12:07:29.439547Z","shell.execute_reply":"2024-07-10T12:09:48.351132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import numpy as np\n# from keras.models import Model, clone_model\n# from keras.layers import Input, Dense\n# from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\n# from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint, LearningRateScheduler\n# import tensorflow as tf\n# from sklearn.model_selection import train_test_split\n\n\n# # Train and evaluate the model on each subset\n# for i, train_df in enumerate(train_list):\n#     print(f\"Processing DataFrame {i + 1}/{len(train_list)}\")\n#     # Define a dictionary to map categories to float values\n    \n\n    \n#     # Create the base model\n#     base_model = create_model(X.shape[1])\n    \n#     # Create a copy of the base model for each iteration\n#     model = clone_model(base_model)\n#     model.set_weights(base_model.get_weights())  # Load weights from the base model\n    \n#     # Prepare data\n#     X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n    \n#     #Define callbacks\n#     early_stopping = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True, mode='min')\n#     reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.05, patience=3, min_lr=0.0001, mode='min')\n#     checkpoint = ModelCheckpoint('best_model.keras', monitor='val_acc', save_best_only=True, mode='max')\n# #     lr_scheduler = LearningRateScheduler(lr_schedule)\n\n#     # Compile and fit the model\n#     model.compile(loss='binary_crossentropy', optimizer='Adam', metrics=[tf.keras.metrics.AUC(curve='PR', name='avg_precision')])\n#     model.fit(X_train, y_train, \n#               validation_data=(X_test, y_test), \n#               epochs=8, \n#               batch_size=64) \n#               #callbacks=[reduce_lr, early_stopping, checkpoint, lr_scheduler])\n    \n#     # Model evaluation\n#     y_pred = model.predict(XT)\n#     preds.append(y_pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert list of predictions to numpy array\npreds = np.array(preds)\n\n# Compute the mean of the predictions\nmean_preds = np.mean(preds, axis=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#mean_preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df['binds'] = mean_preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['id'] = test_df_maccs['id']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['protein_name'] = test_df_maccs['protein_name']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['binds'] = 0\ntest_df.loc[test_df['protein_name']=='BRD4', 'binds'] = mean_preds[(test_df['protein_name']=='BRD4').values, 0]\ntest_df.loc[test_df['protein_name']=='HSA', 'binds'] = mean_preds[(test_df['protein_name']=='HSA').values, 1]\ntest_df.loc[test_df['protein_name']=='sEH', 'binds'] = mean_preds[(test_df['protein_name']=='sEH').values, 2]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[['id', 'binds']].to_csv('submission.csv', index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}