{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":67356,"databundleVersionId":8006601,"sourceType":"competition"},{"sourceId":8042988,"sourceType":"datasetVersion","datasetId":4740586}],"dockerImageVersionId":30684,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This nobebook is to create train and test tables with consolidated smiles index from greySnow's Shrunken Dataset - https://www.kaggle.com/datasets/shlomoron/belka-shrunken-train-set\n\nHere is the new dataset - https://www.kaggle.com/datasets/makio323/belka-shrunken-with-unified-smiles-index\n\nAs train and test share some building blocks, using the unified smiles index (and/or fingerprints) may worthy to use.\n","metadata":{}},{"cell_type":"code","source":"!pip install duckdb\n!pip install venny4py","metadata":{"execution":{"iopub.status.busy":"2024-04-13T21:12:27.987127Z","iopub.execute_input":"2024-04-13T21:12:27.98748Z","iopub.status.idle":"2024-04-13T21:12:55.00907Z","shell.execute_reply.started":"2024-04-13T21:12:27.987451Z","shell.execute_reply":"2024-04-13T21:12:55.007659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nimport pickle\nimport duckdb as dd\nimport matplotlib.pyplot as plt\nfrom matplotlib_venn import venn3, venn2\nfrom venny4py.venny4py import *","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-13T21:12:55.0115Z","iopub.execute_input":"2024-04-13T21:12:55.011882Z","iopub.status.idle":"2024-04-13T21:12:56.731978Z","shell.execute_reply.started":"2024-04-13T21:12:55.011844Z","shell.execute_reply":"2024-04-13T21:12:56.730642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EDA on Building Blocks in train and test","metadata":{}},{"cell_type":"code","source":"# Reading all dictionaries from the greySnow's shrunken dataset\n\nBBs_dict_1 = pickle.load(open('/kaggle/input/belka-shrunken-train-set/train_dicts/BBs_dict_1.p', 'br'))\nBBs_dict_2 = pickle.load(open('/kaggle/input/belka-shrunken-train-set/train_dicts/BBs_dict_2.p', 'br'))\nBBs_dict_3 = pickle.load(open('/kaggle/input/belka-shrunken-train-set/train_dicts/BBs_dict_3.p', 'br'))\nBBs_dict_1_test = pickle.load(open('/kaggle/input/belka-shrunken-train-set/test_dicts/BBs_dict_1_test.p', 'br'))\nBBs_dict_2_test = pickle.load(open('/kaggle/input/belka-shrunken-train-set/test_dicts/BBs_dict_2_test.p', 'br'))\nBBs_dict_3_test = pickle.load(open('/kaggle/input/belka-shrunken-train-set/test_dicts/BBs_dict_3_test.p', 'br'))\n\nBBs_dict_reverse_1 = pickle.load(open('/kaggle/input/belka-shrunken-train-set/train_dicts/BBs_dict_reverse_1.p', 'br'))\nBBs_dict_reverse_2 = pickle.load(open('/kaggle/input/belka-shrunken-train-set/train_dicts/BBs_dict_reverse_2.p', 'br'))\nBBs_dict_reverse_3 = pickle.load(open('/kaggle/input/belka-shrunken-train-set/train_dicts/BBs_dict_reverse_3.p', 'br'))\nBBs_dict_reverse_1_test = pickle.load(open('/kaggle/input/belka-shrunken-train-set/test_dicts/BBs_dict_reverse_1_test.p', 'br'))\nBBs_dict_reverse_2_test = pickle.load(open('/kaggle/input/belka-shrunken-train-set/test_dicts/BBs_dict_reverse_2_test.p', 'br'))\nBBs_dict_reverse_3_test = pickle.load(open('/kaggle/input/belka-shrunken-train-set/test_dicts/BBs_dict_reverse_3_test.p', 'br'))","metadata":{"execution":{"iopub.status.busy":"2024-04-13T21:12:56.733647Z","iopub.execute_input":"2024-04-13T21:12:56.734131Z","iopub.status.idle":"2024-04-13T21:12:56.7958Z","shell.execute_reply.started":"2024-04-13T21:12:56.734097Z","shell.execute_reply":"2024-04-13T21:12:56.794844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bb_1 = BBs_dict_1.keys()\nbb_2 = BBs_dict_2.keys()\nbb_3 = BBs_dict_3.keys()\n\nbb_1_test = BBs_dict_1_test.keys()\nbb_2_test = BBs_dict_2_test.keys()\nbb_3_test = BBs_dict_3_test.keys()","metadata":{"execution":{"iopub.status.busy":"2024-04-13T21:12:56.798187Z","iopub.execute_input":"2024-04-13T21:12:56.798841Z","iopub.status.idle":"2024-04-13T21:12:56.803713Z","shell.execute_reply.started":"2024-04-13T21:12:56.798809Z","shell.execute_reply":"2024-04-13T21:12:56.802658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Among blocks in train \n\nNo overlapping between block1 and block2/block3, and large overlapping between block2 and block3","metadata":{}},{"cell_type":"code","source":"venn3([bb_1, bb_2, bb_3], ('BB1', 'BB2', 'BB3'))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-13T21:12:56.805276Z","iopub.execute_input":"2024-04-13T21:12:56.806019Z","iopub.status.idle":"2024-04-13T21:12:56.974447Z","shell.execute_reply.started":"2024-04-13T21:12:56.805981Z","shell.execute_reply":"2024-04-13T21:12:56.973308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Among blocks in test\n\nSimilar parttern of train","metadata":{}},{"cell_type":"code","source":"venn3([bb_1_test, bb_2_test, bb_3_test], ('BB1_test', 'BB2_test', 'BB3_test'))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-13T21:12:56.976531Z","iopub.execute_input":"2024-04-13T21:12:56.977407Z","iopub.status.idle":"2024-04-13T21:12:57.111734Z","shell.execute_reply.started":"2024-04-13T21:12:56.977359Z","shell.execute_reply":"2024-04-13T21:12:57.110481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Between train block1 and test block1\n\nAll the blocks in trains are included in test, and test has new blockes \n","metadata":{}},{"cell_type":"code","source":"venn2([bb_1, bb_1_test], ('BB1', 'BB1_test'))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-13T21:12:57.113979Z","iopub.execute_input":"2024-04-13T21:12:57.114888Z","iopub.status.idle":"2024-04-13T21:12:57.238106Z","shell.execute_reply.started":"2024-04-13T21:12:57.114836Z","shell.execute_reply":"2024-04-13T21:12:57.236833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Among train block2, block3, and test block2, block3\n\nQuite a large overlappings","metadata":{}},{"cell_type":"code","source":"sets = {\n    'BB2': set(bb_2),\n    'BB3': set(bb_3),\n    'BB2_test': set(bb_2_test),\n    'BB3_test': set(bb_3_test)\n}\n    \nvenny4py(sets=sets)","metadata":{"execution":{"iopub.status.busy":"2024-04-13T21:12:57.240299Z","iopub.execute_input":"2024-04-13T21:12:57.241254Z","iopub.status.idle":"2024-04-13T21:12:57.713403Z","shell.execute_reply.started":"2024-04-13T21:12:57.241206Z","shell.execute_reply":"2024-04-13T21:12:57.712166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create new train and test tables","metadata":{}},{"cell_type":"markdown","source":"## Consolidated dictionary","metadata":{}},{"cell_type":"code","source":"BBs_smiles = (BBs_dict_1 | BBs_dict_2 | BBs_dict_3 | BBs_dict_1_test | BBs_dict_2_test | BBs_dict_3_test).keys()\nBBs_dict = {x:np.int16(y) for x,y in zip(BBs_smiles, range(len(BBs_smiles)) )}\nBBs_dict_reverse = {np.int16(y):x for y,x in zip(range(len(BBs_smiles)), BBs_smiles)}","metadata":{"execution":{"iopub.status.busy":"2024-04-13T21:12:57.714704Z","iopub.execute_input":"2024-04-13T21:12:57.715068Z","iopub.status.idle":"2024-04-13T21:12:57.728169Z","shell.execute_reply.started":"2024-04-13T21:12:57.715032Z","shell.execute_reply":"2024-04-13T21:12:57.726815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(BBs_dict)","metadata":{"execution":{"iopub.status.busy":"2024-04-13T21:12:57.731327Z","iopub.execute_input":"2024-04-13T21:12:57.732308Z","iopub.status.idle":"2024-04-13T21:12:57.740349Z","shell.execute_reply.started":"2024-04-13T21:12:57.732264Z","shell.execute_reply":"2024-04-13T21:12:57.739273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read train and test from the shrunken dataset","metadata":{}},{"cell_type":"code","source":"con = dd.connect()","metadata":{"execution":{"iopub.status.busy":"2024-04-13T21:12:57.741556Z","iopub.execute_input":"2024-04-13T21:12:57.741845Z","iopub.status.idle":"2024-04-13T21:12:57.757093Z","shell.execute_reply.started":"2024-04-13T21:12:57.741822Z","shell.execute_reply":"2024-04-13T21:12:57.755497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfp = \"/kaggle/input/belka-shrunken-train-set/train.parquet\"\nsqlc = f\"(SELECT * FROM '{fp}')\"\ntrain = con.query(sqlc).df()\n\nprint(len(train))\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-13T21:12:57.758705Z","iopub.execute_input":"2024-04-13T21:12:57.759159Z","iopub.status.idle":"2024-04-13T21:13:46.079618Z","shell.execute_reply.started":"2024-04-13T21:12:57.759122Z","shell.execute_reply":"2024-04-13T21:13:46.078472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.dtypes","metadata":{"execution":{"iopub.status.busy":"2024-04-13T21:13:46.081291Z","iopub.execute_input":"2024-04-13T21:13:46.081978Z","iopub.status.idle":"2024-04-13T21:13:46.091056Z","shell.execute_reply.started":"2024-04-13T21:13:46.08192Z","shell.execute_reply":"2024-04-13T21:13:46.089796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfp = \"/kaggle/input/belka-shrunken-train-set/test.parquet\"\nsqlc = f\"(SELECT * FROM '{fp}')\"\ntest = con.query(sqlc).df()\n\nprint(len(test))\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-13T21:13:46.094111Z","iopub.execute_input":"2024-04-13T21:13:46.094516Z","iopub.status.idle":"2024-04-13T21:13:46.98934Z","shell.execute_reply.started":"2024-04-13T21:13:46.094485Z","shell.execute_reply":"2024-04-13T21:13:46.988036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.dtypes","metadata":{"execution":{"iopub.status.busy":"2024-04-13T21:13:46.990575Z","iopub.execute_input":"2024-04-13T21:13:46.990886Z","iopub.status.idle":"2024-04-13T21:13:46.999234Z","shell.execute_reply.started":"2024-04-13T21:13:46.99086Z","shell.execute_reply":"2024-04-13T21:13:46.998005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"con.close()","metadata":{"execution":{"iopub.status.busy":"2024-04-13T21:13:47.000823Z","iopub.execute_input":"2024-04-13T21:13:47.001356Z","iopub.status.idle":"2024-04-13T21:13:47.102902Z","shell.execute_reply.started":"2024-04-13T21:13:47.001318Z","shell.execute_reply":"2024-04-13T21:13:47.101682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Assign consolidated index","metadata":{}},{"cell_type":"code","source":"train['buildingblock1_smiles'] = [BBs_dict[BBs_dict_reverse_1[x]] for x in train.buildingblock1_smiles]\ntrain['buildingblock2_smiles'] = [BBs_dict[BBs_dict_reverse_2[x]] for x in train.buildingblock2_smiles]\ntrain['buildingblock3_smiles'] = [BBs_dict[BBs_dict_reverse_3[x]] for x in train.buildingblock3_smiles]","metadata":{"execution":{"iopub.status.busy":"2024-04-13T21:13:47.105013Z","iopub.execute_input":"2024-04-13T21:13:47.105588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.dtypes","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['buildingblock1_smiles'] = [BBs_dict[BBs_dict_reverse_1_test[x]] for x in test.buildingblock1_smiles]\ntest['buildingblock2_smiles'] = [BBs_dict[BBs_dict_reverse_2_test[x]] for x in test.buildingblock2_smiles]\ntest['buildingblock3_smiles'] = [BBs_dict[BBs_dict_reverse_3_test[x]] for x in test.buildingblock3_smiles]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.dtypes","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Write files","metadata":{}},{"cell_type":"code","source":"train.to_parquet('train.parquet', index = False)\ntrain.to_csv('train.csv', index = False)\n\ntest.to_parquet('test.parquet', index = False)\ntest.to_csv('test.csv', index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pickle.dump(BBs_dict, open('BBs_dict.p', 'bw'))\npickle.dump(BBs_dict_reverse, open('BBs_dict_reverse.p', 'bw'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}