{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":67356,"databundleVersionId":8006601,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Fingerprint Tips and Tricks\n\nMany people are looking at fingerprint-related features for this competition. I've spent a lot of time working with fingerprints, so I thought I'd compile some tips and tricks for generating and working with molecular fingerprints.","metadata":{}},{"cell_type":"code","source":"!pip install rdkit","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:03.079179Z","iopub.execute_input":"2024-05-31T21:34:03.079629Z","iopub.status.idle":"2024-05-31T21:34:21.155149Z","shell.execute_reply.started":"2024-05-31T21:34:03.079581Z","shell.execute_reply":"2024-05-31T21:34:21.153927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nfrom functools import partial\nimport time\n\nfrom scipy.spatial import distance\n\nimport torch\nimport torch.nn as nn\nfrom torch.nn.utils.rnn import pad_sequence\nfrom torch.profiler import profile, record_function, ProfilerActivity\n\nfrom rdkit import Chem\nfrom rdkit.Chem import rdMolDescriptors\nfrom rdkit.Chem import DataStructs\nfrom rdkit.Chem import rdFingerprintGenerator","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:27.724741Z","iopub.execute_input":"2024-05-31T21:34:27.725154Z","iopub.status.idle":"2024-05-31T21:34:31.778112Z","shell.execute_reply.started":"2024-05-31T21:34:27.725120Z","shell.execute_reply":"2024-05-31T21:34:31.776617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"To start, lets create a small dataset from 100 random smiles from the competition dataset","metadata":{}},{"cell_type":"code","source":"smiles = ['CC1CN(c2cc(CNc3nc(Nc4cc(C(=O)N[Dy])ccc4Cl)nc(Nc4nccnc4Cl)n3)ccn2)CCO1',\n       'O=C(N[Dy])c1cncc(Nc2nc(NCC3(N4CCOCC4)CC3)nc(Nc3c(O)ncnc3O)n2)n1',\n       'Cc1cnc(Cl)nc1Nc1nc(NCC(C)c2c(Cl)cccc2Cl)nc(N[C@@H](Cc2ccc([N+](=O)[O-])cc2)C(=O)N[Dy])n1',\n       'O=C(N[Dy])c1cccc(Nc2nc(NCCN3CCCS3(=O)=O)nc(Nc3cc(Cl)c([N+](=O)[O-])cn3)n2)c1',\n       'C#CC[C@H](Nc1nc(NCCS(=O)(=O)Nc2ccccc2)nc(NCc2ccnc(C(N)=O)c2)n1)C(=O)N[Dy]',\n       'COc1ccc(Nc2nc(Nc3cccc(S(C)(=O)=O)c3)nc(Nc3ccc(OC)c([N+](=O)[O-])c3)n2)c(C(=O)N[Dy])c1',\n       'O=C(N[Dy])[C@@H](CCC1CCCCC1)Nc1nc(NCc2nccn2-c2ccccc2)nc(Nc2ccc(O)cc2C(F)(F)F)n1',\n       'O=C(N[Dy])[C@H]1CC[C@@H](Nc2nc(Nc3ccnc(-c4ccccc4)c3)nc(Nc3cc(C4CC4)[nH]n3)n2)C1',\n       'CCSC1CCC1(O)CNc1nc(Nc2cccc(I)c2C(=O)N[Dy])nc(Nc2cccc3ocnc23)n1',\n       'COc1ccnc(Nc2nc(NCC#Cc3cccnc3)nc(N[C@H](Cc3ccc(Cl)c(Cl)c3)C(=O)N[Dy])n2)c1',\n       'CC(CNc1nc(Nc2cc[nH]n2)nc(N[C@@H](CCC2CCCCC2)C(=O)N[Dy])n1)S(N)(=O)=O',\n       'O=C(N[Dy])[C@@H]1Cc2ccc(O)cc2CN1c1nc(NCc2nnc(-c3ccncc3)[nH]2)nc(NCC2(N3CCOCC3)CCOCC2)n1',\n       'COc1cccc(Nc2nc(Nc3ccc4c(c3)oc3ccccc34)nc(N[C@@H]3CN(C(=O)OC(C)(C)C)C[C@H]3C(=O)N[Dy])n2)n1',\n       'Cn1ccnc1Cn1c(CCCNc2nc(Nc3ccnc(-c4ccccc4)c3)nc(N[C@@H](CCCN=[N+]=[N-])C(=O)N[Dy])n2)nc2c1CCCC2',\n       'COC1(C(F)(F)CNc2nc(NCc3nnc(-c4ccncc4)[nH]3)nc(N3CCC[C@H]3C(=O)N[Dy])n2)CCOCC1',\n       'COC(=O)c1cc(Nc2nc(Nc3ccc(Br)c(C(=O)N[Dy])c3)nc(Nc3ccc(C#N)cc3[N+](=O)[O-])n2)ccc1C',\n       'Cc1cc(Cl)cc(Nc2nc(NCC3CC4(CC4)CO3)nc(Nc3ccc(CC(=O)N[Dy])cc3)n2)c1',\n       'CCS(=O)(=O)N1CC(CNc2nc(NCCC3CCCC3(F)F)nc(Nc3nc(-c4ccc(C(=O)N[Dy])cc4)cs3)n2)C1',\n       'COCC1(CNc2nc(NCC(O)c3ccc(Cl)s3)nc(N[C@@H](CC(=O)N[Dy])Cc3ccc([N+](=O)[O-])cc3)n2)CCCCC1',\n       'Cc1cc(F)ccc1Nc1nc(NCc2cnns2)nc(NC2(C(=O)N[Dy])CCc3ccccc32)n1',\n       'COC(=O)c1c[nH]nc1Nc1nc(Nc2cc(Br)cnc2C(=O)N[Dy])nc(Nc2cc(C)nn2-c2ccccc2)n1',\n       'CC(C)(CCC#N)CNc1nc(Nc2cccnc2C(=O)N[Dy])nc(Nc2ncc(Cl)cc2F)n1',\n       'N#Cc1c(Nc2nc(NCC3(O)CC3)nc(Nc3c(C(=O)O)cccc3C(=O)N[Dy])n2)sc2c1CCCC2',\n       'Cc1cccc(CCCNc2nc(Nc3ccc(Br)c(C(=O)N[Dy])c3)nc(Nc3nccs3)n2)n1',\n       'COC1(C(F)(F)CNc2nc(Nc3[nH]c(=O)ncc3F)nc(N3CCCC[C@H]3C(=O)N[Dy])n2)CCOCC1',\n       'O=C(N[Dy])c1sc2ncccc2c1Nc1nc(NCCc2nccn2C(F)F)nc(NCc2cc(-c3ccccc3)[nH]n2)n1',\n       'Cc1cc(C(=O)N[Dy])ccc1Nc1nc(NCCN2CC[C@@H](O)C2)nc(NCc2cnns2)n1',\n       'COc1cncc(Nc2nc(NCc3cn(C)nc3Br)nc(Nc3c(Br)cccc3C(=O)N[Dy])n2)n1',\n       'COc1cnc(Nc2nc(NCC3CC4CC3C3CC43)nc(N3CCOC[C@@H]3C(=O)N[Dy])n2)nc1',\n       'O=C(N[Dy])c1ccccc1C(=O)c1ccccc1Nc1nc(NCc2cccc3c2OCO3)nc(Nc2ccc(O)cc2C(F)(F)F)n1',\n       'COC(=O)c1ccc2nc(Nc3nc(NCCc4ccno4)nc(N4CCCC[C@H]4C(=O)N[Dy])n3)sc2c1',\n       'C#CC[C@H](CC(=O)N[Dy])Nc1nc(NCCNC(=O)C(=C)C)nc(Nc2nc(-c3ccc(Cl)c(Cl)c3)cs2)n1',\n       'CCSCCNc1nc(NCC2(F)CCCCC2)nc(Nc2ccc(C(=O)N[Dy])cc2F)n1',\n       'N#Cc1c(Nc2nc(Nc3cncc(F)c3)nc(N[C@H](CC(=O)N[Dy])Cc3ccc(Br)cc3)n2)sc2c1CCCC2',\n       'COc1cnc(CNc2nc(Nc3nnc(C)s3)nc(Nc3c(C(=O)N[Dy])ccc(F)c3OC)n2)cn1',\n       'CCOC(=O)c1ncccc1Nc1nc(NCc2nnc3c(=O)[nH]ccn23)nc(Nc2nc(-c3ccc(C(=O)N[Dy])cc3)cs2)n1',\n       'O=C(NCCNc1nc(NCc2cc3n(n2)CCCO3)nc(N[C@@H](Cc2c(F)c(F)c(F)c(F)c2F)C(=O)N[Dy])n1)c1ccn[nH]1',\n       'O=C(N[Dy])[C@H](Cc1ccc(F)cc1F)Nc1nc(NCc2ccsc2C(F)(F)F)nc(Nc2cc(=O)[nH]c(=S)[nH]2)n1',\n       'CN(CC(F)(F)F)C(=O)CNc1nc(NCC(C)(O)CN2CCOCC2)nc(Nc2c(OC(F)(F)F)cccc2C(=O)N[Dy])n1',\n       'CN1CCN(Cc2cccc(Nc3nc(Nc4ccc(N5CCCC5=O)cc4)nc(NC4(C(=O)N[Dy])CC4)n3)c2)CC1',\n       'COc1cc2c(cc1CNc1nc(NC[C@@H]3C[C@H](F)CN3Cc3ccnn3C)nc(N[C@H](CC(=O)N[Dy])Cc3cccs3)n1)OCO2',\n       'COc1ccc([C@H](Nc2nc(NCCC(=O)NS(C)(=O)=O)nc(NCc3ccc(Oc4ccc(Cl)cc4Cl)c(C)c3)n2)C(=O)N[Dy])cc1',\n       'O=C1CN(CCCNc2nc(Nc3ccc(F)c(C(F)(F)F)c3)nc(N3CC[C@@H]3C(=O)N[Dy])n2)CCN1',\n       'COc1cnc(Nc2nc(NCc3nc(C)ccc3O)nc(N[C@H](Cc3ccc(I)cc3)C(=O)N[Dy])n2)nc1',\n       'NC(=O)C1=NOC(CNc2nc(NCc3c[nH]c4ccccc34)nc(N[C@@H](Cc3ccco3)C(=O)N[Dy])n2)C1',\n       'O=C(N[Dy])c1cccc(I)c1Nc1nc(NCc2cc(C(F)(F)F)co2)nc(Nc2cc(CO)ccn2)n1',\n       'COC(=O)c1nccnc1Nc1nc(NC[C@H]2CC[C@H](C(N)=O)CC2)nc(Nc2ccc(C#N)cc2C(=O)N[Dy])n1',\n       'COC(=O)c1cc(F)cc(Nc2nc(Nc3ccc(Cl)c(C#N)c3)nc(Nc3ccc(C(=O)N[Dy])cc3C(F)(F)F)n2)c1',\n       'O=C(N[Dy])c1c(Br)ccc(F)c1Nc1nc(NCC2Cc3ccccc3NC2=O)nc(Nc2nccnc2Br)n1',\n       'Cc1cc(Br)c(C(=O)N[Dy])cc1Nc1nc(NCc2cnc3n2CCOC3)nc(NCC(F)(F)CC2CC2)n1',\n       'COc1cccc(F)c1CNc1nc(NCCN2C(=O)SC(=Cc3cccs3)C2=O)nc(N[C@H](Cc2ccc([N+](=O)[O-])cc2)C(=O)N[Dy])n1',\n       'C#Cc1cccc(Nc2nc(Nc3ccc4c(cnn4C(=O)OC(C)(C)C)c3)nc(Nc3nc(-c4ccc(C(=O)N[Dy])cc4)cs3)n2)c1',\n       'Cc1cc(Nc2nc(NCc3ccon3)nc(N[C@@H](Cc3ccc(F)c(F)c3)C(=O)N[Dy])n2)ccc1O',\n       'CC(CNc1nc(NCCCc2nc3c(n2Cc2nccn2C)CCCC3)nc(Nc2c(C(=O)O)cccc2C(=O)N[Dy])n1)c1nccs1',\n       'COC(=O)c1ccc(Nc2nc(NC[C@@H]3C[C@H](F)CN3Cc3ccnn3C)nc(Nc3ccc(F)cc3C(=O)N[Dy])n2)cc1O',\n       'COC(=O)c1occc1Nc1nc(NC[C@@H]2CCC(=O)N2)nc(N[C@@H](Cc2ccsc2)C(=O)N[Dy])n1',\n       'Cc1ccc(F)c(Nc2nc(NCc3noc(C4CCOCC4)n3)nc(N[C@@H](CC(=O)N[Dy])Cc3ccc(C(F)(F)F)cc3)n2)c1',\n       'COCc1ccc(Nc2nc(Nc3nnc(C)o3)nc(Nc3c(C)cc(Cl)cc3C(=O)N[Dy])n2)cc1',\n       'O=C(N[Dy])c1ccc(Nc2nc(NCc3cccs3)nc(Nc3nccc(-c4cccnc4)n3)n2)c(Cl)c1',\n       'CC(C)(C)OCC(Nc1nc(NCC(=O)NC2CCC2)nc(Nc2nccc(Cl)n2)n1)C(=O)N[Dy]',\n       'COc1c(F)cccc1Nc1nc(Nc2ccc(O)c(C)c2)nc(Nc2cccc(Cl)c2C(=O)N[Dy])n1',\n       'C=CCOC(C)CNc1nc(NCC(C)Cn2cccn2)nc(Nc2c(I)cccc2C(=O)N[Dy])n1',\n       'CN(CCNc1nc(NCC(O)COc2cccc(Cl)c2Cl)nc(N[C@H]2CCC[C@@H]2C(=O)N[Dy])n1)C1CCOCC1',\n       'COc1cc(Nc2nc(NCCCc3cc(=O)[nH][nH]3)nc(NCc3ccnc(C(N)=O)c3)n2)c(C(=O)N[Dy])cc1OC',\n       'O=C(N[Dy])[C@H](Nc1nc(NCCn2cc(C3CC3)nn2)nc(NCc2ncon2)n1)C1CCCC1',\n       'COc1ccnc(Nc2nc(NCc3cc(Br)cc4cccnc34)nc(Nc3ccc(C(=O)N[Dy])c(Cl)c3)n2)n1',\n       'CCOC(=O)c1ccc(O)c(Nc2nc(Nc3cnc(Cl)c(Cl)c3)nc(Nc3ccc(Cl)nc3C(=O)N[Dy])n2)c1',\n       'O=C(N[Dy])c1ccc(Nc2nc(NCCOc3ccc(F)c(F)c3)nc(NCC3CCOC4(CCOCC4)C3)n2)cc1C(F)(F)F',\n       'COc1cccc(-c2cc(Nc3nc(NCCC4CN(c5ncnc6[nH]ncc56)c5ccccc54)nc(Nc4ccc(CC(=O)N[Dy])cc4)n3)on2)c1',\n       'COc1ccc(CNc2nc(Nc3ccc(N4CCOCC4)cc3)nc(N3Cc4ccccc4C[C@@H]3CC(=O)N[Dy])n2)c(C)c1OC',\n       'O=C(NCCNc1nc(Nc2ccc(O)cn2)nc(Nc2ccc(Cl)cc2C(=O)N[Dy])n1)c1cnccn1',\n       'COc1nc(C)ccc1CNc1nc(NCc2ccc(SC)o2)nc(Nc2cc(I)ccc2C(=O)N[Dy])n1',\n       'O=C(C[C@@H](Cc1ccc(F)cc1)Nc1nc(NCc2ccc[n+]([O-])c2)nc(Nc2nc(-c3cccc([N+](=O)[O-])c3)cs2)n1)N[Dy]',\n       'O=C(N[Dy])c1ccncc1Nc1nc(NCCc2c[nH]c3cc(Cl)ccc23)nc(Nc2nc3cccc(Br)n3n2)n1',\n       'Cc1cc(F)ccc1Nc1nc(Nc2cc(C(F)(F)F)cnc2Cl)nc(Nc2c(I)c(C(=O)O)c(I)c(C(=O)N[Dy])c2I)n1',\n       'O=C(N[Dy])c1ccc(Nc2nc(NCc3ccc4ccccc4n3)nc(NCC3CCN(CC(F)F)CC3)n2)cn1',\n       'C=CCOCCCNc1nc(NCC2CCCn3ccnc32)nc(N[C@@H](CCCN=[N+]=[N-])C(=O)N[Dy])n1',\n       'COc1cc(CNc2nc(NCC3(S(C)=O)CCC3)nc(Nc3ccc(Cl)cc3C(=O)N[Dy])n2)sn1',\n       'Cc1cc(Nc2nc(NCc3noc4ccc(F)cc34)nc(N[C@@H](CC(=O)N[Dy])c3ccc(C#N)cc3)n2)n[nH]1',\n       'COc1ccc2nnc(CNc3nc(NCC(C)N4CCC4)nc(Nc4c(Cl)cccc4C(=O)N[Dy])n3)n2n1',\n       'COC(=O)c1cnc(Nc2nc(NCCc3c(C)[nH][nH]c3=O)nc(Nc3ncc(C(=O)N[Dy])s3)n2)cn1',\n       'Cn1cc(Nc2nc(NCC(F)(F)C(F)(F)F)nc(Nc3cc(F)c(Br)cc3C(=O)N[Dy])n2)ccc1=O',\n       'Cn1nccc1[C@@H]1OCC[C@H]1CNc1nc(NCC2OCCc3ccsc32)nc(N[C@@H](C(=O)N[Dy])C2CCCCC2)n1',\n       'CN1CCN(Cc2ccccc2Nc2nc(NCc3cc4ccccc4[nH]c3=O)nc(N3CCN(C(=O)OC(C)(C)C)C3C(=O)N[Dy])n2)CC1',\n       'Cc1cc(F)cc(Nc2nc(NCc3ccncn3)nc(N[C@@H](CC(=O)N[Dy])Cc3ccc(F)cc3)n2)c1',\n       'Cc1cccc(Nc2nc(NCc3cc(=O)c(O)co3)nc(N[C@H](CC(=O)N[Dy])Cc3ccc(C(F)(F)F)cc3)n2)c1Cl',\n       'COc1cc(Nc2nc(NCC3(F)CCOC3)nc(Nc3ncnc4[nH]cnc34)n2)c(C(=O)N[Dy])c(OC)c1',\n       'N#Cc1cnc2c(C#N)cnn2c1Nc1nc(NCC2(CO)CCOC2)nc(Nc2ccc(C(=O)N[Dy])cc2O)n1',\n       'Cc1cc(Nc2nc(Nc3nc(-c4ccc(Cl)cc4)cs3)nc(Nc3cc(F)c(Br)cc3C(=O)N[Dy])n2)on1',\n       'Cc1cccc2oc(CCNc3nc(NC[C@@H]4C[C@H](F)CN4Cc4ccnn4C)nc(Nc4cc(Br)ccc4C(=O)N[Dy])n3)nc12',\n       'COc1cc(C(=O)N[Dy])c(Nc2nc(NCCC3CCCC3(F)F)nc(NCc3cncn3C)n2)cn1',\n       'Cc1cc(Nc2nc(Nc3n[nH]c4ccc([N+](=O)[O-])cc34)nc(N3CCC[C@@]3(C)C(=O)N[Dy])n2)nnc1Cl',\n       'CC(C)(C#N)c1ccc(Nc2nc(NCC3(CO)CCOC3)nc(N[C@@H](CC(=O)N[Dy])Cc3ccccc3Cl)n2)cc1',\n       'CCOC(=O)c1cnn(C)c1Nc1nc(Nc2cc(N3CCCC3)ccn2)nc(Nc2c(F)ccc(Br)c2C(=O)N[Dy])n1',\n       'COC(=O)c1cc(Nc2nc(Nc3ccc(O)c(C)c3)nc(Nc3ccc(C(=O)N[Dy])c([N+](=O)[O-])c3)n2)cs1',\n       'CC(C)c1nc(Nc2nc(Nc3cc(Cl)ncn3)nc(N[C@@H](CC(=O)N[Dy])c3cccc(Cl)c3Cl)n2)sc1Br',\n       'O=C(N[Dy])c1c(F)cccc1Nc1nc(Nc2ccc([N+](=O)[O-])c(C(F)(F)F)c2)nc(Nc2ncc(Cl)cc2F)n1',\n       'O=C(N[Dy])[C@@H]1C[C@@H]2CCCC[C@@H]2N1c1nc(NCc2ccncn2)nc(Nc2ccncc2[N+](=O)[O-])n1',\n       'C#CC[C@@](C)(Nc1nc(NCc2ccc(CN3C(=O)CNC3=O)cc2)nc(Nc2cc(Cl)nc(Cl)n2)n1)C(=O)N[Dy]',\n       'COC1(CNc2nc(NCC3(S(C)=O)CCC3)nc(Nc3cc(F)cc(F)c3C(=O)N[Dy])n2)CCOC1']\n\nmols = [Chem.MolFromSmiles(i) for i in smiles]","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:33.207716Z","iopub.execute_input":"2024-05-31T21:34:33.208214Z","iopub.status.idle":"2024-05-31T21:34:33.271073Z","shell.execute_reply.started":"2024-05-31T21:34:33.208180Z","shell.execute_reply":"2024-05-31T21:34:33.269952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fingerprint Datastructs\n\nLets start by creating a random fingerprint and inspecting it. We will use the `rdFingerprintGenerator` approach with an `AdditionalOutput` object to capture additional fingerprint details","metadata":{}},{"cell_type":"code","source":"def get_fp_with_ao(mol, radius=2, fpSize=2048):\n    fpg = rdFingerprintGenerator.GetMorganGenerator(radius=radius,fpSize=fpSize)\n    \n    ao = rdFingerprintGenerator.AdditionalOutput()\n    ao.AllocateAtomCounts()\n    ao.AllocateAtomToBits()\n    ao.AllocateBitInfoMap()\n    \n    fp = fpg.GetFingerprint(mol, additionalOutput=ao)\n    return fp, ao","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:33.564222Z","iopub.execute_input":"2024-05-31T21:34:33.564626Z","iopub.status.idle":"2024-05-31T21:34:33.571080Z","shell.execute_reply.started":"2024-05-31T21:34:33.564596Z","shell.execute_reply":"2024-05-31T21:34:33.570059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fp, ao = get_fp_with_ao(mols[0])","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:33.765966Z","iopub.execute_input":"2024-05-31T21:34:33.766365Z","iopub.status.idle":"2024-05-31T21:34:33.771750Z","shell.execute_reply.started":"2024-05-31T21:34:33.766335Z","shell.execute_reply":"2024-05-31T21:34:33.770546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The fingerprint is an RDKit object `rdkit.DataStructs.cDataStructs.ExplicitBitVect`.\n\nRDKit implements fingerprints efficiently as sparse vectors. Typically we want to keep fingerprints in this format. We will see examples later of why this matters for performance.","metadata":{}},{"cell_type":"code","source":"fp","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:34.092257Z","iopub.execute_input":"2024-05-31T21:34:34.093322Z","iopub.status.idle":"2024-05-31T21:34:34.101688Z","shell.execute_reply.started":"2024-05-31T21:34:34.093279Z","shell.execute_reply":"2024-05-31T21:34:34.100629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_bits = fp.GetNumBits()\non_bits = fp.GetNumOnBits()\n\non_bits_list = list(fp.GetOnBits())\n\nprint(f'Total bits: {total_bits}, On Bits: {on_bits}, Sparsity: {on_bits/total_bits}')","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:34.256484Z","iopub.execute_input":"2024-05-31T21:34:34.256877Z","iopub.status.idle":"2024-05-31T21:34:34.265160Z","shell.execute_reply.started":"2024-05-31T21:34:34.256847Z","shell.execute_reply":"2024-05-31T21:34:34.264055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The `AdditionalOutput` contains extra information about the fingerprint. The `GetBitInfoMap()` function gives us a dictionary mapping filled bits to atoms in the molecule.\n\nThe value tuples here represent `(atom_idx, radius)` for the fingerprint bit. So the pair:\n\n```\n47: ((26, 2),)\n```\n\nTells us that bit 47 relates to atom number 26 with a radius of 2 around the atom.","metadata":{}},{"cell_type":"code","source":"bit_info = ao.GetBitInfoMap()\nbit_info","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:34.599160Z","iopub.execute_input":"2024-05-31T21:34:34.599554Z","iopub.status.idle":"2024-05-31T21:34:34.619514Z","shell.execute_reply.started":"2024-05-31T21:34:34.599524Z","shell.execute_reply":"2024-05-31T21:34:34.618332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can also see that a specific bit might map to multiple features in the molecule. For example:\n\n```\n255: ((7, 1), (8, 2)),\n```\n\nThe bit 255 relates to two features - one at atom 7 with radius 1, and the other at atom 8 with radius 2.\n\nWe can use the `DrawMorganBits` function to visualize what is happening here. We see that bit 255 maps to two different (but in this case overlapping) substructures in the molecule. This is due to hash collisions, which we will talk about later.","metadata":{}},{"cell_type":"code","source":"bit_idx = 255\n\nChem.Draw.DrawMorganBits([(mols[0], bit_idx, bit_info, i) \n                           for i in range(len(bit_info[bit_idx]))])","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:34.924039Z","iopub.execute_input":"2024-05-31T21:34:34.924442Z","iopub.status.idle":"2024-05-31T21:34:34.965916Z","shell.execute_reply.started":"2024-05-31T21:34:34.924413Z","shell.execute_reply":"2024-05-31T21:34:34.964745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can also highlight fingerprint features on a molecule","metadata":{}},{"cell_type":"code","source":"def get_bit_highlights(mol, atom_idx, radius):\n    env = Chem.FindAtomEnvironmentOfRadiusN(mol, radius, atom_idx)\n    \n    atoms = set((atom_idx, ))\n    \n    for bond_idx in env:\n        bond = mol.GetBondWithIdx(bond_idx)\n        \n        start_atom = bond.GetBeginAtomIdx()\n        end_atom = bond.GetEndAtomIdx()\n        \n        atoms.update([start_atom, end_atom])\n        \n    bonds = set()\n    for atom_idx in atoms:\n        atom = mol.GetAtomWithIdx(atom_idx)\n        for bond in atom.GetBonds():\n            bonds.add(bond.GetIdx())\n            \n    return atoms, bonds","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:35.231168Z","iopub.execute_input":"2024-05-31T21:34:35.231638Z","iopub.status.idle":"2024-05-31T21:34:35.239583Z","shell.execute_reply.started":"2024-05-31T21:34:35.231604Z","shell.execute_reply":"2024-05-31T21:34:35.238431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For a single fingerprint feature:","metadata":{}},{"cell_type":"code","source":"bit_atoms, bit_bonds = get_bit_highlights(mols[0], 7, 1)\nChem.Draw.MolToImage(mols[0], \n                     highlightAtoms=list(bit_atoms), \n                     highlightBonds=list(bit_bonds))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:35.522967Z","iopub.execute_input":"2024-05-31T21:34:35.523380Z","iopub.status.idle":"2024-05-31T21:34:35.552369Z","shell.execute_reply.started":"2024-05-31T21:34:35.523347Z","shell.execute_reply":"2024-05-31T21:34:35.551399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can also visualize multiple features. In this case, we visualize the `(7,1)` and `(8,2)` features corresponding to bit 255","metadata":{}},{"cell_type":"code","source":"def plot_bit_features(mol, bit_info, bit_idx, **draw_kwargs):\n    bit_tuples = bit_info[bit_idx]\n\n    plot_mols = []\n    atom_highlights = []\n    bond_highlights = []\n\n    for (atom_idx, radius) in bit_tuples:\n        bit_atoms, bit_bonds = get_bit_highlights(mol, atom_idx, radius)\n        atom_highlights.append(list(bit_atoms))\n        bond_highlights.append(list(bit_bonds))\n        plot_mols.append(mol)\n\n    img = Chem.Draw.MolsToGridImage(plot_mols, \n                              highlightAtomLists=atom_highlights, \n                              highlightBondLists=bond_highlights,\n                              **draw_kwargs\n                             )\n    return img","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:35.806746Z","iopub.execute_input":"2024-05-31T21:34:35.807144Z","iopub.status.idle":"2024-05-31T21:34:35.814240Z","shell.execute_reply.started":"2024-05-31T21:34:35.807110Z","shell.execute_reply":"2024-05-31T21:34:35.812941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_bit_features(mols[0], bit_info, 255, molsPerRow=2, subImgSize=(300,300))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:35.963421Z","iopub.execute_input":"2024-05-31T21:34:35.963810Z","iopub.status.idle":"2024-05-31T21:34:35.992402Z","shell.execute_reply.started":"2024-05-31T21:34:35.963780Z","shell.execute_reply":"2024-05-31T21:34:35.991306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For another example, looking at the atoms related to bit 1855:","metadata":{}},{"cell_type":"code","source":"plot_bit_features(mols[0], bit_info, 1855, molsPerRow=2, subImgSize=(300,300))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:36.243726Z","iopub.execute_input":"2024-05-31T21:34:36.244113Z","iopub.status.idle":"2024-05-31T21:34:36.284909Z","shell.execute_reply.started":"2024-05-31T21:34:36.244085Z","shell.execute_reply":"2024-05-31T21:34:36.283672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Morgan Fingerprint Radius\n\nMorgan fingerprints and other forms of circular fingerprint look at atoms in a radius around a central atom. For each atom in the molecule, the Morgan algorithm looks at that atom's neighbors from 1-hop to n-hop where n is the radius parameter passed to the fingerprint generator.\n\nSince the algorithm looks at neighbors from 1-n hops, larger radius fingerprints are guaranteed to include features from smaller radius fingerprints","metadata":{}},{"cell_type":"code","source":"fpg2 = rdFingerprintGenerator.GetMorganGenerator(radius=2,fpSize=2048)\nfpg3 = rdFingerprintGenerator.GetMorganGenerator(radius=3,fpSize=2048)\nfpg4 = rdFingerprintGenerator.GetMorganGenerator(radius=4,fpSize=2048)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:36.539175Z","iopub.execute_input":"2024-05-31T21:34:36.539565Z","iopub.status.idle":"2024-05-31T21:34:36.545964Z","shell.execute_reply.started":"2024-05-31T21:34:36.539536Z","shell.execute_reply":"2024-05-31T21:34:36.544958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fp2 = fpg2.GetFingerprint(mols[0])\nfp3 = fpg3.GetFingerprint(mols[0])\nfp4 = fpg4.GetFingerprint(mols[0])","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:36.690314Z","iopub.execute_input":"2024-05-31T21:34:36.690712Z","iopub.status.idle":"2024-05-31T21:34:36.697309Z","shell.execute_reply.started":"2024-05-31T21:34:36.690674Z","shell.execute_reply":"2024-05-31T21:34:36.696151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here we see that the `radius=2` fingerprint has 73 \"on\" bits. When we look at the intersection of `fp2 & fp3` and `fp2 * fp4`, we see the intersections all have 73 \"on\" bits, meaning all 73 bits from the `radius=2` fingerprint are present in the `radius=3` and `radius=4` fingerprints at the exact same bit index","metadata":{}},{"cell_type":"code","source":"fp2.GetNumOnBits(), (fp2 & fp3).GetNumOnBits(), (fp2 & fp4).GetNumOnBits()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:37.000420Z","iopub.execute_input":"2024-05-31T21:34:37.000821Z","iopub.status.idle":"2024-05-31T21:34:37.008259Z","shell.execute_reply.started":"2024-05-31T21:34:37.000788Z","shell.execute_reply":"2024-05-31T21:34:37.007166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see the same for the `radius=3` fingerprint. The fingerprint has 104 \"on\" bits. The intersection `fp2 & fp3` only has 73 \"on\" bits, showing that the `radius=3` fingerprint has bits that are not present in the `radius=2` fingerprint. We also see that `fp3 & fp4` has the same number of \"on\" bits as `fp3`, showing that the `radius=4` fingerprint fully includes the `radius=3` fingerprint","metadata":{}},{"cell_type":"code","source":"fp3.GetNumOnBits(), (fp2 & fp3).GetNumOnBits(), (fp3 & fp4).GetNumOnBits()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:37.319726Z","iopub.execute_input":"2024-05-31T21:34:37.320100Z","iopub.status.idle":"2024-05-31T21:34:37.327417Z","shell.execute_reply.started":"2024-05-31T21:34:37.320073Z","shell.execute_reply":"2024-05-31T21:34:37.326300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Bit Collisions\n\nLets talk a bit more about bit collisions. Fingerprint functions are hash functions that map molecular substructures to a fixed length vector. Given the diversity of chemical features, it is inevitable that hash collisions will occurr. \n\nTo see a more extreme example, lets reduce our fingerprint size from 2048 to 32. This will increase hash collisions of disparate features.","metadata":{}},{"cell_type":"code","source":"fp, ao = get_fp_with_ao(mols[0], fpSize=32)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:37.640178Z","iopub.execute_input":"2024-05-31T21:34:37.641159Z","iopub.status.idle":"2024-05-31T21:34:37.646642Z","shell.execute_reply.started":"2024-05-31T21:34:37.641124Z","shell.execute_reply":"2024-05-31T21:34:37.645498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Feature 11 is generated by several very different substructures","metadata":{}},{"cell_type":"code","source":"bit_idx = 11\nbit_info = ao.GetBitInfoMap()\n\nChem.Draw.DrawMorganBits([(mols[0], bit_idx, bit_info, i) \n                           for i in range(len(bit_info[bit_idx]))])","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:38.285321Z","iopub.execute_input":"2024-05-31T21:34:38.286157Z","iopub.status.idle":"2024-05-31T21:34:38.344588Z","shell.execute_reply.started":"2024-05-31T21:34:38.286128Z","shell.execute_reply":"2024-05-31T21:34:38.343555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_bit_features(mols[0], bit_info, 11, molsPerRow=3, subImgSize=(300,300))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:38.518566Z","iopub.execute_input":"2024-05-31T21:34:38.518946Z","iopub.status.idle":"2024-05-31T21:34:38.594303Z","shell.execute_reply.started":"2024-05-31T21:34:38.518914Z","shell.execute_reply":"2024-05-31T21:34:38.593107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The impact of hash collisions can be reduced by using larger fingerprints. We will discuss later ways to do this efficiently. ","metadata":{}},{"cell_type":"markdown","source":"## Fingerprint Types\n\nRDKit provides several fingerprint algorithms and fingerprint representations\n\nFingerprint Algorithms:\n\n* Morgan fingerprints - circular fingerprints similar to ECFP/FCFP fingerprints\n    * FCFP variants use pre-defined SMARTs patterns to add features for donor/acceptor/acidic/basic/halogen/aromatic. Pass `atomInvariantsGenerator=rdFingerprintGenerator.GetMorganFeatureAtomInvGen()` to `GetMorganGenerator` to include these features\n* RDKit fingerprints - linear fingerprints \n\nFor brevity, I'll skip the algorithmic differences. The big picture is Morgan fingerprints look at an atom and all its n-hop neighbors, while rdkit fingerprints only look at linear paths within the molecule.\n\nFingerprint Representations:\n\n* Standard fingerprint: fixed length vector\n* Sparse fingerprint: \"unfolded\" fingerprints of arbitrary length\n* Count fingerprint: fixed length vector with int vals (ie not binary) denoting the frequency or count of a given feature\n* Sparse count fingerprint: \"unfolded\" version of count fingerprint\n\nFor ML purposes, we typically want the fixed length versions for feature consistency.\n\nWe can look at the relative sizes of different fingerprints:","metadata":{}},{"cell_type":"code","source":"mol = Chem.MolFromSmiles(smiles[0])\nfpg = rdFingerprintGenerator.GetMorganGenerator(radius=2,fpSize=2048)\n\ndef get_fp_with_ao(mol, fp_func):\n    ao = rdFingerprintGenerator.AdditionalOutput()\n    ao.AllocateAtomCounts()\n    ao.AllocateAtomToBits()\n    ao.AllocateBitInfoMap()\n    \n    fp = fp_func(mol, additionalOutput=ao)\n    return fp, ao\n\n# bit vectors:\nfp, ao_fp = get_fp_with_ao(mol, fpg.GetFingerprint)\n\nsfp, ao_sfp = get_fp_with_ao(mol, fpg.GetSparseFingerprint)\n\n# count vectors:\ncfp, ao_cfp = get_fp_with_ao(mol, fpg.GetCountFingerprint)\n\nscfp, ao_scfp = get_fp_with_ao(mol, fpg.GetSparseCountFingerprint)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:39.023969Z","iopub.execute_input":"2024-05-31T21:34:39.024349Z","iopub.status.idle":"2024-05-31T21:34:39.034402Z","shell.execute_reply.started":"2024-05-31T21:34:39.024321Z","shell.execute_reply":"2024-05-31T21:34:39.033322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'fp: {type(fp)} {len(fp)}, {fp.GetNumOnBits()}')\nprint(f'sfp: {type(sfp)} {len(sfp)}, {sfp.GetNumOnBits()}')\nprint(f'cfp: {type(cfp)} {cfp.GetLength()}, {len(cfp.GetNonzeroElements())}')\nprint(f'scfp: {type(scfp)} {scfp.GetLength()}, {len(scfp.GetNonzeroElements())}')","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:39.196126Z","iopub.execute_input":"2024-05-31T21:34:39.196984Z","iopub.status.idle":"2024-05-31T21:34:39.203514Z","shell.execute_reply.started":"2024-05-31T21:34:39.196947Z","shell.execute_reply":"2024-05-31T21:34:39.202433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mol = Chem.MolFromSmiles(smiles[0])\nfpg = rdFingerprintGenerator.GetRDKitFPGenerator(minPath=1, maxPath=7, fpSize=2048)\n\ndef get_fp_with_ao(mol, fp_func):\n    ao = rdFingerprintGenerator.AdditionalOutput()\n    ao.AllocateAtomCounts()\n    ao.AllocateAtomToBits()\n    ao.AllocateBitInfoMap()\n    \n    fp = fp_func(mol, additionalOutput=ao)\n    return fp, ao\n\n# bit vectors:\nfp, ao_fp = get_fp_with_ao(mol, fpg.GetFingerprint)\n\nsfp, ao_sfp = get_fp_with_ao(mol, fpg.GetSparseFingerprint)\n\n# count vectors:\ncfp, ao_cfp = get_fp_with_ao(mol, fpg.GetCountFingerprint)\n\nscfp, ao_scfp = get_fp_with_ao(mol, fpg.GetSparseCountFingerprint)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:39.355497Z","iopub.execute_input":"2024-05-31T21:34:39.356127Z","iopub.status.idle":"2024-05-31T21:34:39.387849Z","shell.execute_reply.started":"2024-05-31T21:34:39.356094Z","shell.execute_reply":"2024-05-31T21:34:39.386465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'fp: {type(fp)} {len(fp)}, {fp.GetNumOnBits()}')\nprint(f'sfp: {type(sfp)} {len(sfp)}, {sfp.GetNumOnBits()}')\nprint(f'cfp: {type(cfp)} {cfp.GetLength()}, {len(cfp.GetNonzeroElements())}')\nprint(f'scfp: {type(scfp)} {scfp.GetLength()}, {len(scfp.GetNonzeroElements())}')","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:39.519910Z","iopub.execute_input":"2024-05-31T21:34:39.520300Z","iopub.status.idle":"2024-05-31T21:34:39.527631Z","shell.execute_reply.started":"2024-05-31T21:34:39.520272Z","shell.execute_reply":"2024-05-31T21:34:39.526440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fingerprint Sparsity\n\nWe can see the Morgan fingerprints and RDKit fingerprints have different degrees of sparsity. We can plot how this evolves with fingerprint size.","metadata":{}},{"cell_type":"code","source":"fp_generators = [\n    partial(rdFingerprintGenerator.GetMorganGenerator, radius=2),\n    partial(rdFingerprintGenerator.GetMorganGenerator, radius=3),\n    partial(rdFingerprintGenerator.GetRDKitFPGenerator, minPath=1, maxPath=7)\n]\n\nfp_names = ['morgan_radius_2', 'morgan_radius_3', 'rdkit']\n\nfp_sizes = [128*(2**i) for i in range(15)]\n\nn_threads = 4\n\noutputs = []","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:39.851574Z","iopub.execute_input":"2024-05-31T21:34:39.852443Z","iopub.status.idle":"2024-05-31T21:34:39.858654Z","shell.execute_reply.started":"2024-05-31T21:34:39.852411Z","shell.execute_reply":"2024-05-31T21:34:39.857611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note that here we use \n\n```\nfps = fpg.GetFingerprints(mols, numThreads=n_threads)\n```\n\nThis generates fingerprints in parallel using multiple threads. This is the best way to generate bulk fingerprints efficiently ","metadata":{}},{"cell_type":"code","source":"for i in range(len(fp_generators)):\n    gen_func = fp_generators[i]\n    \n    x_vals = []\n    y_vals = [[], []]\n    \n    for size in fp_sizes:\n        fpg = gen_func(fpSize=size)\n        fps = fpg.GetFingerprints(mols, numThreads=n_threads)\n        \n        bit_counts = np.array([i.GetNumOnBits() for i in fps]).mean()\n        sparsity = bit_counts/size\n        \n        x_vals.append(size)\n        y_vals[0].append(bit_counts)\n        y_vals[1].append(sparsity)\n        \n    outputs.append((x_vals, y_vals))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:40.199292Z","iopub.execute_input":"2024-05-31T21:34:40.200317Z","iopub.status.idle":"2024-05-31T21:34:42.384932Z","shell.execute_reply.started":"2024-05-31T21:34:40.200273Z","shell.execute_reply":"2024-05-31T21:34:42.383974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(12,4))\nlabels = ['Mean Bit Count', 'Sparsity']\n\nfor i in range(len(fp_names)):\n    name = fp_names[i]\n    x_vals, (bit_counts, sparsity) = outputs[i]\n    \n    axes[0].plot(x_vals, bit_counts, label=name)\n    axes[1].plot(x_vals, sparsity, label=name)\n    \nfor i, ax in enumerate(axes.flat):\n    ax.set_xscale('log')\n    ax.set_xlabel('Fingerprint Size (log scale)')\n    ax.set_ylabel(labels[i])\n    ax.legend()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:42.386863Z","iopub.execute_input":"2024-05-31T21:34:42.387218Z","iopub.status.idle":"2024-05-31T21:34:43.785365Z","shell.execute_reply.started":"2024-05-31T21:34:42.387190Z","shell.execute_reply":"2024-05-31T21:34:43.784312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fingerprint Conversion\n\nFor ML purposes, we typically need our fingerprints to be a numpy array, pytorch tensor or similar.\n\nThis conversion can actually cause a significant compute and memory overhead. For this reason, RDKit provides helper functions to make this process more efficient. \n\nBelow we compare two methods for casting fingerprints to numpy arrays - one using standard numpy and the other using RDKit's interal functions","metadata":{}},{"cell_type":"code","source":"fpg = rdFingerprintGenerator.GetMorganGenerator(radius=2, fpSize=2048)\nfps = fpg.GetFingerprints(mols, numThreads=4)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:43.786609Z","iopub.execute_input":"2024-05-31T21:34:43.786960Z","iopub.status.idle":"2024-05-31T21:34:43.804787Z","shell.execute_reply.started":"2024-05-31T21:34:43.786933Z","shell.execute_reply":"2024-05-31T21:34:43.803862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fp_to_np1(fp):\n    return np.array(fp, dtype=np.int8)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:43.807124Z","iopub.execute_input":"2024-05-31T21:34:43.807433Z","iopub.status.idle":"2024-05-31T21:34:43.812355Z","shell.execute_reply.started":"2024-05-31T21:34:43.807408Z","shell.execute_reply":"2024-05-31T21:34:43.811264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfps_np = [fp_to_np1(i) for i in fps]","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:43.813694Z","iopub.execute_input":"2024-05-31T21:34:43.814616Z","iopub.status.idle":"2024-05-31T21:34:44.004868Z","shell.execute_reply.started":"2024-05-31T21:34:43.814589Z","shell.execute_reply":"2024-05-31T21:34:44.003668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fp_to_np2(fp):\n    arr = np.zeros((1,), dtype=np.int8)\n    DataStructs.ConvertToNumpyArray(fp, arr)\n    return arr","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:44.006134Z","iopub.execute_input":"2024-05-31T21:34:44.006455Z","iopub.status.idle":"2024-05-31T21:34:44.013044Z","shell.execute_reply.started":"2024-05-31T21:34:44.006428Z","shell.execute_reply":"2024-05-31T21:34:44.011948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfps_np = [fp_to_np2(i) for i in fps]","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:44.014488Z","iopub.execute_input":"2024-05-31T21:34:44.014810Z","iopub.status.idle":"2024-05-31T21:34:44.039587Z","shell.execute_reply.started":"2024-05-31T21:34:44.014781Z","shell.execute_reply":"2024-05-31T21:34:44.038446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see the RDKit version is ~10x faster than pure numpy.\n\nAlso note that in both cases we specify the output numpy array will be of type `int8`. If you don't specify this explicitly, numpy will cast the fingerprint to `int64`, which uses substantially more memory.","metadata":{}},{"cell_type":"markdown","source":"## Bit Packing\n\nOne technique we can use to reduce fingerprint size is packing bits into a `uint8` array","metadata":{}},{"cell_type":"code","source":"fp_packed = np.packbits(fps_np[0])\nfp_packed.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:44.041161Z","iopub.execute_input":"2024-05-31T21:34:44.041603Z","iopub.status.idle":"2024-05-31T21:34:44.051018Z","shell.execute_reply.started":"2024-05-31T21:34:44.041567Z","shell.execute_reply":"2024-05-31T21:34:44.049953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here we take our sparse binary fingerprint of size 2048 and pack it into a slightly less sparse uint8 array of size 256","metadata":{}},{"cell_type":"code","source":"print(fps_np[0])\nprint(fp_packed)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:44.591609Z","iopub.execute_input":"2024-05-31T21:34:44.592432Z","iopub.status.idle":"2024-05-31T21:34:44.599300Z","shell.execute_reply.started":"2024-05-31T21:34:44.592397Z","shell.execute_reply":"2024-05-31T21:34:44.598194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fingerprint Similarity\n\nWe often want to compare fingerprints to each other using some similarity/distance metric.\n\nSimilar to the numpy conversion, RDKit provides helper functions to compute these metrics directly on RDKit fingerprint vectors that are much more efficient compared to other methods\n\nTo start, lets benchmark tanimoto similarity using scipy:","metadata":{}},{"cell_type":"code","source":"fps_np = [fp_to_np2(i) for i in fps]","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:45.399186Z","iopub.execute_input":"2024-05-31T21:34:45.400239Z","iopub.status.idle":"2024-05-31T21:34:45.422398Z","shell.execute_reply.started":"2024-05-31T21:34:45.400206Z","shell.execute_reply":"2024-05-31T21:34:45.420904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntanimotos_scipy = 1-distance.cdist(fps_np, fps_np, metric='jaccard')","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:45.904702Z","iopub.execute_input":"2024-05-31T21:34:45.905416Z","iopub.status.idle":"2024-05-31T21:34:45.976859Z","shell.execute_reply.started":"2024-05-31T21:34:45.905381Z","shell.execute_reply":"2024-05-31T21:34:45.975799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now compare to the built-in RDKit function:","metadata":{}},{"cell_type":"code","source":"%%time\ntanimotos_rdkit = np.array([DataStructs.BulkTanimotoSimilarity(i, fps) for i in fps])","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:46.728991Z","iopub.execute_input":"2024-05-31T21:34:46.729907Z","iopub.status.idle":"2024-05-31T21:34:46.740095Z","shell.execute_reply.started":"2024-05-31T21:34:46.729862Z","shell.execute_reply":"2024-05-31T21:34:46.738942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Ignoring the time taken to convert the RDKit fingerprints to numpy arrays, the calculation of tanimoto similarity is ~15x faster","metadata":{}},{"cell_type":"markdown","source":"It's also worth noting that Tanimoto (and many other metrics) can be computed on fingerprints with bitwise operations.\n\nFor example, using RDKit fingerprints:","metadata":{}},{"cell_type":"code","source":"# tanimoto similarity between fp[0] and fp[1]\n(fps[0] & fps[1]).GetNumOnBits() / (fps[0] | fps[1]).GetNumOnBits()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:47.795733Z","iopub.execute_input":"2024-05-31T21:34:47.796410Z","iopub.status.idle":"2024-05-31T21:34:47.803814Z","shell.execute_reply.started":"2024-05-31T21:34:47.796376Z","shell.execute_reply":"2024-05-31T21:34:47.802475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can also do this in batch with numpy (although it is still slower compared to RDKit methods)","metadata":{}},{"cell_type":"code","source":"def tanimoto_bitwise(fps):\n    intersection = (fps_np[:,None] & fps_np[None]).sum(-1)\n    union = (fps_np[:,None] | fps_np[None]).sum(-1)\n    \n    return intersection / union","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:48.491134Z","iopub.execute_input":"2024-05-31T21:34:48.491486Z","iopub.status.idle":"2024-05-31T21:34:48.497497Z","shell.execute_reply.started":"2024-05-31T21:34:48.491461Z","shell.execute_reply":"2024-05-31T21:34:48.496430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pack into single array\nfps_np = np.array(fps_np)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:48.951175Z","iopub.execute_input":"2024-05-31T21:34:48.951573Z","iopub.status.idle":"2024-05-31T21:34:48.957179Z","shell.execute_reply.started":"2024-05-31T21:34:48.951541Z","shell.execute_reply":"2024-05-31T21:34:48.956120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntanimotos3 = tanimoto_bitwise(fps)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:49.384936Z","iopub.execute_input":"2024-05-31T21:34:49.385317Z","iopub.status.idle":"2024-05-31T21:34:49.435022Z","shell.execute_reply.started":"2024-05-31T21:34:49.385290Z","shell.execute_reply":"2024-05-31T21:34:49.433913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Substructure Search\n\nWe can use fingerprints to determine if one molecule is a substructure of another. If fingerprint A contains all the bits of fingerprint B, we can conclude that molecule B is a substructure of molecule A.\n\nNote that we can only use this using RDKit linear fingerprints, Morgan fingerprints do not work for this. \n\nMorgan fingerprints look at atoms in a n-hop radius around a given atom, so the bits created are influenced by features outside the substructure.\n\nRDKit linear fingerprints do not have this issue, so they can be used for substructure searching.","metadata":{}},{"cell_type":"code","source":"mols[0]","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:50.147183Z","iopub.execute_input":"2024-05-31T21:34:50.147563Z","iopub.status.idle":"2024-05-31T21:34:50.162917Z","shell.execute_reply.started":"2024-05-31T21:34:50.147534Z","shell.execute_reply":"2024-05-31T21:34:50.161760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"substructure = Chem.MolFromSmiles('c1cc(CNc2ncncn2)ccn1')\nsubstructure","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:50.580017Z","iopub.execute_input":"2024-05-31T21:34:50.580951Z","iopub.status.idle":"2024-05-31T21:34:50.595353Z","shell.execute_reply.started":"2024-05-31T21:34:50.580907Z","shell.execute_reply":"2024-05-31T21:34:50.594283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fpg = rdFingerprintGenerator.GetRDKitFPGenerator()\nfp = fpg.GetFingerprint(mol)\nfp_sub = fpg.GetFingerprint(substructure)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:50.863577Z","iopub.execute_input":"2024-05-31T21:34:50.864070Z","iopub.status.idle":"2024-05-31T21:34:50.874345Z","shell.execute_reply.started":"2024-05-31T21:34:50.864023Z","shell.execute_reply":"2024-05-31T21:34:50.873319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This function returns a float value between 0 and 1 denoting the percent of fingerprint bits in the substructure contained in the superstructure. When the substructure query is a perfect substructure of the superstructure, this function returns 1","metadata":{}},{"cell_type":"code","source":"def is_substructure(substructure_fp, superstructure_fp):\n    shared_bits = (substructure_fp & superstructure_fp).GetNumOnBits()\n    return shared_bits / substructure_fp.GetNumOnBits()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:51.314666Z","iopub.execute_input":"2024-05-31T21:34:51.315071Z","iopub.status.idle":"2024-05-31T21:34:51.320585Z","shell.execute_reply.started":"2024-05-31T21:34:51.315040Z","shell.execute_reply":"2024-05-31T21:34:51.319348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"is_substructure(fp_sub, fp)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:51.496061Z","iopub.execute_input":"2024-05-31T21:34:51.496467Z","iopub.status.idle":"2024-05-31T21:34:51.503946Z","shell.execute_reply.started":"2024-05-31T21:34:51.496436Z","shell.execute_reply":"2024-05-31T21:34:51.502647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can also use this for fuzzy searching. Here we replace one atom in the substructure to see how it impacts the output value","metadata":{}},{"cell_type":"code","source":"substructure = Chem.MolFromSmiles('c1cc(CCc2ncncn2)ccn1')\nfp_sub = fpg.GetFingerprint(substructure)\nsubstructure","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:51.931794Z","iopub.execute_input":"2024-05-31T21:34:51.932502Z","iopub.status.idle":"2024-05-31T21:34:51.947285Z","shell.execute_reply.started":"2024-05-31T21:34:51.932469Z","shell.execute_reply":"2024-05-31T21:34:51.946116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"is_substructure(fp_sub, fp)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:52.277276Z","iopub.execute_input":"2024-05-31T21:34:52.277644Z","iopub.status.idle":"2024-05-31T21:34:52.424758Z","shell.execute_reply.started":"2024-05-31T21:34:52.277618Z","shell.execute_reply":"2024-05-31T21:34:52.423556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## \"Sequence of Ints\" Featurization\n\nFingerprints tend to be sparse, especially Morgan fingerprints and large fingerprints. It can be useful to represent fingerprints as a sequence of int values representing \"on\" bits instead of using the full sparse vector","metadata":{}},{"cell_type":"code","source":"fpg = rdFingerprintGenerator.GetMorganGenerator(radius=2, fpSize=2048)\nfps = fpg.GetFingerprints(mols, numThreads=4)\n\nfp_tensor = torch.tensor(fps[0]).float()\nfp_ints = torch.tensor(fps[0].GetOnBits()).long()\nfp_ints","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:52.943248Z","iopub.execute_input":"2024-05-31T21:34:52.943617Z","iopub.status.idle":"2024-05-31T21:34:53.006125Z","shell.execute_reply.started":"2024-05-31T21:34:52.943590Z","shell.execute_reply":"2024-05-31T21:34:53.005076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"If you are familiar with NLP, this may look familiar. Modern NLP approaches process text by tokenizing, converting tokens to int values, and representing sequences as variable length sequences of ints.\n\nBy converting a fingerprint to a sequence of ints, we can plug fingerprint values into various NLP pipelines. We could, for example, use FP-ints to train a transformer model.\n\nWe can also use this representation to deal with large fingerprints efficiently.\n\nA common modeling approach is to train a MLP on molecular fingerprints. This usually involves a matmul operation between a fingerprint vector and a weight matrix. This is computationally inefficient, especially when using large fingerprints. Multiplying a weight matrix with a binary vector is equivalent to doing an embedding lookup with the \"on\" bits and summing the embedding vectors.\n\nTo show this, we will use three different methods to compute the same value:\n\n* Linear layer\n* Embedding layer with sum\n* Embedding bag layer","metadata":{}},{"cell_type":"code","source":"layer = nn.Linear(2048, 256, bias=False)\n\n# assign same weights to embedding\nembedding = nn.Embedding(2048, 256)\nembedding.weight.data = layer.weight.data.T\n\n# assign same weights to embedding bag\nembedding_bag = nn.EmbeddingBag(2048, 256, mode='sum')\nembedding_bag.weight.data = layer.weight.data.T","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:53.787141Z","iopub.execute_input":"2024-05-31T21:34:53.787512Z","iopub.status.idle":"2024-05-31T21:34:53.819079Z","shell.execute_reply.started":"2024-05-31T21:34:53.787483Z","shell.execute_reply":"2024-05-31T21:34:53.818150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sparse vector matmul with weight matrix\nout1 = layer(fp_tensor)\n\n# embedding lookup with sum\nout2 = embedding(fp_ints).sum(0)\n\n# embedding bag lookup with sum aggregation\nout3 = embedding_bag(fp_ints, torch.tensor([0,], dtype=torch.long))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:54.111106Z","iopub.execute_input":"2024-05-31T21:34:54.111782Z","iopub.status.idle":"2024-05-31T21:34:54.182958Z","shell.execute_reply.started":"2024-05-31T21:34:54.111753Z","shell.execute_reply":"2024-05-31T21:34:54.182076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can verify the results are the same. Due to numeric issues, we need to set `atol=1e-7`. We can achieve tighter similarity if we use higher precision tensors.\n\nThe point is we can take an expensive sparse vector x weight matrix matmul and replace it with a cheap embedding bag lookup","metadata":{}},{"cell_type":"code","source":"assert torch.allclose(out1, out3, atol=1e-7)\nassert torch.allclose(out1, out2, atol=1e-7)\nassert torch.allclose(out2, out3, atol=1e-7)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:54.852033Z","iopub.execute_input":"2024-05-31T21:34:54.852837Z","iopub.status.idle":"2024-05-31T21:34:54.872980Z","shell.execute_reply.started":"2024-05-31T21:34:54.852804Z","shell.execute_reply":"2024-05-31T21:34:54.872117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Using the embedding bag approach becomes more advantageous as we use larger fingerprints. Working with \"on\" bits in the \"sequence of ints\" format with an embedding bag allows us to efficiently use very large fingerprints, which in turn allows us to reduce hash collisions.\n\nTo illustrate, lets profile the linear layer matmul approach and embedding bag approach on GPU (we omit the embedding approach because usually you use a standard embedding when you _don't_ want to aggregate embeddings, whereas the embedding bag approach gives the same output as the linear layer)","metadata":{}},{"cell_type":"code","source":"def profile_function(function, record_name, *args, **kwargs):\n    \n    torch.cuda.empty_cache()\n    torch.cuda.reset_peak_memory_stats()\n    starting_mem = torch.cuda.max_memory_allocated()\n\n    with profile(activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA], \n                record_shapes=True, \n                 profile_memory=True, \n                 with_flops=True, \n                 with_stack=True) as prof:\n        with record_function(record_name):\n            function(*args, **kwargs)\n\n    mem_usage = torch.cuda.max_memory_allocated() - starting_mem\n    \n    return prof, mem_usage","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:55.742764Z","iopub.execute_input":"2024-05-31T21:34:55.743722Z","iopub.status.idle":"2024-05-31T21:34:55.750936Z","shell.execute_reply.started":"2024-05-31T21:34:55.743672Z","shell.execute_reply":"2024-05-31T21:34:55.749671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test_linear_layer(fps, d_in, d_out):\n    layer = nn.Linear(d_in, d_out, bias=False)\n    layer.cuda()\n    \n    batch = torch.stack([torch.from_numpy(fp_to_np2(i)) for i in fps]).float().cuda()\n    \n    layer(batch)\n    \ndef test_embedding_bag(fps, n_bits, d_embedding):\n    \n    embedding_bag = nn.EmbeddingBag(n_bits, d_embedding, mode='sum')\n    embedding_bag.cuda()\n    \n    batch = [torch.tensor(i.GetOnBits()).long() for i in fps]\n    lengths = torch.tensor([0] + [i.shape[0] for i in batch[:-1]])\n\n    offsets = torch.cumsum(lengths, 0).cuda()\n    batch = torch.cat(batch).cuda()\n    \n    embedding_bag(batch, offsets)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:56.174236Z","iopub.execute_input":"2024-05-31T21:34:56.175243Z","iopub.status.idle":"2024-05-31T21:34:56.184104Z","shell.execute_reply.started":"2024-05-31T21:34:56.175207Z","shell.execute_reply":"2024-05-31T21:34:56.182815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fp_size = 40960\nembedding_size = 256\n\nfpg = rdFingerprintGenerator.GetMorganGenerator(radius=2, fpSize=fp_size)\nfps = fpg.GetFingerprints(mols*10, numThreads=4) # use 1000 inputs for scale","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:56.734971Z","iopub.execute_input":"2024-05-31T21:34:56.735374Z","iopub.status.idle":"2024-05-31T21:34:56.827040Z","shell.execute_reply.started":"2024-05-31T21:34:56.735339Z","shell.execute_reply":"2024-05-31T21:34:56.826057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"linear_out = profile_function(\n    test_linear_layer, \n    'linear_layer', \n    fps, \n    fp_size, \n    embedding_size)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:34:58.355961Z","iopub.execute_input":"2024-05-31T21:34:58.356779Z","iopub.status.idle":"2024-05-31T21:35:04.930118Z","shell.execute_reply.started":"2024-05-31T21:34:58.356746Z","shell.execute_reply":"2024-05-31T21:35:04.928960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time.sleep(5) # cuda bugs out if you do two profiles back to back\nembedding_bag_out = profile_function(\n    test_embedding_bag, \n    'embedding_bag', \n    fps, \n    fp_size, \n    embedding_size)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:35:08.005850Z","iopub.execute_input":"2024-05-31T21:35:08.006959Z","iopub.status.idle":"2024-05-31T21:35:09.729812Z","shell.execute_reply.started":"2024-05-31T21:35:08.006923Z","shell.execute_reply":"2024-05-31T21:35:09.728676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 3, figsize=(19, 4))\n\nlabels = ['Linear Layer', 'Embedding Bag']\nmem_usages = [linear_out[-1], embedding_bag_out[-1]]\nprofs = [linear_out[0], embedding_bag_out[0]]\ntas = [i.key_averages().total_average() for i in profs]\n\naxes[0].bar(labels, mem_usages)\naxes[0].set_ylabel('Memory Usage')\n\naxes[1].bar(labels, [i.self_cpu_time_total for i in tas])\naxes[1].set_ylabel('CPU Time Total')\n\naxes[2].bar(labels, [i.self_cuda_time_total for i in tas])\naxes[2].set_ylabel('Cuda Time Total')","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:35:11.956050Z","iopub.execute_input":"2024-05-31T21:35:11.957062Z","iopub.status.idle":"2024-05-31T21:35:13.033222Z","shell.execute_reply.started":"2024-05-31T21:35:11.957025Z","shell.execute_reply":"2024-05-31T21:35:13.031935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Conclusion\n\nI hope you found this notebook helpful. Feel free to drop your own fingerprint tips in the comments","metadata":{}}]}