{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":67356,"databundleVersionId":8006601,"sourceType":"competition"}],"dockerImageVersionId":30684,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom scipy import sparse\nimport os, gc\n\nimport warnings\nimport pandas as pd\nfrom pandas.errors import SettingWithCopyWarning\nwarnings.simplefilter(action='ignore', category=(SettingWithCopyWarning))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-15T12:42:40.847840Z","iopub.execute_input":"2024-06-15T12:42:40.848225Z","iopub.status.idle":"2024-06-15T12:42:42.111292Z","shell.execute_reply.started":"2024-06-15T12:42:40.848195Z","shell.execute_reply":"2024-06-15T12:42:42.110119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mol_bits=2048\nchunck_size=10**4\nsample_nrows=13*10**6 #plus positive targets. 9,5M*3_Targets rows\ncreate_ecfp=True","metadata":{"execution":{"iopub.status.busy":"2024-06-15T12:42:49.075668Z","iopub.execute_input":"2024-06-15T12:42:49.076254Z","iopub.status.idle":"2024-06-15T12:42:49.082625Z","shell.execute_reply.started":"2024-06-15T12:42:49.076219Z","shell.execute_reply":"2024-06-15T12:42:49.081140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp=pd.read_csv('/kaggle/input/leash-BELKA/train.csv', nrows = 5)","metadata":{"execution":{"iopub.status.busy":"2024-06-15T12:43:01.394240Z","iopub.execute_input":"2024-06-15T12:43:01.394604Z","iopub.status.idle":"2024-06-15T12:43:01.424186Z","shell.execute_reply.started":"2024-06-15T12:43:01.394574Z","shell.execute_reply":"2024-06-15T12:43:01.423056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install rdkit\n!pip install mapply","metadata":{"execution":{"iopub.status.busy":"2024-06-15T12:43:05.444466Z","iopub.execute_input":"2024-06-15T12:43:05.444861Z","iopub.status.idle":"2024-06-15T12:43:37.304828Z","shell.execute_reply.started":"2024-06-15T12:43:05.444822Z","shell.execute_reply":"2024-06-15T12:43:37.303534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from rdkit import Chem\nfrom rdkit.Chem import AllChem\nimport mapply\n\nmapply.init( n_workers=-1,progressbar=False)\n\n#Coversion to ECFPs  from https://www.kaggle.com/code/andrewdblevins/leash-tutorial-ecfps-and-random-forest\n# Generate ECFPs\ndef gen_ecfp(molecule, radius=2, bits=mol_bits):\n    if molecule is None:\n        return None\n    return np.array(AllChem.GetMorganFingerprintAsBitVect(molecule, radius, nBits=bits))\n\ndef gen_molecule(molecule_smiles):\n    if molecule_smiles is None:\n        return None    \n    molecule = Chem.MolFromSmiles(molecule_smiles)\n   # Me = Chem.MolFromSmiles('C')\n   # Dy = Chem.MolFromSmiles('[Dy]')\n   # molecule = AllChem.ReplaceSubstructs(molecule, Dy, Me)    \n   # Chem.SanitizeMol(molecule)\n    return np.array(molecule)\n\ndef generate_ecfp_chunck(df):\n    df['molecule'] = df['molecule_smiles'].mapply(gen_molecule)\n    df['ecfp'] = df['molecule'].mapply(gen_ecfp)\n    return sparse.csr_matrix(np.stack(df['ecfp'], axis=0).astype(np.uint8))\n\ndef generate_ecfp(df):\n    n_chunks = ((len(df) - 1) // chunck_size) + 1\n    df_ecfp=generate_ecfp_chunck(df[0:chunck_size])\n\n    for i in range(1,n_chunks):\n        a=i*chunck_size\n        b=i*chunck_size+chunck_size\n        #print(a,b)\n        #print(df_tmp.shape)\n        df_ecfp=sparse.vstack([df_ecfp,generate_ecfp_chunck(df[a:b].copy())]) \n    return df_ecfp","metadata":{"execution":{"iopub.status.busy":"2024-05-08T19:26:18.470163Z","iopub.execute_input":"2024-05-08T19:26:18.470626Z","iopub.status.idle":"2024-05-08T19:26:18.483857Z","shell.execute_reply.started":"2024-05-08T19:26:18.470569Z","shell.execute_reply":"2024-05-08T19:26:18.48274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get unique block in test and train","metadata":{}},{"cell_type":"code","source":"print('Loading test data')\ntest=pd.read_parquet('/kaggle/input/leash-BELKA/test.parquet', engine = 'pyarrow')\ntest_unique=test.buildingblock1_smiles.unique().tolist()+test.buildingblock2_smiles.unique().tolist()+test.buildingblock3_smiles.unique().tolist()\ntest_unique=set(test_unique)\ntest=pd.DataFrame(test['molecule_smiles'].unique(),columns=['molecule_smiles'])\n\nif create_ecfp:\n    print('Generating ecfp from test data')\n    test_ecfp=generate_ecfp(test)\n    sparse.save_npz(\"test_ecfp.npz\", test_ecfp)\n    del test_ecfp\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-08T19:26:21.427601Z","iopub.execute_input":"2024-05-08T19:26:21.428562Z","iopub.status.idle":"2024-05-08T19:26:24.033316Z","shell.execute_reply.started":"2024-05-08T19:26:21.428441Z","shell.execute_reply":"2024-05-08T19:26:24.030403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"import sys\nobjects=[]\nfor name,obj in locals().items():\n    objects.append([name,sys.getsizeof(obj)])\nsorted(objects,key=lambda x: x[1],reverse=True)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T09:54:23.179875Z","iopub.execute_input":"2024-04-24T09:54:23.180893Z","iopub.status.idle":"2024-04-24T09:54:23.196403Z","shell.execute_reply.started":"2024-04-24T09:54:23.180855Z","shell.execute_reply":"2024-04-24T09:54:23.194807Z"}}},{"cell_type":"code","source":"train_unique=[]\nfor col in ['buildingblock1_smiles','buildingblock2_smiles','buildingblock3_smiles']:\n    BBs = pd.read_parquet('/kaggle/input/leash-BELKA/train.parquet', engine = 'pyarrow', columns=[col])[::3]\n    block_unique=BBs[col].unique().tolist()\n    print(f'Unique Blocks train {col}:',len(block_unique))\n    train_unique+=block_unique\ntrain_unique=set(train_unique)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Unique Blocks in test:',len(test_unique))\nprint('Unique Blocks in train:',len(train_unique))\nunique_blocks=train_unique.union(test_unique)\nprint('Unique Blocks in data:',len(unique_blocks))\nblocks_dict={list(unique_blocks)[i]:i for i in range(len(unique_blocks))}\nnp.save('blocks_dict.npy', blocks_dict) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del unique_blocks,train_unique,block_unique,BBs,test_unique,test\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create New DataFrame.","metadata":{}},{"cell_type":"markdown","source":"Encode building block smiles to integers.","metadata":{}},{"cell_type":"code","source":"BBs_1=pd.read_parquet('/kaggle/input/leash-BELKA/train.parquet', engine = 'pyarrow', columns=['buildingblock1_smiles'])[::3]['buildingblock1_smiles'].map(blocks_dict).values.astype('uint16')\nBBs_2=pd.read_parquet('/kaggle/input/leash-BELKA/train.parquet', engine = 'pyarrow', columns=['buildingblock2_smiles'])[::3]['buildingblock2_smiles'].map(blocks_dict).values.astype('uint16')\nBBs_3=pd.read_parquet('/kaggle/input/leash-BELKA/train.parquet', engine = 'pyarrow', columns=['buildingblock3_smiles'])[::3]['buildingblock3_smiles'].map(blocks_dict).values.astype('uint16')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Protein_name bins rows to columns.","metadata":{}},{"cell_type":"code","source":"bin_BRD4=pd.read_parquet('/kaggle/input/leash-BELKA/train.parquet', engine = 'pyarrow', columns=['binds'])[::3].values.ravel().astype('uint8')\nbin_HSA=pd.read_parquet('/kaggle/input/leash-BELKA/train.parquet', engine = 'pyarrow', columns=['binds'])[1::3].values.ravel().astype('uint8')\nbin_sEH=pd.read_parquet('/kaggle/input/leash-BELKA/train.parquet', engine = 'pyarrow', columns=['binds'])[2::3].values.ravel().astype('uint8')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create new DataFrame","metadata":{}},{"cell_type":"code","source":"#molecule_smiles= pd.read_parquet('/kaggle/input/leash-BELKA/train.parquet', engine = 'pyarrow', columns=['molecule_smiles'])[::3]\nmolecule_smiles=pd.read_csv('/kaggle/input/leash-BELKA/train.csv',usecols=[4], skiprows=lambda x: x%3 != 0).values.ravel()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = {'buildingblock1_smiles':BBs_1,\n        'buildingblock2_smiles':BBs_2,\n        'buildingblock3_smiles':BBs_3,\n        'molecule_smiles': molecule_smiles,\n        'binds_BRD4':bin_BRD4, \n        'binds_HSA':bin_HSA, \n        'binds_sEH':bin_sEH}\n\ntrain = pd.DataFrame(data=data)\ntrain.head(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Create Sample Data.\nTARGETS=['binds_BRD4','binds_HSA','binds_sEH']\n\nprint('Original Data')\nprint('Shape :',train.shape)\nprint('block1 :',train.buildingblock1_smiles.nunique())\nprint('block2 :',train.buildingblock2_smiles.nunique())\nprint('block3 :',train.buildingblock3_smiles.nunique())  \nfor target in TARGETS:\n    print(f'Positive Cases Ratio for {target}:',(train[target].sum()/train.shape[0]))\n          \nprint('#**************************#')\n    \ntt1=train[(train.binds_BRD4 != 0) | (train.binds_HSA !=0) | (train.binds_sEH !=0)]\ntt2=train[(train.binds_BRD4 == 0) & (train.binds_HSA ==0) & (train.binds_sEH ==0)].sample(n=sample_nrows,random_state=42)  \ntrain=pd.concat([tt1,tt2]).sample(frac=1,random_state=42).reset_index(drop=True) \ntrain.to_csv('train_sample.csv',index=False)\nprint('Sampled Data 9,5M rows')\nprint('Shape :',train.shape)\nprint('block1 :',train.buildingblock1_smiles.nunique())\nprint('block2 :',train.buildingblock2_smiles.nunique())\nprint('block3 :',train.buildingblock3_smiles.nunique())    \nfor target in TARGETS:\n    print(f'Positive Cases Ratio for {target} :', target,(train[target].sum()/train.shape[0]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del tt1,tt2,molecule_smiles,BBs_1,BBs_2,BBs_3,bin_BRD4,bin_HSA,bin_sEH\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if create_ecfp:\n    print('Generating ecfp from train data')\n    train_ecfp=generate_ecfp(train[['molecule_smiles']])\n    sparse.save_npz(\"train_ecfp.npz\", train_ecfp)\n\n    del train,train_ecfp\n    gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}