{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":67356,"databundleVersionId":8006601,"sourceType":"competition"}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport joblib\nimport duckdb\n\nfrom rdkit import Chem\nfrom rdkit.Chem import AllChem, Descriptors, Crippen, rdMolDescriptors\nfrom sklearn.preprocessing import MinMaxScaler, MaxAbsScaler\nimport concurrent.futures\nimport pandas as pd\nimport numpy as np\nfrom mordred import Calculator, descriptors","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## PreProcessing Functions and their resource utilizations","metadata":{}},{"cell_type":"markdown","source":"#### mol_descriptors/ utilizaed: parallelize_dataframe","metadata":{}},{"cell_type":"code","source":"\n# Initialize Mordred calculator with all descriptors\ncalc = Calculator(descriptors, ignore_3D=True)\n\ndef mol_descriptors(smiles_list, pre=''):\n    moldescriptors = []\n    for smile in smiles_list:\n        mol = Chem.MolFromSmiles(smile)\n        if mol:\n            rdkit_desc = {\n                pre+'MolecularWeight': Descriptors.MolWt(mol),\n                pre+'LogP': Descriptors.MolLogP(mol),\n                pre+'TPSA': Descriptors.TPSA(mol),\n                pre+'NumRotatableBonds': Descriptors.NumRotatableBonds(mol),\n                pre+'NumHDonors': Descriptors.NumHDonors(mol),\n                pre+'NumHAcceptors': Descriptors.NumHAcceptors(mol),\n                pre+'NumRings': Descriptors.RingCount(mol),\n                pre+'NumAromaticRings': Descriptors.NumAromaticRings(mol),\n                pre+'ExactMass': Descriptors.ExactMolWt(mol),\n                pre+'HeavyAtomCount': Descriptors.HeavyAtomCount(mol),\n                pre+'NumValenceElectrons': Descriptors.NumValenceElectrons(mol),\n                pre+'FractionCSP3': Descriptors.FractionCSP3(mol),\n                pre+'MolMR': Descriptors.MolMR(mol),\n                pre+'FormalCharge': Chem.GetFormalCharge(mol),\n                pre+'NumAliphaticRings': Descriptors.NumAliphaticRings(mol),\n                pre+'NumSaturatedRings': Descriptors.NumSaturatedRings(mol),\n                pre+'NumHeteroatoms': Descriptors.NumHeteroatoms(mol),\n                pre+'NumSaturatedCarbocycles': Descriptors.NumSaturatedCarbocycles(mol),\n                pre+'NumAliphaticHeterocycles': Descriptors.NumAliphaticHeterocycles(mol),\n                pre+'NumAromaticHeterocycles': Descriptors.NumAromaticHeterocycles(mol),\n                pre+'AtomCount': mol.GetNumAtoms(),\n                pre+'NumSingleBonds': len([bond for bond in mol.GetBonds() if bond.GetBondType() == Chem.rdchem.BondType.SINGLE]),\n                pre+'NumDoubleBonds': len([bond for bond in mol.GetBonds() if bond.GetBondType() == Chem.rdchem.BondType.DOUBLE]),\n                pre+'NumTripleBonds': len([bond for bond in mol.GetBonds() if bond.GetBondType() == Chem.rdchem.BondType.TRIPLE]),\n                pre+'NumAromaticBonds': len([bond for bond in mol.GetBonds() if bond.GetIsAromatic()]),\n                pre+'MolecularConnectivityIndex': Descriptors.MolLogP(mol),  # Example\n                pre+'Kier_Hall_Alpha': Descriptors.Kappa3(mol),  # Example\n                pre+'HOMO': Descriptors.MaxAbsEStateIndex(mol),  # Example (approximation)\n                pre+'LUMO': Descriptors.MinAbsEStateIndex(mol),  # Example (approximation)\n            }\n            \n            # Calculate Mordred descriptors\n            mordred_desc = calc(mol)\n            mordred_desc = {f\"{pre}{str(key)}\": value for key, value in mordred_desc.items()}\n            \n            # Combine RDKit and Mordred descriptors\n            rdkit_desc.update(mordred_desc)\n            moldescriptors.append(rdkit_desc)\n            \n        else:\n            # If molecule is invalid, append None for all descriptors\n            rdkit_desc = {key: None for key in [\n                pre+'MolecularWeight',\n                pre+'LogP',\n                pre+'TPSA',\n                pre+'NumRotatableBonds',\n                pre+'NumHDonors',\n                pre+'NumHAcceptors',\n                pre+'NumRings',\n                pre+'NumAromaticRings',\n                pre+'ExactMass',\n                pre+'HeavyAtomCount',\n                pre+'NumValenceElectrons',\n                pre+'FractionCSP3',\n                pre+'MolMR',\n                pre+'FormalCharge',\n                pre+'NumAliphaticRings',\n                pre+'NumSaturatedRings',\n                pre+'NumHeteroatoms',\n                pre+'NumSaturatedCarbocycles',\n                pre+'NumAliphaticHeterocycles',\n                pre+'NumAromaticHeterocycles',\n                pre+'AtomCount',\n                pre+'NumSingleBonds',\n                pre+'NumDoubleBonds',\n                pre+'NumTripleBonds',\n                pre+'NumAromaticBonds',\n                pre+'MolecularConnectivityIndex',\n                pre+'Kier_Hall_Alpha',\n                pre+'HOMO',\n                pre+'LUMO',\n            ]}\n            \n            # Add None for Mordred descriptors\n            mordred_desc = {f\"{pre}{str(desc)}\": None for desc in calc.descriptors}\n            rdkit_desc.update(mordred_desc)\n            \n            moldescriptors.append(rdkit_desc)\n            \n    return moldescriptors\n\ndef parallelize_dataframe(df, molecule_smiles_cat, pre='', n_cores=5):\n    df_split = np.array_split(df, n_cores)\n    smiles_lists = [chunk[molecule_smiles_cat].tolist() for chunk in df_split]\n    \n    with concurrent.futures.ProcessPoolExecutor(max_workers=n_cores) as executor:\n        results = executor.map(mol_descriptors, smiles_lists, [pre]*n_cores)\n    \n    results = [item for sublist in results for item in sublist]\n    \n    descriptors_df = pd.DataFrame(results)\n    return pd.concat([df.reset_index(drop=True), descriptors_df.reset_index(drop=True)], axis=1)\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### ecfp/ utilized: parallelize_ecfp","metadata":{}},{"cell_type":"code","source":"def ecfp(smiles_list, pre=''):\n    descriptors = []\n    for smile in smiles_list:\n        descriptors.append({pre+'ecfp': list(AllChem.GetMorganFingerprintAsBitVect(Chem.MolFromSmiles(smile), 2, nBits=262))})\n    return descriptors\n    # Generate ECFPs\n    #   filtered_result[pre+'molecule_ecfp'] = filtered_result[molecule_smiles_cat].apply(lambda smile: list(AllChem.GetMorganFingerprintAsBitVect(Chem.MolFromSmiles(smile), 100, nBits=2000)))\n    #   return filtered_result\n\ndef parallelize_ecfp(df, molecule_smiles_cat, pre='', n_cores=5):\n    df_split = np.array_split(df, n_cores)\n    smiles_lists = [chunk[molecule_smiles_cat].tolist() for chunk in df_split]\n    \n    with concurrent.futures.ProcessPoolExecutor(max_workers=n_cores) as executor:\n        results = executor.map(ecfp, smiles_lists, [pre]*n_cores)\n    \n    results = [item for sublist in results for item in sublist]\n    \n    descriptors_df = pd.DataFrame(results)\n    return pd.concat([df.reset_index(drop=True), descriptors_df.reset_index(drop=True)], axis=1)\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Normalization","metadata":{}},{"cell_type":"code","source":"def normalize_df(df, columns_to_scale, scalers):\n    for column in columns_to_scale:\n        if column not in scalers:\n            scaler = MaxAbsScaler()\n            scalers[column] = scaler  # Save the scaler in case you need it later\n            df[column] = scaler.fit_transform(df[[column]])\n        else:\n            df[column] = scalers[column].transform(df[[column]])\n    return df\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Get 5000000 records of bindings and the same number of non-bindings and store to a csv","metadata":{}},{"cell_type":"code","source":"train_path = '/kaggle/input/leash-BELKA/train.parquet'\n# test_path = '/kaggle/input/leash-BELKA/test.parquet'\nrange = 5000000\ncon = duckdb.connect()\n\ndf = con.query(f\"\"\"(SELECT *\n                        FROM parquet_scan('{train_path}')\n                        WHERE binds = 0\n                        ORDER BY random()\n                        LIMIT {range})\n                        UNION ALL\n                        (SELECT *\n                        FROM parquet_scan('{train_path}')\n                        WHERE binds = 1\n                        ORDER BY random()\n                        LIMIT {range})\"\"\").df()\n\ncon.close()\ndf.to_csv('somepart.csv')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Processing Building Block one, two, three","metadata":{}},{"cell_type":"markdown","source":"#### Block one","metadata":{}},{"cell_type":"code","source":"b1processed = parallelize_dataframe(df, 'buildingblock1_smiles', 'b1')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Block two","metadata":{}},{"cell_type":"code","source":"b2processed = parallelize_dataframe(b1processed, 'buildingblock2_smiles', 'b2')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Block three","metadata":{}},{"cell_type":"code","source":"b3processed = parallelize_dataframe(b2processed, 'buildingblock3_smiles', 'b3')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Store the extracted features beside the original data in another csv","metadata":{}},{"cell_type":"code","source":"b3processed.to_csv('somepart_processed.csv')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Normalization of all the extracted features, the Scaler is set in the function\n#### the columns are normalized separately","metadata":{}},{"cell_type":"code","source":"scalers = {}\nnormalized = normalize_df(b3processed, [\n  'b1MolecularWeight',\n  'b1LogP',\n  'b1TPSA',\n  'b1NumRotatableBonds',\n  'b1NumHDonors',\n  'b1NumHAcceptors',\n  'b1NumRings',\n  'b1NumAromaticRings',\n  'b1ExactMass',\n  'b1HeavyAtomCount',\n  'b1NumValenceElectrons',\n  'b1FractionCSP3',\n  'b1MolMR',\n  'b1FormalCharge',\n  'b1NumAliphaticRings',\n  'b1NumSaturatedRings',\n  'b1NumHeteroatoms',\n  'b1NumHeterocycles',\n  'b1NumSaturatedCarbocycles',\n  'b1NumAliphaticHeterocycles',\n  'b1NumAromaticHeterocycles',\n  'b1BondCount',\n  'b1AtomCount',\n  'b1NumSingleBonds',\n  'b1NumDoubleBonds',\n  'b1NumTripleBonds',\n  'b1NumAromaticBonds',\n\n  'b2MolecularWeight',\n  'b2LogP',\n  'b2TPSA',\n  'b2NumRotatableBonds',\n  'b2NumHDonors',\n  'b2NumHAcceptors',\n  'b2NumRings',\n  'b2NumAromaticRings',\n  'b2ExactMass',\n  'b2HeavyAtomCount',\n  'b2NumValenceElectrons',\n  'b2FractionCSP3',\n  'b2MolMR',\n  'b2FormalCharge',\n  'b2NumAliphaticRings',\n  'b2NumSaturatedRings',\n  'b2NumHeteroatoms',\n  'b2NumHeterocycles',\n  'b2NumSaturatedCarbocycles',\n  'b2NumAliphaticHeterocycles',\n  'b2NumAromaticHeterocycles',\n  'b2BondCount',\n  'b2AtomCount',\n  'b2NumSingleBonds',\n  'b2NumDoubleBonds',\n  'b2NumTripleBonds',\n  'b2NumAromaticBonds',\n\n  'b3MolecularWeight',\n  'b3LogP',\n  'b3TPSA',\n  'b3NumRotatableBonds',\n  'b3NumHDonors',\n  'b3NumHAcceptors',\n  'b3NumRings',\n  'b3NumAromaticRings',\n  'b3ExactMass',\n  'b3HeavyAtomCount',\n  'b3NumValenceElectrons',\n  'b3FractionCSP3',\n  'b3MolMR',\n  'b3FormalCharge',\n  'b3NumAliphaticRings',\n  'b3NumSaturatedRings',\n  'b3NumHeteroatoms',\n  'b3NumHeterocycles',\n  'b3NumSaturatedCarbocycles',\n  'b3NumAliphaticHeterocycles',\n  'b3NumAromaticHeterocycles',\n  'b3BondCount',\n  'b3AtomCount',\n  'b3NumSingleBonds',\n  'b3NumDoubleBonds',\n  'b3NumTripleBonds',\n  'b3NumAromaticBonds',\n], scalers)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Store the Scalers for later (for test ds)","metadata":{}},{"cell_type":"code","source":"joblib.dump(scalers, 'scalers.joblib')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Store the normalized features again beside the original data in another csv","metadata":{}},{"cell_type":"code","source":"normalized.to_csv('somepart_normalized.csv')","metadata":{},"execution_count":null,"outputs":[]}]}