{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":67356,"databundleVersionId":8006601,"sourceType":"competition"}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-07-08T18:16:14.060585Z","iopub.execute_input":"2024-07-08T18:16:14.061181Z","iopub.status.idle":"2024-07-08T18:16:15.068079Z","shell.execute_reply.started":"2024-07-08T18:16:14.061157Z","shell.execute_reply":"2024-07-08T18:16:15.067118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install pandas scikit-learn numpy tqdm pyarrow rdkit\n","metadata":{"execution":{"iopub.status.busy":"2024-07-08T18:16:21.962196Z","iopub.execute_input":"2024-07-08T18:16:21.96314Z","iopub.status.idle":"2024-07-08T18:16:37.009297Z","shell.execute_reply.started":"2024-07-08T18:16:21.963109Z","shell.execute_reply":"2024-07-08T18:16:37.008212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Incremental Ensemble Model Training\n\nThis script trains an ensemble model on a large Parquet dataset using incremental learning techniques:\n\n## Libraries:\n- Pandas, Scikit-learn (SGDClassifier, GaussianNB, VotingClassifier), RDKit, PyArrow, tqdm.\n  \n## Functions:\n- Converts SMILES to feature vectors.\n- Reads Parquet data in chunks.\n- Samples, preprocesses, and encodes categorical variables.\n- Trains base models incrementally and creates a soft-voting ensemble.\n  \n\n\nEfficiently handles large datasets while training complex models, ensuring memory and processing efficiency.\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.svm import SVC\nfrom sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\nimport numpy as np\nfrom tqdm.notebook import tqdm\nimport pyarrow.parquet as pq\nfrom rdkit import Chem\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.ensemble import VotingClassifier\n\n# Function to convert SMILES to feature vector\ndef smiles_to_features(smiles):\n    mol = Chem.MolFromSmiles(smiles)\n    if mol is None:\n        return None\n    \n    # Calculate molecular descriptors (example: here using Morgan fingerprints)\n    fp = Chem.RDKFingerprint(mol)\n    arr = np.zeros((1,))\n    Chem.DataStructs.ConvertToNumpyArray(fp, arr)\n    \n    return arr\n\n# Function to preprocess data\ndef preprocess_data(data):\n    X = []\n    y = data['binds'].tolist()\n    \n    for _, row in tqdm(data.iterrows(), total=len(data)):\n        features = smiles_to_features(row['molecule_smiles'])\n        if features is not None:\n            X.append(features)\n    \n    return np.array(X), np.array(y)\n\n# Function to read data in chunks using PyArrow\ndef read_parquet_in_chunks(file_path, chunk_size):\n    parquet_file = pq.ParquetFile(file_path)\n    num_row_groups = parquet_file.num_row_groups\n\n    for i in range(num_row_groups):\n        df = parquet_file.read_row_group(i).to_pandas()\n        yield df\n\n# Function to sample and preprocess data chunks\ndef sample_and_preprocess(file_path, sample_size, chunk_size):\n    sampled_chunks = []\n    remaining_sample_size = sample_size\n\n    for chunk in read_parquet_in_chunks(file_path, chunk_size):\n        if remaining_sample_size <= 0:\n            break\n        sampled_chunk = chunk.sample(frac=min(remaining_sample_size / len(chunk), 1.0), random_state=42)\n        sampled_chunks.append(sampled_chunk)\n        remaining_sample_size -= len(sampled_chunk)\n\n    if sampled_chunks:\n        sampled_df = pd.concat(sampled_chunks)\n        return sampled_df\n    else:\n        return None\n\n# Function to train ensemble model in chunks\ndef train_ensemble_in_chunks(file_path, sample_size, chunk_size):\n    sampled_df = sample_and_preprocess(file_path, sample_size, chunk_size)\n    if sampled_df is None:\n        print(\"No data sampled.\")\n        return\n    \n    # Check class distribution\n    class_distribution = sampled_df['binds'].value_counts()\n    print(\"Class distribution before training:\", class_distribution)\n    \n    # Ensure there are at least two classes\n    if len(class_distribution) < 2:\n        raise ValueError(\"The number of classes has to be greater than one; got {} class\".format(len(class_distribution)))\n\n    # Encode categorical variable\n    le = LabelEncoder()\n    sampled_df['protein_name'] = le.fit_transform(sampled_df['protein_name'])\n\n    # Preprocess train data\n    X, y = preprocess_data(sampled_df)\n\n    # Split data into train and test sets\n    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n    # Define base models with partial_fit method\n    sgd_model = SGDClassifier(loss='log', max_iter=1000, tol=1e-3, random_state=42)\n    nb_model = GaussianNB()\n\n    # Train base models incrementally\n    sgd_model.partial_fit(X_train, y_train, classes=np.unique(y_train))\n    nb_model.partial_fit(X_train, y_train, classes=np.unique(y_train))\n\n    # Define ensemble model using VotingClassifier\n    ensemble_model = VotingClassifier(estimators=[\n        ('sgd', sgd_model),\n        ('nb', nb_model)\n    ], voting='soft')\n\n    # Fit ensemble model on the training data\n    ensemble_model.fit(X_train, y_train)\n\n    # Predict on training data\n    y_train_pred = ensemble_model.predict(X_train)\n\n    # Calculate training accuracy\n    train_accuracy = accuracy_score(y_train, y_train_pred)\n    print(\"Training accuracy:\", train_accuracy)\n\n    # Predict on test data\n    y_test_pred = ensemble_model.predict(X_test)\n\n    # Calculate test accuracy\n    test_accuracy = accuracy_score(y_test, y_test_pred)\n    print(\"Test accuracy:\", test_accuracy)\n\n    return ensemble_model\n\n# Example usage for training ensemble model in chunks\ntrain_file_path = '/kaggle/input/leash-BELKA/train.parquet'\nsample_size_train = 100000\nchunk_size_train = 5000\n\nbest_ensemble_model = train_ensemble_in_chunks(train_file_path, sample_size_train, chunk_size_train)\n","metadata":{"execution":{"iopub.status.busy":"2024-07-08T18:21:22.121732Z","iopub.execute_input":"2024-07-08T18:21:22.122078Z","iopub.status.idle":"2024-07-08T18:26:04.978258Z","shell.execute_reply.started":"2024-07-08T18:21:22.122045Z","shell.execute_reply":"2024-07-08T18:26:04.97732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Overview of Evaluating Ensemble Model on Test Data in Chunks\n\n## Problem Description\n- Evaluates an ensemble model's performance on test data stored in a Parquet file (`test.parquet`).\n- Data is processed in chunks due to its size, ensuring efficient evaluation and prediction.\n\n## Data Preparation\n- Reads test data in chunks using PyArrow for efficient memory usage.\n- Samples a specified number of data points (`sample_size_test`) from the test dataset.\n\n## Preprocessing\n- Converts SMILES representations of molecules into feature vectors using RDKit (Morgan fingerprints).\n- Encodes categorical variable (`protein_name`) using `LabelEncoder` for compatibility with machine learning models.\n\n## Model Evaluation\n- Evaluates the pre-trained ensemble model on preprocessed test data chunks.\n- Predicts binding probabilities using the ensemble model and outputs the results.\n\n## Output\n- Saves the predictions to a CSV file named `test_predictions.csv` for further analysis.\n- Displays the first few rows of the prediction results to verify output correctness.\n\n\n\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom tqdm.notebook import tqdm\nimport pyarrow.parquet as pq\nfrom rdkit import Chem\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.ensemble import VotingClassifier\n\n# Function to convert SMILES to feature vector\ndef smiles_to_features(smiles):\n    mol = Chem.MolFromSmiles(smiles)\n    if mol is None:\n        return None\n    \n    # Calculate molecular descriptors (example: here using Morgan fingerprints)\n    fp = Chem.RDKFingerprint(mol)\n    arr = np.zeros((1,))\n    Chem.DataStructs.ConvertToNumpyArray(fp, arr)\n    \n    return arr\n\n# Function to preprocess data\ndef preprocess_data(data):\n    X = []\n    \n    for _, row in tqdm(data.iterrows(), total=len(data)):\n        features = smiles_to_features(row['molecule_smiles'])\n        if features is not None:\n            X.append(features)\n    \n    return np.array(X)\n\n# Function to read data in chunks using PyArrow\ndef read_parquet_in_chunks(file_path, chunk_size):\n    parquet_file = pq.ParquetFile(file_path)\n    num_row_groups = parquet_file.num_row_groups\n\n    for i in range(num_row_groups):\n        df = parquet_file.read_row_group(i).to_pandas()\n        yield df\n\n# Function to sample and preprocess data chunks\ndef sample_and_preprocess(file_path, sample_size, chunk_size):\n    sampled_chunks = []\n    remaining_sample_size = sample_size\n\n    for chunk in read_parquet_in_chunks(file_path, chunk_size):\n        if remaining_sample_size <= 0:\n            break\n        sampled_chunk = chunk.sample(frac=min(remaining_sample_size / len(chunk), 1.0), random_state=42)\n        sampled_chunks.append(sampled_chunk)\n        remaining_sample_size -= len(sampled_chunk)\n\n    if sampled_chunks:\n        sampled_df = pd.concat(sampled_chunks)\n        return sampled_df\n    else:\n        return None\n\n# Function to evaluate model on test data in chunks and predict binding probability\ndef evaluate_model_in_chunks(model, file_path, sample_size, chunk_size):\n    sampled_df = sample_and_preprocess(file_path, sample_size, chunk_size)\n    if sampled_df is None:\n        print(\"No data sampled.\")\n        return\n    \n    # Encode categorical variable\n    le = LabelEncoder()\n    sampled_df['protein_name'] = le.fit_transform(sampled_df['protein_name'])\n\n    # Preprocess test data\n    X = preprocess_data(sampled_df)\n\n    # Predict binding probability\n    y_pred_prob = model.predict_proba(X)[:, 1]\n\n    # Create output DataFrame\n    output_df = pd.DataFrame({'id': sampled_df['id'], 'binds': y_pred_prob})\n\n    # Save the output to a CSV file\n    output_df.to_csv('test_predictions.csv', index=False)\n    print(\"Predictions saved to test_predictions.csv\")\n\n    # Display the first few rows of the output DataFrame\n    print(output_df.head())\n\n# Example usage for evaluating ensemble model on test data in chunks\ntest_file_path = '/kaggle/input/leash-BELKA/test.parquet'\nsample_size_test = 1674898\nchunk_size_test = 5000\n\n# Assuming you have the best_ensemble_model already trained\nevaluate_model_in_chunks(best_ensemble_model, test_file_path, sample_size_test, chunk_size_test)\n","metadata":{"execution":{"iopub.status.busy":"2024-07-08T18:26:11.731222Z","iopub.execute_input":"2024-07-08T18:26:11.731598Z"},"trusted":true},"execution_count":null,"outputs":[]}]}