{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is a fork of \"Fork of NLP_Regression 12a31a\" (https://www.kaggle.com/code/erotar/fork-of-nlp-regression-12a31a) without modification.  Run the node and the submitssion gets 0.594 in public board.","metadata":{}},{"cell_type":"markdown","source":"# NLP Regression  \nThis is just a simple experiment with NLP and to get low MRRMSE(Mean Rowwise Root Mean Sqared Error)\n\nWe're going to use SMILES embedding.\n\nSteps to do above:\n1. Preprocess the data \n2. Make a TextVectorizer and embedding\n3. Build a Model\n4. Visualize, Evaluate and Repeat \n\nLet's go...\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import tensorflow as tf\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:40.955344Z","iopub.execute_input":"2023-11-10T23:08:40.955715Z","iopub.status.idle":"2023-11-10T23:08:40.960845Z","shell.execute_reply.started":"2023-11-10T23:08:40.955685Z","shell.execute_reply":"2023-11-10T23:08:40.959861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read data","metadata":{}},{"cell_type":"code","source":"data = pd.read_parquet(\"/kaggle/input/open-problems-single-cell-perturbations/de_train.parquet\")\nid_map = pd.read_csv(\"/kaggle/input/open-problems-single-cell-perturbations/id_map.csv\")\nsample_submission = pd.read_csv(\"/kaggle/input/open-problems-single-cell-perturbations/sample_submission.csv\")\nRNDST1=271828\nRNDST2=314159","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:40.962469Z","iopub.execute_input":"2023-11-10T23:08:40.962749Z","iopub.status.idle":"2023-11-10T23:08:46.147085Z","shell.execute_reply.started":"2023-11-10T23:08:40.962725Z","shell.execute_reply":"2023-11-10T23:08:46.145986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" **🔑Tip:** Stare at the data for a while to get some insights and ideas to run experiments.","metadata":{}},{"cell_type":"code","source":"data.head(20)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:46.149112Z","iopub.execute_input":"2023-11-10T23:08:46.149412Z","iopub.status.idle":"2023-11-10T23:08:46.192971Z","shell.execute_reply.started":"2023-11-10T23:08:46.149386Z","shell.execute_reply":"2023-11-10T23:08:46.191998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Aux functions ","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.callbacks import ModelCheckpoint\n\ndef create_model_checkpoint(filepath, monitor='val_mae', save_best_only=True,\n                            save_weights_only=True, mode='auto', verbose=0):\n    \"\"\"\n    Create a ModelCheckpoint callback for saving the best model weights during training.\n\n    Args:\n        filepath (str): Filepath to save the best weights.\n        monitor (str): Metric to monitor (e.g., 'val_loss' or 'val_mae').\n        save_best_only (bool): Save only the best weights.\n        save_weights_only (bool): Save only the model's weights, not the entire model.\n        mode (str): One of {'auto', 'min', 'max'}. In 'min' mode, it saves when the monitored metric decreases.\n        verbose (int): Verbosity mode. 0 = silent, 1 = progress bar, 2 = one line per epoch.\n\n    Returns:\n        keras.callbacks.ModelCheckpoint: ModelCheckpoint callback.\n    \"\"\"\n    checkpoint = ModelCheckpoint(\n        filepath=filepath,\n        monitor=monitor,\n        save_best_only=save_best_only,\n        save_weights_only=save_weights_only,\n        mode=mode,\n        verbose=verbose\n    )\n    return checkpoint\n","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:46.194605Z","iopub.execute_input":"2023-11-10T23:08:46.194989Z","iopub.status.idle":"2023-11-10T23:08:46.203253Z","shell.execute_reply.started":"2023-11-10T23:08:46.194958Z","shell.execute_reply":"2023-11-10T23:08:46.202174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_training_history(history, metrics):\n    \"\"\"\n    Plot training history curves for loss and evaluation metrics on the same line.\n\n    Args:\n        history (keras.callbacks.History): Training history object.\n        metrics (list): List of metric names to plot.\n\n    Returns:\n        None\n    \"\"\"\n    loss = history.history['loss']\n    val_loss = history.history['val_loss']\n\n    epochs = range(len(loss))\n\n    plt.figure(figsize=(12, 6))\n\n    # Plot loss\n    plt.subplot(1, 2, 1)\n    plt.plot(epochs, loss, label='Training Loss', color=\"blue\")\n    plt.plot(epochs, val_loss, label='Validation Loss', color=\"red\")\n    plt.title('Loss')\n    plt.xlabel('Epochs')\n    plt.legend()\n\n    # Plot specified evaluation metrics on the same line\n    for metric in metrics:\n        train_metric_name = f'Training {metric.capitalize()}'\n        val_metric_name = f'Validation {metric.capitalize()}'\n        train_metric = history.history[metric]\n        val_metric = history.history['val_' + metric]\n\n        plt.subplot(1, 2, 2)\n        plt.plot(epochs, train_metric, label=train_metric_name, color=\"green\")\n        plt.plot(epochs, val_metric, label=val_metric_name, color=\"orange\")\n\n    plt.title('Metrics')\n    plt.xlabel('Epochs')\n    plt.legend(loc='upper right')\n\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:46.206244Z","iopub.execute_input":"2023-11-10T23:08:46.207282Z","iopub.status.idle":"2023-11-10T23:08:46.217032Z","shell.execute_reply.started":"2023-11-10T23:08:46.207244Z","shell.execute_reply":"2023-11-10T23:08:46.215972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_columns(data, id_map):\n    sm_name_to_smiles = data.set_index('sm_name')['SMILES'].to_dict()\n    sm_lincs_id = data.set_index('sm_name')[\"sm_lincs_id\"].to_dict()\n\n    id_map['SMILES'] = id_map['sm_name'].map(sm_name_to_smiles)\n    id_map['sm_lincs_id'] = id_map['sm_name'].map(sm_lincs_id)\n\n    return id_map","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:46.218128Z","iopub.execute_input":"2023-11-10T23:08:46.218454Z","iopub.status.idle":"2023-11-10T23:08:46.239298Z","shell.execute_reply.started":"2023-11-10T23:08:46.218421Z","shell.execute_reply":"2023-11-10T23:08:46.238222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_absolute_error\n\ndef calculate_mae_and_mrrmse(model, data, y_true, scaler=None):\n    \"\"\"\n    Calculate Mean Absolute Error (MAE) and Mean Rowwise Root Mean Squared Error (MRRMSE).\n\n    Parameters:\n    - model: The trained  model.\n    - data: The input data for prediction.\n    - y_true: The true target values.\n    - scaler: The scaler used for data normalization.\n\n    Returns:\n    - None\n    \"\"\"\n    # Predict using the model\n    y_pred_original = model.predict(data, batch_size=1)\n\n    if scaler:\n        \n        y_pred = scaler.inverse_transform(y_pred_original)\n        y_true = scaler.inverse_transform(y_true)\n        # Calculate Mean Absolute Error (MAE)\n        mae = mean_absolute_error(y_true , y_pred)\n\n        # Calculate Mean Rowwise Root Mean Squared Error (MRRMSE)\n        rowwise_rmse = np.sqrt(np.mean(np.square(y_true - y_pred), axis=1))\n        mrrmse_score = np.mean(rowwise_rmse)\n    else:\n       # Calculate Mean Absolute Error (MAE)\n        mae = mean_absolute_error(y_true , y_pred_original)\n\n        # Calculate Mean Rowwise Root Mean Squared Error (MRRMSE)\n        rowwise_rmse = np.sqrt(np.mean(np.square(y_true - y_pred_original), axis=1))\n        mrrmse_score = np.mean(rowwise_rmse)\n    # Print the results\n    print(f\"Mean Absolute Error (MAE): {mae}\")\n    print(f\"Mean Rowwise Root Mean Squared Error (MRRMSE): {mrrmse_score}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:46.240494Z","iopub.execute_input":"2023-11-10T23:08:46.241142Z","iopub.status.idle":"2023-11-10T23:08:46.252303Z","shell.execute_reply.started":"2023-11-10T23:08:46.241102Z","shell.execute_reply":"2023-11-10T23:08:46.251360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold\n\ndef k_fold_valid(model, features_list=[],full_labels=None , num_folds=7, random_state=RNDST1, scaler=None):\n    # Initialize lists to store the model's performance scores\n        mae_scores = []\n        mrrmse_scores = []\n\n        # Initialize the KFold object\n        kf = KFold(n_splits=num_folds, shuffle=True, random_state=random_state)\n\n        # Loop through the K folds\n        for train_index, val_index in kf.split(full_labels):\n        # Convert indices to integers and split the data\n            train_index = train_index.astype(int)\n            val_index = val_index.astype(int)\n\n            X_train_list = [features[train_index] for features in features_list]\n            X_val_list = [features[val_index] for features in features_list]\n            y_train_ = full_labels[train_index]\n            y_val_ = full_labels[val_index]\n\n            # Train your model on X_train_list and y_train_\n            # Example: model.fit(X_train_list, y_train_, epochs=20, verbose=0)\n\n            # Make predictions on the validation set\n            y_pred = model.predict(X_val_list, batch_size=1)\n\n            if scaler:\n                # Inverse transform predictions and true labels\n                y_pred = scaler.inverse_transform(y_pred)\n                y_val_ = scaler.inverse_transform(y_val_)\n\n        # Calculate the Mean Absolute Error (MAE)\n            mae = mean_absolute_error(y_val_, y_pred)\n            mae_scores.append(mae)\n\n        # Calculate the Mean Rowwise Root Mean Squared Error (MRRMSE)\n            rowwise_rmse = np.sqrt(np.mean(np.square(y_val_ - y_pred), axis=1))\n            mrrmse_score = np.mean(rowwise_rmse)\n            mrrmse_scores.append(mrrmse_score)\n        # Calculate the mean of MAE and MRRMSE scores\n            mean_mae = np.mean(mae_scores)\n            mean_mrrmse = np.mean(mrrmse_scores)\n\n        # Usage example:\n        print(f'Average MAE across {num_folds} folds: {mean_mae:.4f} ')\n        print(f'Average MRRMSE across {num_folds} folds: {mean_mrrmse:.4f}')","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:46.253694Z","iopub.execute_input":"2023-11-10T23:08:46.254566Z","iopub.status.idle":"2023-11-10T23:08:46.269174Z","shell.execute_reply.started":"2023-11-10T23:08:46.254414Z","shell.execute_reply":"2023-11-10T23:08:46.268136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing data ","metadata":{"execution":{"iopub.status.busy":"2023-10-10T09:17:28.606984Z","iopub.execute_input":"2023-10-10T09:17:28.607326Z","iopub.status.idle":"2023-10-10T09:17:28.611868Z","shell.execute_reply.started":"2023-10-10T09:17:28.607301Z","shell.execute_reply":"2023-10-10T09:17:28.610695Z"}}},{"cell_type":"code","source":"# Shuffle data because we use full features for final training \ndata = data.sample(frac=1.0, random_state=RNDST2)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:46.270393Z","iopub.execute_input":"2023-11-10T23:08:46.271280Z","iopub.status.idle":"2023-11-10T23:08:46.326687Z","shell.execute_reply.started":"2023-11-10T23:08:46.271254Z","shell.execute_reply":"2023-11-10T23:08:46.325410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_columns(data, id_map)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:46.328222Z","iopub.execute_input":"2023-11-10T23:08:46.328681Z","iopub.status.idle":"2023-11-10T23:08:46.426189Z","shell.execute_reply.started":"2023-11-10T23:08:46.328642Z","shell.execute_reply":"2023-11-10T23:08:46.424917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_type_feature = pd.DataFrame(data[\"cell_type\"], columns=[\"cell_type\"])\nsmiles_feature = pd.DataFrame(data[\"SMILES\"], columns=[\"SMILES\"])\nlabels = data.drop([\"cell_type\",\"sm_name\",\"sm_lincs_id\",\"SMILES\",\"control\"], axis=1)\n\n# for test\ntest_feature_smiles = pd.DataFrame(id_map[\"SMILES\"], columns=[\"SMILES\"])\ntest_feature_cell_type = pd.DataFrame(id_map[\"cell_type\"], columns=[\"cell_type\"])","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:46.431000Z","iopub.execute_input":"2023-11-10T23:08:46.432049Z","iopub.status.idle":"2023-11-10T23:08:46.479351Z","shell.execute_reply.started":"2023-11-10T23:08:46.432010Z","shell.execute_reply":"2023-11-10T23:08:46.478005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_type_feature.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:46.481017Z","iopub.execute_input":"2023-11-10T23:08:46.481341Z","iopub.status.idle":"2023-11-10T23:08:46.499478Z","shell.execute_reply.started":"2023-11-10T23:08:46.481312Z","shell.execute_reply":"2023-11-10T23:08:46.498201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"smiles_feature.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:46.500766Z","iopub.execute_input":"2023-11-10T23:08:46.501177Z","iopub.status.idle":"2023-11-10T23:08:46.514776Z","shell.execute_reply.started":"2023-11-10T23:08:46.501138Z","shell.execute_reply":"2023-11-10T23:08:46.513624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_feature_cell_type.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:46.516530Z","iopub.execute_input":"2023-11-10T23:08:46.516906Z","iopub.status.idle":"2023-11-10T23:08:46.534735Z","shell.execute_reply.started":"2023-11-10T23:08:46.516873Z","shell.execute_reply":"2023-11-10T23:08:46.528724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_feature_smiles.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:46.536227Z","iopub.execute_input":"2023-11-10T23:08:46.536521Z","iopub.status.idle":"2023-11-10T23:08:46.545526Z","shell.execute_reply.started":"2023-11-10T23:08:46.536496Z","shell.execute_reply":"2023-11-10T23:08:46.544330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:46.546911Z","iopub.execute_input":"2023-11-10T23:08:46.547339Z","iopub.status.idle":"2023-11-10T23:08:46.586077Z","shell.execute_reply.started":"2023-11-10T23:08:46.547300Z","shell.execute_reply":"2023-11-10T23:08:46.585004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Normalize labels\nfrom sklearn.preprocessing import StandardScaler\n\nscaler = StandardScaler()\nnorm_label = scaler.fit_transform(labels)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:46.587407Z","iopub.execute_input":"2023-11-10T23:08:46.588036Z","iopub.status.idle":"2023-11-10T23:08:47.137681Z","shell.execute_reply.started":"2023-11-10T23:08:46.587998Z","shell.execute_reply":"2023-11-10T23:08:47.136781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Onehotencode\nfrom sklearn.preprocessing import OneHotEncoder\n\nencoder = OneHotEncoder()\none_hot_celltype = encoder.fit_transform(cell_type_feature)\none_hot_test_cell_type = encoder.transform(test_feature_cell_type)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.138971Z","iopub.execute_input":"2023-11-10T23:08:47.139305Z","iopub.status.idle":"2023-11-10T23:08:47.147239Z","shell.execute_reply.started":"2023-11-10T23:08:47.139276Z","shell.execute_reply":"2023-11-10T23:08:47.146147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain_cell_type, temp_cell_type, train_labels, temp_labels = train_test_split(one_hot_celltype.toarray(),\n                                                                            norm_label,\n                                                                            test_size=0.3, random_state=RNDST2)\n\nval_cell_type, test_cell_type, val_labels , test_labels = train_test_split(temp_cell_type, temp_cell_type, test_size=0.6, random_state=RNDST2)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.148384Z","iopub.execute_input":"2023-11-10T23:08:47.148663Z","iopub.status.idle":"2023-11-10T23:08:47.275727Z","shell.execute_reply.started":"2023-11-10T23:08:47.148639Z","shell.execute_reply":"2023-11-10T23:08:47.274611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_smiles, temp_smiles, train_labels, temp_labels = train_test_split(smiles_feature[\"SMILES\"].to_numpy(),\n                                                                            norm_label,\n                                                                            test_size=0.3, random_state=RNDST2)\n\nval_smiles, test_smiles, val_labels , test_labels = train_test_split(temp_smiles, temp_labels, test_size=0.6, random_state=RNDST2)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.277175Z","iopub.execute_input":"2023-11-10T23:08:47.277585Z","iopub.status.idle":"2023-11-10T23:08:47.405120Z","shell.execute_reply.started":"2023-11-10T23:08:47.277546Z","shell.execute_reply":"2023-11-10T23:08:47.404198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_smiles = smiles_feature[\"SMILES\"].values \nfull_cell_type = one_hot_celltype.toarray()\nfull_labels = norm_label","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.406146Z","iopub.execute_input":"2023-11-10T23:08:47.406406Z","iopub.status.idle":"2023-11-10T23:08:47.411658Z","shell.execute_reply.started":"2023-11-10T23:08:47.406384Z","shell.execute_reply":"2023-11-10T23:08:47.410794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_smiles = test_feature_smiles[\"SMILES\"].values\nfinal_cell_type = one_hot_test_cell_type.toarray()","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.413062Z","iopub.execute_input":"2023-11-10T23:08:47.413413Z","iopub.status.idle":"2023-11-10T23:08:47.423195Z","shell.execute_reply.started":"2023-11-10T23:08:47.413350Z","shell.execute_reply":"2023-11-10T23:08:47.422410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_smiles[:3]","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.424455Z","iopub.execute_input":"2023-11-10T23:08:47.425111Z","iopub.status.idle":"2023-11-10T23:08:47.439456Z","shell.execute_reply.started":"2023-11-10T23:08:47.425083Z","shell.execute_reply":"2023-11-10T23:08:47.438496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_cell_type","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.440605Z","iopub.execute_input":"2023-11-10T23:08:47.440970Z","iopub.status.idle":"2023-11-10T23:08:47.451007Z","shell.execute_reply.started":"2023-11-10T23:08:47.440899Z","shell.execute_reply":"2023-11-10T23:08:47.450076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_cell_type","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.452448Z","iopub.execute_input":"2023-11-10T23:08:47.453232Z","iopub.status.idle":"2023-11-10T23:08:47.463111Z","shell.execute_reply.started":"2023-11-10T23:08:47.453200Z","shell.execute_reply":"2023-11-10T23:08:47.462207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_smiles[:3]","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.464270Z","iopub.execute_input":"2023-11-10T23:08:47.465009Z","iopub.status.idle":"2023-11-10T23:08:47.475366Z","shell.execute_reply.started":"2023-11-10T23:08:47.464974Z","shell.execute_reply":"2023-11-10T23:08:47.474327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_smiles[:10]","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.476770Z","iopub.execute_input":"2023-11-10T23:08:47.477320Z","iopub.status.idle":"2023-11-10T23:08:47.488443Z","shell.execute_reply.started":"2023-11-10T23:08:47.477285Z","shell.execute_reply":"2023-11-10T23:08:47.487478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the lengths\nlen(train_smiles), len(train_smiles), len(val_smiles), len(val_smiles), len(test_smiles), len(test_smiles)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.489636Z","iopub.execute_input":"2023-11-10T23:08:47.490339Z","iopub.status.idle":"2023-11-10T23:08:47.499768Z","shell.execute_reply.started":"2023-11-10T23:08:47.490306Z","shell.execute_reply":"2023-11-10T23:08:47.498974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split chars \nWe will split the `SMILES` into chars and then vectorize.","metadata":{}},{"cell_type":"code","source":"# Function to split sentences into characters\ndef split_chars(text):\n    return \" \".join(list(text))","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.507046Z","iopub.execute_input":"2023-11-10T23:08:47.507315Z","iopub.status.idle":"2023-11-10T23:08:47.511805Z","shell.execute_reply.started":"2023-11-10T23:08:47.507291Z","shell.execute_reply":"2023-11-10T23:08:47.510786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = \"CC(C)c1cc(C(=O)N2Cc3ccc(CN4CCN(C)CC4)cc3C2)c(O)cc1O\"\nsplit_chars(sample)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.512887Z","iopub.execute_input":"2023-11-10T23:08:47.513199Z","iopub.status.idle":"2023-11-10T23:08:47.525177Z","shell.execute_reply.started":"2023-11-10T23:08:47.513176Z","shell.execute_reply":"2023-11-10T23:08:47.524300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Split the smiles","metadata":{}},{"cell_type":"code","source":"train_char_smiles = [split_chars(feature) for feature in train_smiles]\nval_char_smiles = [split_chars(feature) for feature in val_smiles]\ntest_char_smiles = [split_chars(feature) for feature in test_smiles]","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.526282Z","iopub.execute_input":"2023-11-10T23:08:47.526572Z","iopub.status.idle":"2023-11-10T23:08:47.539857Z","shell.execute_reply.started":"2023-11-10T23:08:47.526549Z","shell.execute_reply":"2023-11-10T23:08:47.538889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For final training\nfull_train_smiles = [split_chars(feature) for feature in full_smiles]","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.541038Z","iopub.execute_input":"2023-11-10T23:08:47.541303Z","iopub.status.idle":"2023-11-10T23:08:47.552984Z","shell.execute_reply.started":"2023-11-10T23:08:47.541280Z","shell.execute_reply":"2023-11-10T23:08:47.552205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For prediction\nfinal_test_smiles = [split_chars(feature) for feature in final_test_smiles]","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.554145Z","iopub.execute_input":"2023-11-10T23:08:47.554413Z","iopub.status.idle":"2023-11-10T23:08:47.566952Z","shell.execute_reply.started":"2023-11-10T23:08:47.554390Z","shell.execute_reply":"2023-11-10T23:08:47.566191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_train_smiles[:3] ","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.568155Z","iopub.execute_input":"2023-11-10T23:08:47.568450Z","iopub.status.idle":"2023-11-10T23:08:47.580783Z","shell.execute_reply.started":"2023-11-10T23:08:47.568425Z","shell.execute_reply":"2023-11-10T23:08:47.579777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_smiles[:3]","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.581996Z","iopub.execute_input":"2023-11-10T23:08:47.582673Z","iopub.status.idle":"2023-11-10T23:08:47.592424Z","shell.execute_reply.started":"2023-11-10T23:08:47.582639Z","shell.execute_reply":"2023-11-10T23:08:47.591608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_char_smiles[:3]","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.593556Z","iopub.execute_input":"2023-11-10T23:08:47.593847Z","iopub.status.idle":"2023-11-10T23:08:47.604851Z","shell.execute_reply.started":"2023-11-10T23:08:47.593811Z","shell.execute_reply":"2023-11-10T23:08:47.603972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Char vectorizer","metadata":{}},{"cell_type":"code","source":"# What's the average character length?\nchar_len_smiles = [len(feature) for feature in full_smiles]\nmean_char_smiles = np.mean(char_len_smiles)\nmean_char_smiles","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.605857Z","iopub.execute_input":"2023-11-10T23:08:47.606138Z","iopub.status.idle":"2023-11-10T23:08:47.617316Z","shell.execute_reply.started":"2023-11-10T23:08:47.606115Z","shell.execute_reply":"2023-11-10T23:08:47.616213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  Find what character length covers 95% of sequences\noutput_seq_char_len = int(np.percentile(char_len_smiles, 95))\noutput_seq_char_len","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.618477Z","iopub.execute_input":"2023-11-10T23:08:47.619293Z","iopub.status.idle":"2023-11-10T23:08:47.631011Z","shell.execute_reply.started":"2023-11-10T23:08:47.619266Z","shell.execute_reply":"2023-11-10T23:08:47.630099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the distribution of our sequences at character-level\nplt.hist(char_len_smiles, bins=7);","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.631996Z","iopub.execute_input":"2023-11-10T23:08:47.632244Z","iopub.status.idle":"2023-11-10T23:08:47.921746Z","shell.execute_reply.started":"2023-11-10T23:08:47.632222Z","shell.execute_reply":"2023-11-10T23:08:47.920763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Get all the unique characters in smiles","metadata":{"execution":{"iopub.status.busy":"2023-10-10T09:36:47.578256Z","iopub.execute_input":"2023-10-10T09:36:47.578652Z","iopub.status.idle":"2023-10-10T09:36:47.585849Z","shell.execute_reply.started":"2023-10-10T09:36:47.578623Z","shell.execute_reply":"2023-10-10T09:36:47.5846Z"}}},{"cell_type":"code","source":"unique_characters = []\n\n# Iterate over the \"features\" and extract unique characters\nfor feature in full_train_smiles:\n    unique_characters.extend(set(feature))\n\n# Remove duplicates by converting the list to a set and then back to a list\nunique_characters = list(set(unique_characters))\n\n# Print the list of unique characters\nprint(unique_characters)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.922804Z","iopub.execute_input":"2023-11-10T23:08:47.923096Z","iopub.status.idle":"2023-11-10T23:08:47.930817Z","shell.execute_reply.started":"2023-11-10T23:08:47.923072Z","shell.execute_reply":"2023-11-10T23:08:47.929882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(unique_characters)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.932281Z","iopub.execute_input":"2023-11-10T23:08:47.932664Z","iopub.status.idle":"2023-11-10T23:08:47.943659Z","shell.execute_reply.started":"2023-11-10T23:08:47.932628Z","shell.execute_reply":"2023-11-10T23:08:47.942808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.layers.experimental.preprocessing import TextVectorization\n\n# Create char-level token vectorizer instance\nNUM_CHAR_TOKENS = len(unique_characters) + 2 \nchar_vectorizer_smiles = TextVectorization(max_tokens=NUM_CHAR_TOKENS,\n                                    output_sequence_length=output_seq_char_len,\n                                    standardize=None,\n                                    split=\"character\",\n                                    name=\"char_vectorizer_SMILES\")\n\n# Adapt character vectorizer to training characters\nchar_vectorizer_smiles.adapt(train_char_smiles)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:47.944615Z","iopub.execute_input":"2023-11-10T23:08:47.944861Z","iopub.status.idle":"2023-11-10T23:08:51.550470Z","shell.execute_reply.started":"2023-11-10T23:08:47.944838Z","shell.execute_reply":"2023-11-10T23:08:51.549002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the config of our char vectorizer\nchar_vectorizer_smiles.get_config()","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:51.552166Z","iopub.execute_input":"2023-11-10T23:08:51.552509Z","iopub.status.idle":"2023-11-10T23:08:51.559268Z","shell.execute_reply.started":"2023-11-10T23:08:51.552479Z","shell.execute_reply":"2023-11-10T23:08:51.558362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\n\n# Test out character vectorizer\nrandom_train_feature = random.choice(train_char_smiles)\nprint(f\"Charified text:\\n{random_train_feature}\")\nprint(f\"\\nLength of feature: {len(random_train_feature.split())}\")\nvectorized_feature = char_vectorizer_smiles([random_train_feature])\nprint(f\"\\nVectorized feature:\\n{vectorized_feature}\")\nprint(f\"\\nLength of vectorized feature: {len(vectorized_feature[0])}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:51.560536Z","iopub.execute_input":"2023-11-10T23:08:51.560910Z","iopub.status.idle":"2023-11-10T23:08:51.610642Z","shell.execute_reply.started":"2023-11-10T23:08:51.560877Z","shell.execute_reply":"2023-11-10T23:08:51.609552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"You'll notice sequences with a length shorter than 193 (output_seq_char_length) get padded with zeros on the end, this ensures all sequences passed to our model are the same length.","metadata":{}},{"cell_type":"code","source":"# Check character vocabulary characteristics\nchar_vocab = char_vectorizer_smiles.get_vocabulary()\nprint(f\"Number of different characters in character vocab: {len(char_vocab)}\")\nprint(f\"5 most common characters: {char_vocab[:5]}\")\nprint(f\"5 least common characters: {char_vocab[-5:]}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:51.611873Z","iopub.execute_input":"2023-11-10T23:08:51.613586Z","iopub.status.idle":"2023-11-10T23:08:51.620702Z","shell.execute_reply.started":"2023-11-10T23:08:51.613558Z","shell.execute_reply":"2023-11-10T23:08:51.619744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Char embedding","metadata":{}},{"cell_type":"code","source":"# Create char embedding layer\nchar_embedding = tf.keras.layers.Embedding(input_dim=NUM_CHAR_TOKENS, # number of different characters\n                              output_dim=16, # embedding dimension of each character \n                              name=\"char_embedding_SMILES\")","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:51.621887Z","iopub.execute_input":"2023-11-10T23:08:51.622218Z","iopub.status.idle":"2023-11-10T23:08:51.631097Z","shell.execute_reply.started":"2023-11-10T23:08:51.622185Z","shell.execute_reply":"2023-11-10T23:08:51.630256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test out character embedding layer\nprint(f\"Charified text (before vectorization and embedding):\\n{random_train_feature}\\n\")\nchar_embed_example = char_embedding(char_vectorizer_smiles([random_train_feature]))\nprint(f\"Embedded chars (after vectorization and embedding):\\n{char_embed_example}\\n\")\nprint(f\"Character embedding shape: {char_embed_example.shape}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:51.632248Z","iopub.execute_input":"2023-11-10T23:08:51.632844Z","iopub.status.idle":"2023-11-10T23:08:51.677517Z","shell.execute_reply.started":"2023-11-10T23:08:51.632809Z","shell.execute_reply":"2023-11-10T23:08:51.676609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert list to array\ntrain_chars = np.array(train_char_smiles)\nval_chars = np.array(val_char_smiles)\ntest_chars = np.array(test_char_smiles)\nfull_train_chars = np.array(full_train_smiles)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:51.678662Z","iopub.execute_input":"2023-11-10T23:08:51.678986Z","iopub.status.idle":"2023-11-10T23:08:51.686023Z","shell.execute_reply.started":"2023-11-10T23:08:51.678952Z","shell.execute_reply":"2023-11-10T23:08:51.684950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Building models","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.utils import plot_model\nfrom tensorflow.keras import Model\nfrom tensorflow.keras.layers import LSTM, Dense, concatenate, Conv1D, GlobalMaxPooling1D, Input, Flatten, GaussianNoise, GlobalAveragePooling1D","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:51.687390Z","iopub.execute_input":"2023-11-10T23:08:51.687726Z","iopub.status.idle":"2023-11-10T23:08:51.696267Z","shell.execute_reply.started":"2023-11-10T23:08:51.687699Z","shell.execute_reply":"2023-11-10T23:08:51.695480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 0 : Conv1D","metadata":{}},{"cell_type":"code","source":"tf.random.set_seed(RNDST2)\n\n# Define the SMILES input\nsmiles_input = Input(shape=(1,), dtype=\"string\", name=\"smiles_input\")\nchar_vectors = char_vectorizer_smiles(smiles_input)\nchar_embeddings = char_embedding(char_vectors)\nconv_layer = Conv1D(64, kernel_size=5, padding=\"same\", activation=\"relu\")(char_embeddings)\nsmiles_output = GlobalMaxPooling1D()(conv_layer)\n\n# Define the cell type input\ncell_type_input = Input(shape=(6,), name=\"cell_type_input\")\ncell_type_output = Dense(32, activation=\"relu\")(cell_type_input)\n\n# Concatenate the two inputs\nconcatenated = concatenate([smiles_output, cell_type_output], name=\"concatenate_layer\")\n\n# Define the neural network architecture\nx = Dense(128, activation=\"relu\")(concatenated)\noutput = Dense(18211, activation=\"linear\")(x)\n\n# Create the model\nmodel_0 = Model(inputs=[smiles_input, cell_type_input], outputs=output, name=\"model_0\")\n\n# Compile the model\nmodel_0.compile(loss=\"mae\", optimizer=tf.keras.optimizers.Adam(), metrics=[\"mae\"])","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:51.697331Z","iopub.execute_input":"2023-11-10T23:08:51.697625Z","iopub.status.idle":"2023-11-10T23:08:51.839859Z","shell.execute_reply.started":"2023-11-10T23:08:51.697591Z","shell.execute_reply":"2023-11-10T23:08:51.838828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model(model_0, show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:51.841169Z","iopub.execute_input":"2023-11-10T23:08:51.841445Z","iopub.status.idle":"2023-11-10T23:08:52.118981Z","shell.execute_reply.started":"2023-11-10T23:08:51.841421Z","shell.execute_reply":"2023-11-10T23:08:52.117891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_0_history = model_0.fit(\n    x = [train_chars, train_cell_type],   \n    y =train_labels,\n        epochs=60,\n        verbose=0,\n        validation_data=([val_chars, val_cell_type], val_labels),\n        callbacks=[create_model_checkpoint(\"model_0\")])","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:08:52.120303Z","iopub.execute_input":"2023-11-10T23:08:52.120605Z","iopub.status.idle":"2023-11-10T23:09:12.627875Z","shell.execute_reply.started":"2023-11-10T23:08:52.120579Z","shell.execute_reply":"2023-11-10T23:09:12.626381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_0.load_weights(\"model_0\")\nprint(\"Scores on test data:\\n\")\ncalculate_mae_and_mrrmse(model=model_0, data=[test_chars, test_cell_type], y_true=test_labels, scaler=scaler)\nprint(\"\\nScores on full data:\\n\")\ncalculate_mae_and_mrrmse(model=model_0, data=[full_train_chars, full_cell_type], y_true=full_labels, scaler=scaler)\nprint(\"\\nPlot training and validation curves:\\n\")\nplot_training_history(model_0_history , metrics=[\"mae\"])","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:09:12.630451Z","iopub.execute_input":"2023-11-10T23:09:12.630730Z","iopub.status.idle":"2023-11-10T23:09:16.056212Z","shell.execute_reply.started":"2023-11-10T23:09:12.630706Z","shell.execute_reply":"2023-11-10T23:09:16.055346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"k_fold_valid(model=model_0, features_list=[full_train_chars, np.array(full_cell_type)], full_labels=full_labels, num_folds=10, random_state=RNDST1, scaler=scaler)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:09:16.057543Z","iopub.execute_input":"2023-11-10T23:09:16.057842Z","iopub.status.idle":"2023-11-10T23:09:19.156416Z","shell.execute_reply.started":"2023-11-10T23:09:16.057815Z","shell.execute_reply":"2023-11-10T23:09:19.155385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model_1: Lstm","metadata":{}},{"cell_type":"code","source":"# Embed the SMILES strings\ninputs_smiles = Input(shape=(1,), dtype=\"string\", name=\"smiles_input\")\nchar_vectors = char_vectorizer_smiles(inputs_smiles)\nchar_embeddings = char_embedding(char_vectors)\nx = LSTM(16, return_sequences=True)(char_embeddings)\nx = GlobalAveragePooling1D()(x)\noutput_smiles = Dense(32, activation=\"relu\")(x)\n\ncell_type_input = tf.keras.Input(shape=(6,), dtype=tf.float32)\nx = Dense(16, activation=\"relu\")(cell_type_input)\n\n# Concatenate the SMILES embedding and the cell type one-hot encoding\ny = concatenate([output_smiles, x])\ny = Dense(128, activation='relu')(y)\noutput_layer = Dense(18211, activation='linear')(y)\n\nmodel_1 = Model(inputs=[inputs_smiles, cell_type_input], outputs=output_layer)\n\nmodel_1.compile(loss=\"mae\",optimizer=tf.keras.optimizers.Adam(learning_rate= 0.001),metrics=[\"mae\"])","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:09:19.157662Z","iopub.execute_input":"2023-11-10T23:09:19.157966Z","iopub.status.idle":"2023-11-10T23:09:19.517974Z","shell.execute_reply.started":"2023-11-10T23:09:19.157940Z","shell.execute_reply":"2023-11-10T23:09:19.516903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model(model_1, show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:09:19.519279Z","iopub.execute_input":"2023-11-10T23:09:19.519647Z","iopub.status.idle":"2023-11-10T23:09:19.636604Z","shell.execute_reply.started":"2023-11-10T23:09:19.519612Z","shell.execute_reply":"2023-11-10T23:09:19.635682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_1_history = model_1.fit(x =[train_chars, train_cell_type],\n                              y=train_labels,\n                              epochs=100,\n                              verbose=0,\n                              validation_data=([val_chars, val_cell_type], val_labels),\n                              callbacks=[create_model_checkpoint(\"model_1\")])","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:09:19.637839Z","iopub.execute_input":"2023-11-10T23:09:19.638157Z","iopub.status.idle":"2023-11-10T23:09:41.762597Z","shell.execute_reply.started":"2023-11-10T23:09:19.638132Z","shell.execute_reply":"2023-11-10T23:09:41.761531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_1.load_weights(\"model_1\")\nprint(\"Scores on test data:\\n\")\ncalculate_mae_and_mrrmse(model=model_1, data=[test_smiles, test_cell_type], y_true=test_labels, scaler=scaler)\nprint(\"\\nScores on full data:\\n\")\ncalculate_mae_and_mrrmse(model=model_1, data=[full_smiles, full_cell_type], y_true=full_labels, scaler=scaler)\nprint(\"\\nPlot training and validation curves:\\n\")\nplot_training_history(model_1_history , metrics=[\"mae\"])","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:09:41.764603Z","iopub.execute_input":"2023-11-10T23:09:41.765023Z","iopub.status.idle":"2023-11-10T23:09:46.167277Z","shell.execute_reply.started":"2023-11-10T23:09:41.764987Z","shell.execute_reply":"2023-11-10T23:09:46.166270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"k_fold_valid(model=model_1, features_list=[full_train_chars, np.array(full_cell_type)], full_labels=full_labels, num_folds=10, random_state=RNDST1, scaler=scaler)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:09:46.168479Z","iopub.execute_input":"2023-11-10T23:09:46.168804Z","iopub.status.idle":"2023-11-10T23:09:50.003624Z","shell.execute_reply.started":"2023-11-10T23:09:46.168775Z","shell.execute_reply":"2023-11-10T23:09:50.002522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 2: Custom neural net ","metadata":{}},{"cell_type":"code","source":"# Define the inputs\nsmiles_input = Input(shape=(1,), dtype=\"string\", name=\"smiles_input\")\ncelltype_input = Input(shape=(6,), dtype=tf.float32, name=\"cell_type_input\")\n\n# Encode SMILES and cell type data\nchar_vectors = char_vectorizer_smiles(smiles_input)\nchar_embeddings = char_embedding(char_vectors)\nembed_flatten = Flatten()(char_embeddings)\nconcatenated_data = concatenate([embed_flatten, celltype_input], name=\"concatenate\")\nx = GaussianNoise(0.09)(concatenated_data)\nx = Dense(512, activation='elu')(x)\nx = Dense(256, activation='elu')(x)\nx = Dense(128, activation='elu')(x)\nx = Dense(256, activation='elu')(x)\nx = Dense(512, activation='elu')(x)\n\n# Output layer\noutput = Dense(18211, activation='linear')(x)\n\n# Create and compile the model\nmodel_2 = Model(inputs=[smiles_input, celltype_input], outputs=output)\n\nmodel_2.compile(loss=\"mae\", optimizer=tf.keras.optimizers.Adam(learning_rate=0.00098), metrics=[\"mae\"])","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:09:50.004893Z","iopub.execute_input":"2023-11-10T23:09:50.005209Z","iopub.status.idle":"2023-11-10T23:09:50.144660Z","shell.execute_reply.started":"2023-11-10T23:09:50.005183Z","shell.execute_reply":"2023-11-10T23:09:50.143887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model(model_2, show_shapes=True, show_layer_activations=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:09:50.145832Z","iopub.execute_input":"2023-11-10T23:09:50.146206Z","iopub.status.idle":"2023-11-10T23:09:50.281421Z","shell.execute_reply.started":"2023-11-10T23:09:50.146172Z","shell.execute_reply":"2023-11-10T23:09:50.280494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_2_history = model_2.fit(\n    x=[train_chars, train_cell_type],\n    y=train_labels,\n    epochs=40,\n    verbose=0,\n    validation_data=([val_chars, val_cell_type], val_labels),\n    callbacks=[create_model_checkpoint(\"model_2\")])","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:09:50.282887Z","iopub.execute_input":"2023-11-10T23:09:50.283232Z","iopub.status.idle":"2023-11-10T23:10:01.308826Z","shell.execute_reply.started":"2023-11-10T23:09:50.283206Z","shell.execute_reply":"2023-11-10T23:10:01.307827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_2.load_weights(\"model_2\")\nprint(\"Scores on test:\\n\")\ncalculate_mae_and_mrrmse(model=model_2, data=[test_chars, test_cell_type], y_true=test_labels, scaler=scaler)\nprint(\"\\nScores on full data:\\n\")\ncalculate_mae_and_mrrmse(model=model_2, data=[full_train_chars, full_cell_type], y_true=full_labels, scaler=scaler)\nprint(\"\\nPlot training and validation curves:\\n\")\nplot_training_history(model_2_history, metrics=[\"mae\"])","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:10:01.310417Z","iopub.execute_input":"2023-11-10T23:10:01.310783Z","iopub.status.idle":"2023-11-10T23:10:04.801318Z","shell.execute_reply.started":"2023-11-10T23:10:01.310753Z","shell.execute_reply":"2023-11-10T23:10:04.800281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make Prediction","metadata":{}},{"cell_type":"markdown","source":"We'll train `Model_2` with full train data for final time to make prediction. ","metadata":{}},{"cell_type":"code","source":"smiles_input = Input(shape=(1,), dtype=\"string\", name=\"smiles_input\")\ncelltype_input = Input(shape=(6,), dtype=tf.float32, name=\"cell_type_input\")\n\nchar_vectors = char_vectorizer_smiles(smiles_input)\nchar_embeddings = char_embedding(char_vectors)\nembed_flatten = Flatten()(char_embeddings)\nconcatenated_data = concatenate([embed_flatten, celltype_input], name=\"concatenate\")\nx = GaussianNoise(0.09)(concatenated_data)\nx = Dense(512, activation='elu')(x)\nx = Dense(256, activation='elu')(x)\nx = Dense(128, activation='elu')(x)\nx = Dense(256, activation='elu')(x)\nx = Dense(512, activation='elu')(x)\noutput = Dense(18211, activation='linear')(x)\n\nmodel = Model(inputs=[smiles_input, celltype_input], outputs=output)\nmodel.compile(loss=\"mae\", optimizer=tf.keras.optimizers.Adam(learning_rate=0.00098), metrics=[\"mae\"])\n\nhistory = model.fit(x=[full_train_chars, np.array(full_cell_type)], y=full_labels,\n                              epochs=100,\n                              verbose=0,\n                              validation_data=([test_chars, test_cell_type], test_labels))","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:10:04.802598Z","iopub.execute_input":"2023-11-10T23:10:04.802895Z","iopub.status.idle":"2023-11-10T23:10:24.229567Z","shell.execute_reply.started":"2023-11-10T23:10:04.802869Z","shell.execute_reply":"2023-11-10T23:10:24.228615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Scores on test:\\n\")\ncalculate_mae_and_mrrmse(model=model, data=[test_chars, test_cell_type], y_true=test_labels, scaler=scaler)\nprint(\"\\nScores on full data:\\n\")\ncalculate_mae_and_mrrmse(model=model, data=[full_train_chars, full_cell_type], y_true=full_labels, scaler=scaler)\nprint(\"\\nPlot training and validation curves:\\n\")\nplot_training_history(history, metrics=[\"mae\"])","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:10:24.230919Z","iopub.execute_input":"2023-11-10T23:10:24.231249Z","iopub.status.idle":"2023-11-10T23:10:27.428509Z","shell.execute_reply.started":"2023-11-10T23:10:24.231222Z","shell.execute_reply":"2023-11-10T23:10:27.427503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_columns = sample_submission.columns\nsample_columns = sample_columns[1:]","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:10:27.429806Z","iopub.execute_input":"2023-11-10T23:10:27.430210Z","iopub.status.idle":"2023-11-10T23:10:27.435226Z","shell.execute_reply.started":"2023-11-10T23:10:27.430177Z","shell.execute_reply":"2023-11-10T23:10:27.434282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = model.predict([np.array(final_test_smiles), final_cell_type], batch_size=1)\npreds = scaler.inverse_transform(preds)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:10:27.436595Z","iopub.execute_input":"2023-11-10T23:10:27.437153Z","iopub.status.idle":"2023-11-10T23:10:28.218315Z","shell.execute_reply.started":"2023-11-10T23:10:27.437117Z","shell.execute_reply":"2023-11-10T23:10:28.217498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame(preds, columns=sample_columns)\ndf.insert(0, 'id', range(255))\ndf.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:10:28.219477Z","iopub.execute_input":"2023-11-10T23:10:28.219822Z","iopub.status.idle":"2023-11-10T23:10:36.246999Z","shell.execute_reply.started":"2023-11-10T23:10:28.219789Z","shell.execute_reply":"2023-11-10T23:10:36.245982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2023-11-10T23:10:36.248314Z","iopub.execute_input":"2023-11-10T23:10:36.248583Z","iopub.status.idle":"2023-11-10T23:10:36.280669Z","shell.execute_reply.started":"2023-11-10T23:10:36.248560Z","shell.execute_reply":"2023-11-10T23:10:36.279812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **Note:** The model is not at all tuned. So try tuning hyperparams and learning rate to get better LB score.  And let me know how it goes. ","metadata":{}},{"cell_type":"markdown","source":"# Improving Model\n* Experiment with different `SMILES` embedding dimensions, observe and find the best.\n* Explore various layer types such as `Attention`, `BatchNormalization` `Bidirectional`, `RNN`...\n* Use different model architecture.\n* Fine-tune model hyperparameters and learning rates for optimization.\n* Test out alternative loss functions to determine their impact on model performance.\n* Evaluate the use of `MinMax` Scaler instead of `StandardScaler` for scaling the labels\n* Try vectorization or embedding for the `cell_type`\n*  Increase the number of features and create a more complex model to capture additional patterns.","metadata":{}}]}