{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59094,"databundleVersionId":7010844,"sourceType":"competition"},{"sourceId":5835808,"sourceType":"datasetVersion","datasetId":3354626}],"dockerImageVersionId":30559,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# NLP Regression  \nThis is just a simple experiment with NLP and to get low MRRMSE(Mean Rowwise Root Mean Sqared Error)\n\nWe're going to use SMILES embedding.\n\nSteps to do above:\n1. Preprocess the data \n2. Make a TextVectorizer and embedding\n3. Build a Model\n4. Visualize, Evaluate and Repeat \n\nLet's go...\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"!pip install -U tensorflow==2.14.0","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:57:28.275951Z","iopub.execute_input":"2023-11-12T23:57:28.2763Z","iopub.status.idle":"2023-11-12T23:58:27.225283Z","shell.execute_reply.started":"2023-11-12T23:57:28.27627Z","shell.execute_reply":"2023-11-12T23:58:27.224362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:58:27.227168Z","iopub.execute_input":"2023-11-12T23:58:27.227453Z","iopub.status.idle":"2023-11-12T23:58:32.527603Z","shell.execute_reply.started":"2023-11-12T23:58:27.227425Z","shell.execute_reply":"2023-11-12T23:58:32.526283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.__version__","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:58:32.529824Z","iopub.execute_input":"2023-11-12T23:58:32.531104Z","iopub.status.idle":"2023-11-12T23:58:32.541757Z","shell.execute_reply.started":"2023-11-12T23:58:32.531054Z","shell.execute_reply":"2023-11-12T23:58:32.540678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read data","metadata":{}},{"cell_type":"code","source":"data = pd.read_parquet(\"/kaggle/input/open-problems-single-cell-perturbations/de_train.parquet\")\nid_map = pd.read_csv(\"/kaggle/input/open-problems-single-cell-perturbations/id_map.csv\")\nsample_submission = pd.read_csv(\"/kaggle/input/open-problems-single-cell-perturbations/sample_submission.csv\")\nRNDST1=271828\nRNDST2=314159","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:01.006957Z","iopub.execute_input":"2023-11-12T23:59:01.007364Z","iopub.status.idle":"2023-11-12T23:59:07.407021Z","shell.execute_reply.started":"2023-11-12T23:59:01.007334Z","shell.execute_reply":"2023-11-12T23:59:07.406069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" **🔑Tip:** Stare at the data for a while to get some insights and ideas to run experiments.","metadata":{}},{"cell_type":"code","source":"data.head(20)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:07.408808Z","iopub.execute_input":"2023-11-12T23:59:07.409123Z","iopub.status.idle":"2023-11-12T23:59:07.450376Z","shell.execute_reply.started":"2023-11-12T23:59:07.409096Z","shell.execute_reply":"2023-11-12T23:59:07.449515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Aux functions ","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.callbacks import ModelCheckpoint\n\ndef create_model_checkpoint(filepath, monitor='val_mae', save_best_only=True,\n                            save_weights_only=True, mode='auto', verbose=0):\n    \"\"\"\n    Create a ModelCheckpoint callback for saving the best model weights during training.\n\n    Args:\n        filepath (str): Filepath to save the best weights.\n        monitor (str): Metric to monitor (e.g., 'val_loss' or 'val_mae').\n        save_best_only (bool): Save only the best weights.\n        save_weights_only (bool): Save only the model's weights, not the entire model.\n        mode (str): One of {'auto', 'min', 'max'}. In 'min' mode, it saves when the monitored metric decreases.\n        verbose (int): Verbosity mode. 0 = silent, 1 = progress bar, 2 = one line per epoch.\n\n    Returns:\n        keras.callbacks.ModelCheckpoint: ModelCheckpoint callback.\n    \"\"\"\n    checkpoint = ModelCheckpoint(\n        filepath=filepath,\n        monitor=monitor,\n        save_best_only=save_best_only,\n        save_weights_only=save_weights_only,\n        mode=mode,\n        verbose=verbose\n    )\n    return checkpoint\n","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:07.451605Z","iopub.execute_input":"2023-11-12T23:59:07.451874Z","iopub.status.idle":"2023-11-12T23:59:07.480965Z","shell.execute_reply.started":"2023-11-12T23:59:07.451849Z","shell.execute_reply":"2023-11-12T23:59:07.480086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_training_history(history, metrics):\n    \"\"\"\n    Plot training history curves for loss and evaluation metrics on the same line.\n\n    Args:\n        history (keras.callbacks.History): Training history object.\n        metrics (list): List of metric names to plot.\n\n    Returns:\n        None\n    \"\"\"\n    loss = history.history['loss']\n    val_loss = history.history['val_loss']\n\n    epochs = range(len(loss))\n\n    plt.figure(figsize=(12, 6))\n\n    # Plot loss\n    plt.subplot(1, 2, 1)\n    plt.plot(epochs, loss, label='Training Loss', color=\"blue\")\n    plt.plot(epochs, val_loss, label='Validation Loss', color=\"red\")\n    plt.title('Loss')\n    plt.xlabel('Epochs')\n    plt.legend()\n\n    # Plot specified evaluation metrics on the same line\n    for metric in metrics:\n        train_metric_name = f'Training {metric.capitalize()}'\n        val_metric_name = f'Validation {metric.capitalize()}'\n        train_metric = history.history[metric]\n        val_metric = history.history['val_' + metric]\n\n        plt.subplot(1, 2, 2)\n        plt.plot(epochs, train_metric, label=train_metric_name, color=\"green\")\n        plt.plot(epochs, val_metric, label=val_metric_name, color=\"orange\")\n\n    plt.title('Metrics')\n    plt.xlabel('Epochs')\n    plt.legend(loc='upper right')\n\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:07.483775Z","iopub.execute_input":"2023-11-12T23:59:07.484148Z","iopub.status.idle":"2023-11-12T23:59:07.494923Z","shell.execute_reply.started":"2023-11-12T23:59:07.484113Z","shell.execute_reply":"2023-11-12T23:59:07.494001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_columns(data, id_map):\n    sm_name_to_smiles = data.set_index('sm_name')['SMILES'].to_dict()\n    sm_lincs_id = data.set_index('sm_name')[\"sm_lincs_id\"].to_dict()\n\n    id_map['SMILES'] = id_map['sm_name'].map(sm_name_to_smiles)\n    id_map['sm_lincs_id'] = id_map['sm_name'].map(sm_lincs_id)\n\n    return id_map","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:07.496293Z","iopub.execute_input":"2023-11-12T23:59:07.496863Z","iopub.status.idle":"2023-11-12T23:59:07.512721Z","shell.execute_reply.started":"2023-11-12T23:59:07.496825Z","shell.execute_reply":"2023-11-12T23:59:07.510793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_absolute_error\n\ndef calculate_mae_and_mrrmse(model, data, y_true, scaler=None):\n    \"\"\"\n    Calculate Mean Absolute Error (MAE) and Mean Rowwise Root Mean Squared Error (MRRMSE).\n\n    Parameters:\n    - model: The trained  model.\n    - data: The input data for prediction.\n    - y_true: The true target values.\n    - scaler: The scaler used for data normalization.\n\n    Returns:\n    - None\n    \"\"\"\n    # Predict using the model\n    y_pred_original = model.predict(data, batch_size=1)\n\n    if scaler:\n        \n        y_pred = scaler.inverse_transform(y_pred_original)\n        y_true = scaler.inverse_transform(y_true)\n        # Calculate Mean Absolute Error (MAE)\n        mae = mean_absolute_error(y_true , y_pred)\n\n        # Calculate Mean Rowwise Root Mean Squared Error (MRRMSE)\n        rowwise_rmse = np.sqrt(np.mean(np.square(y_true - y_pred), axis=1))\n        mrrmse_score = np.mean(rowwise_rmse)\n    else:\n       # Calculate Mean Absolute Error (MAE)\n        mae = mean_absolute_error(y_true , y_pred_original)\n\n        # Calculate Mean Rowwise Root Mean Squared Error (MRRMSE)\n        rowwise_rmse = np.sqrt(np.mean(np.square(y_true - y_pred_original), axis=1))\n        mrrmse_score = np.mean(rowwise_rmse)\n    # Print the results\n    print(f\"Mean Absolute Error (MAE): {mae}\")\n    print(f\"Mean Rowwise Root Mean Squared Error (MRRMSE): {mrrmse_score}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:07.515831Z","iopub.execute_input":"2023-11-12T23:59:07.516243Z","iopub.status.idle":"2023-11-12T23:59:07.855239Z","shell.execute_reply.started":"2023-11-12T23:59:07.516208Z","shell.execute_reply":"2023-11-12T23:59:07.854478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.model_selection import KFold\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import mean_absolute_error\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Dense, Flatten, concatenate, GaussianNoise\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import EarlyStopping\n\ndef build_model_4(input_shape, optimizer=Adam(learning_rate=0.001)):\n    # Define the SMILES input\n    smiles_input = Input(shape=input_shape[0], dtype=\"string\", name=\"smiles_input\")\n    char_vectors = char_vectorizer_smiles(smiles_input)\n    char_embeddings = char_embedding(char_vectors)\n    embed_flatten = Flatten()(char_embeddings)\n\n    # Define the cell type input\n    cell_type_input = Input(shape=input_shape[1], dtype=tf.float32, name=\"cell_type_input\")\n\n    # Concatenate SMILES embedding and cell type data\n    concatenated_data = concatenate([embed_flatten, cell_type_input], name=\"concatenate\")\n\n    # Apply Gaussian Noise\n    x = GaussianNoise(0.09)(concatenated_data)\n\n    # Build the neural network layers\n    x = Dense(512, activation='elu')(x)\n    x = Dense(256, activation='elu')(x)\n    x = Dense(128, activation='elu')(x)\n    x = Dense(256, activation='elu')(x)\n    x = Dense(512, activation='elu')(x)\n\n    # Output layer\n    output = Dense(18211, activation='linear')(x)\n\n    # Create the model\n    model = Model(inputs=[smiles_input, cell_type_input], outputs=output)\n\n    # Compile the model\n    model.compile(loss=\"mae\", optimizer=optimizer, metrics=[\"mae\"])\n    return model\n\ndef train_and_predict(model, X_train_list, y_train, X_val_list, y_val, X_test_list, scaler, epochs, early_stopping=10):\n    # Define Early Stopping callback\n    early_stopping_callback = EarlyStopping(monitor='val_loss', patience=early_stopping, restore_best_weights=True)\n\n    # Train the model with early stopping\n    history = model.fit(X_train_list, y_train, epochs=epochs, verbose=0, \n                        validation_data=(X_val_list, y_val), callbacks=[early_stopping_callback])\n\n    # Make predictions on the test set\n    preds = model.predict(X_test_list, batch_size=1)\n    preds = scaler.inverse_transform(preds)\n\n    return history, preds\n\n\ndef plot_training_history_kfold(histories_and_folds):\n    fig, axes = plt.subplots(nrows=2, ncols=1, figsize=(15, 10))\n\n    for history, fold in histories_and_folds:\n        color = plt.cm.jet(fold / len(histories_and_folds))\n\n        # Plot Training Loss\n        train_loss = history.history['loss']\n        min_train_loss_epoch = np.argmin(train_loss)\n        axes[0].plot(train_loss, label=f'Fold {fold + 1}', color=color)\n        axes[0].axvline(min_train_loss_epoch, linestyle='--', color=color, label=f'Min Train Loss (Fold {fold + 1}, Epoch {min_train_loss_epoch + 1})')\n\n        # Plot Validation Loss\n        val_loss = history.history['val_loss']\n        min_val_loss_epoch = np.argmin(val_loss)\n        axes[1].plot(val_loss, label=f'Fold {fold + 1}', color=color)\n        axes[1].axvline(min_val_loss_epoch, linestyle='--', color=color, label=f'Min Val Loss (Fold {fold + 1}, Epoch {min_val_loss_epoch + 1})')\n\n    axes[0].set_xlabel('Epochs')\n    axes[0].set_ylabel('Training Loss')\n    axes[0].set_title('Training Loss Across Folds')\n    axes[0].legend()\n\n    axes[1].set_xlabel('Epochs')\n    axes[1].set_ylabel('Validation Loss')\n    axes[1].set_title('Validation Loss Across Folds')\n    axes[1].legend()\n\n    plt.tight_layout()\n    plt.show()\n\ndef k_fold_predict(create_model_func, features_list=[], full_labels=None, test_data=None, num_folds=7, random_state=None, scaler=None, optimizer=None, epochs=100, early_stopping=10):\n    # Initialize lists to store the predictions\n    all_preds = []\n    all_histories = []\n    # Initialize the KFold object\n    kf = KFold(n_splits=num_folds, shuffle=True, random_state=random_state)\n\n    # Loop through the K folds\n    for fold, (train_index, val_index) in enumerate(kf.split(full_labels)):\n        # Convert indices to integers and split the data\n        train_index = train_index.astype(int)\n        val_index = val_index.astype(int)\n\n        X_train_list = [features[train_index] for features in features_list]\n        X_val_list = [features[val_index] for features in features_list]\n        y_train = full_labels[train_index]\n        y_val = full_labels[val_index]\n\n        # Create and compile the model\n        model = create_model_func(input_shape=[X_train.shape[1:] for X_train in X_train_list], optimizer=optimizer)\n\n        # Train the model and get predictions on the test set\n        history, preds = train_and_predict(model, X_train_list, y_train, X_val_list, y_val, test_data, scaler, epochs=100, early_stopping=early_stopping)\n\n        # Store predictions for this fold\n        all_preds.append(preds)\n        all_histories.append((history, fold))\n        \n    plot_training_history_kfold(all_histories)\n    return all_preds\n\n# Example usage:\n# final_test_data = [np.array(final_test_smiles), final_cell_type]\n# all_preds = k_fold_predict(build_model_4, features_list=[full_train_chars, full_cell_type], full_labels=full_labels,\n#                            test_data=final_test_data, num_folds=5, random_state=42, scaler=scaler, optimizer=Adam(learning_rate=0.001))\n","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:07.85646Z","iopub.execute_input":"2023-11-12T23:59:07.856743Z","iopub.status.idle":"2023-11-12T23:59:09.530851Z","shell.execute_reply.started":"2023-11-12T23:59:07.856718Z","shell.execute_reply":"2023-11-12T23:59:09.53007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing data ","metadata":{"execution":{"iopub.status.busy":"2023-10-10T09:17:28.606984Z","iopub.execute_input":"2023-10-10T09:17:28.607326Z","iopub.status.idle":"2023-10-10T09:17:28.611868Z","shell.execute_reply.started":"2023-10-10T09:17:28.607301Z","shell.execute_reply":"2023-10-10T09:17:28.610695Z"}}},{"cell_type":"code","source":"# Shuffle data because we use full features for final training \ndata = data.sample(frac=1.0, random_state=RNDST2)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:09.532309Z","iopub.execute_input":"2023-11-12T23:59:09.532573Z","iopub.status.idle":"2023-11-12T23:59:09.578751Z","shell.execute_reply.started":"2023-11-12T23:59:09.532541Z","shell.execute_reply":"2023-11-12T23:59:09.577638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_columns(data, id_map)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:09.579933Z","iopub.execute_input":"2023-11-12T23:59:09.580497Z","iopub.status.idle":"2023-11-12T23:59:09.672744Z","shell.execute_reply.started":"2023-11-12T23:59:09.580466Z","shell.execute_reply":"2023-11-12T23:59:09.671878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_type_feature = pd.DataFrame(data[\"cell_type\"], columns=[\"cell_type\"])\nsmiles_feature = pd.DataFrame(data[\"SMILES\"], columns=[\"SMILES\"])\nlabels = data.drop([\"cell_type\",\"sm_name\",\"sm_lincs_id\",\"SMILES\",\"control\"], axis=1)\n\n# for test\ntest_feature_smiles = pd.DataFrame(id_map[\"SMILES\"], columns=[\"SMILES\"])\ntest_feature_cell_type = pd.DataFrame(id_map[\"cell_type\"], columns=[\"cell_type\"])","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:09.676373Z","iopub.execute_input":"2023-11-12T23:59:09.676654Z","iopub.status.idle":"2023-11-12T23:59:09.717724Z","shell.execute_reply.started":"2023-11-12T23:59:09.676628Z","shell.execute_reply":"2023-11-12T23:59:09.716854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_type_feature.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:09.718993Z","iopub.execute_input":"2023-11-12T23:59:09.719388Z","iopub.status.idle":"2023-11-12T23:59:09.732044Z","shell.execute_reply.started":"2023-11-12T23:59:09.719353Z","shell.execute_reply":"2023-11-12T23:59:09.731182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"smiles_feature.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:09.733199Z","iopub.execute_input":"2023-11-12T23:59:09.73349Z","iopub.status.idle":"2023-11-12T23:59:09.744775Z","shell.execute_reply.started":"2023-11-12T23:59:09.733466Z","shell.execute_reply":"2023-11-12T23:59:09.743976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_feature_cell_type.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:09.74584Z","iopub.execute_input":"2023-11-12T23:59:09.746156Z","iopub.status.idle":"2023-11-12T23:59:09.758118Z","shell.execute_reply.started":"2023-11-12T23:59:09.746124Z","shell.execute_reply":"2023-11-12T23:59:09.757158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_feature_smiles.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:09.759273Z","iopub.execute_input":"2023-11-12T23:59:09.759597Z","iopub.status.idle":"2023-11-12T23:59:09.772365Z","shell.execute_reply.started":"2023-11-12T23:59:09.759564Z","shell.execute_reply":"2023-11-12T23:59:09.771443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:09.773704Z","iopub.execute_input":"2023-11-12T23:59:09.77403Z","iopub.status.idle":"2023-11-12T23:59:09.803026Z","shell.execute_reply.started":"2023-11-12T23:59:09.773988Z","shell.execute_reply":"2023-11-12T23:59:09.802182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Normalize labels\nfrom sklearn.preprocessing import StandardScaler\n\nscaler = StandardScaler()\nnorm_label = scaler.fit_transform(labels)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:09.804239Z","iopub.execute_input":"2023-11-12T23:59:09.804499Z","iopub.status.idle":"2023-11-12T23:59:10.328533Z","shell.execute_reply.started":"2023-11-12T23:59:09.804476Z","shell.execute_reply":"2023-11-12T23:59:10.327527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Onehotencode\nfrom sklearn.preprocessing import OneHotEncoder\n\nencoder = OneHotEncoder()\none_hot_celltype = encoder.fit_transform(cell_type_feature)\none_hot_test_cell_type = encoder.transform(test_feature_cell_type)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:10.329843Z","iopub.execute_input":"2023-11-12T23:59:10.330564Z","iopub.status.idle":"2023-11-12T23:59:10.338367Z","shell.execute_reply.started":"2023-11-12T23:59:10.330526Z","shell.execute_reply":"2023-11-12T23:59:10.337351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain_cell_type, temp_cell_type, train_labels, temp_labels = train_test_split(one_hot_celltype.toarray(),\n                                                                            norm_label,\n                                                                            test_size=0.3, random_state=RNDST2)\n\nval_cell_type, test_cell_type, val_labels , test_labels = train_test_split(temp_cell_type, temp_cell_type, test_size=0.6, random_state=RNDST2)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:10.339742Z","iopub.execute_input":"2023-11-12T23:59:10.340332Z","iopub.status.idle":"2023-11-12T23:59:10.468176Z","shell.execute_reply.started":"2023-11-12T23:59:10.340296Z","shell.execute_reply":"2023-11-12T23:59:10.467358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_smiles, temp_smiles, train_labels, temp_labels = train_test_split(smiles_feature[\"SMILES\"].to_numpy(),\n                                                                            norm_label,\n                                                                            test_size=0.3, random_state=RNDST2)\n\nval_smiles, test_smiles, val_labels , test_labels = train_test_split(temp_smiles, temp_labels, test_size=0.6, random_state=RNDST2)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:10.469325Z","iopub.execute_input":"2023-11-12T23:59:10.469608Z","iopub.status.idle":"2023-11-12T23:59:10.591456Z","shell.execute_reply.started":"2023-11-12T23:59:10.469582Z","shell.execute_reply":"2023-11-12T23:59:10.590673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_smiles = smiles_feature[\"SMILES\"].values \nfull_cell_type = one_hot_celltype.toarray()\nfull_labels = norm_label","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:10.592596Z","iopub.execute_input":"2023-11-12T23:59:10.592934Z","iopub.status.idle":"2023-11-12T23:59:10.598152Z","shell.execute_reply.started":"2023-11-12T23:59:10.5929Z","shell.execute_reply":"2023-11-12T23:59:10.597075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_smiles = test_feature_smiles[\"SMILES\"].values\nfinal_cell_type = one_hot_test_cell_type.toarray()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:10.59922Z","iopub.execute_input":"2023-11-12T23:59:10.599485Z","iopub.status.idle":"2023-11-12T23:59:10.608958Z","shell.execute_reply.started":"2023-11-12T23:59:10.599461Z","shell.execute_reply":"2023-11-12T23:59:10.608178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_smiles[:3]","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:10.609948Z","iopub.execute_input":"2023-11-12T23:59:10.61023Z","iopub.status.idle":"2023-11-12T23:59:10.620438Z","shell.execute_reply.started":"2023-11-12T23:59:10.610207Z","shell.execute_reply":"2023-11-12T23:59:10.619544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_cell_type","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:10.621327Z","iopub.execute_input":"2023-11-12T23:59:10.621598Z","iopub.status.idle":"2023-11-12T23:59:10.633058Z","shell.execute_reply.started":"2023-11-12T23:59:10.621566Z","shell.execute_reply":"2023-11-12T23:59:10.632252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_cell_type","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:10.63397Z","iopub.execute_input":"2023-11-12T23:59:10.634276Z","iopub.status.idle":"2023-11-12T23:59:10.644542Z","shell.execute_reply.started":"2023-11-12T23:59:10.634253Z","shell.execute_reply":"2023-11-12T23:59:10.643644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_smiles[:3]","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:10.64552Z","iopub.execute_input":"2023-11-12T23:59:10.645781Z","iopub.status.idle":"2023-11-12T23:59:10.656844Z","shell.execute_reply.started":"2023-11-12T23:59:10.64575Z","shell.execute_reply":"2023-11-12T23:59:10.656068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_smiles[:10]","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:10.657867Z","iopub.execute_input":"2023-11-12T23:59:10.658146Z","iopub.status.idle":"2023-11-12T23:59:10.669053Z","shell.execute_reply.started":"2023-11-12T23:59:10.658116Z","shell.execute_reply":"2023-11-12T23:59:10.668119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the lengths\nlen(train_smiles), len(train_smiles), len(val_smiles), len(val_smiles), len(test_smiles), len(test_smiles)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:10.669988Z","iopub.execute_input":"2023-11-12T23:59:10.670253Z","iopub.status.idle":"2023-11-12T23:59:10.681071Z","shell.execute_reply.started":"2023-11-12T23:59:10.670231Z","shell.execute_reply":"2023-11-12T23:59:10.68024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split chars \nWe will split the `SMILES` into chars and then vectorize.","metadata":{}},{"cell_type":"code","source":"# Function to split sentences into characters\ndef split_chars(text):\n    return \" \".join(list(text))","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:10.685992Z","iopub.execute_input":"2023-11-12T23:59:10.686256Z","iopub.status.idle":"2023-11-12T23:59:10.691991Z","shell.execute_reply.started":"2023-11-12T23:59:10.686234Z","shell.execute_reply":"2023-11-12T23:59:10.691077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = \"CC(C)c1cc(C(=O)N2Cc3ccc(CN4CCN(C)CC4)cc3C2)c(O)cc1O\"\nsplit_chars(sample)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:10.692952Z","iopub.execute_input":"2023-11-12T23:59:10.693254Z","iopub.status.idle":"2023-11-12T23:59:10.705284Z","shell.execute_reply.started":"2023-11-12T23:59:10.69323Z","shell.execute_reply":"2023-11-12T23:59:10.704456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Split the smiles","metadata":{}},{"cell_type":"code","source":"train_char_smiles = [split_chars(feature) for feature in train_smiles]\nval_char_smiles = [split_chars(feature) for feature in val_smiles]\ntest_char_smiles = [split_chars(feature) for feature in test_smiles]","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:10.70628Z","iopub.execute_input":"2023-11-12T23:59:10.706596Z","iopub.status.idle":"2023-11-12T23:59:10.717328Z","shell.execute_reply.started":"2023-11-12T23:59:10.706563Z","shell.execute_reply":"2023-11-12T23:59:10.716573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For final training\nfull_train_smiles = [split_chars(feature) for feature in full_smiles]","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:10.718184Z","iopub.execute_input":"2023-11-12T23:59:10.718429Z","iopub.status.idle":"2023-11-12T23:59:10.733775Z","shell.execute_reply.started":"2023-11-12T23:59:10.718406Z","shell.execute_reply":"2023-11-12T23:59:10.73286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For prediction\nfinal_test_smiles = [split_chars(feature) for feature in final_test_smiles]","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:10.734795Z","iopub.execute_input":"2023-11-12T23:59:10.735126Z","iopub.status.idle":"2023-11-12T23:59:10.745889Z","shell.execute_reply.started":"2023-11-12T23:59:10.735101Z","shell.execute_reply":"2023-11-12T23:59:10.745139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Char vectorizer","metadata":{}},{"cell_type":"code","source":"# What's the average character length?\nchar_len_smiles = [len(feature) for feature in full_smiles]\nmean_char_smiles = np.mean(char_len_smiles)\nmean_char_smiles","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:10.746957Z","iopub.execute_input":"2023-11-12T23:59:10.747703Z","iopub.status.idle":"2023-11-12T23:59:10.758783Z","shell.execute_reply.started":"2023-11-12T23:59:10.747668Z","shell.execute_reply":"2023-11-12T23:59:10.757909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  Find what character length covers 95% of sequences\noutput_seq_char_len = int(np.percentile(char_len_smiles, 95))\noutput_seq_char_len","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:10.759878Z","iopub.execute_input":"2023-11-12T23:59:10.760457Z","iopub.status.idle":"2023-11-12T23:59:10.772555Z","shell.execute_reply.started":"2023-11-12T23:59:10.760425Z","shell.execute_reply":"2023-11-12T23:59:10.771697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the distribution of our sequences at character-level\nplt.hist(char_len_smiles, bins=7);","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:10.773741Z","iopub.execute_input":"2023-11-12T23:59:10.774141Z","iopub.status.idle":"2023-11-12T23:59:11.039557Z","shell.execute_reply.started":"2023-11-12T23:59:10.774108Z","shell.execute_reply":"2023-11-12T23:59:11.038686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Get all the unique characters in smiles","metadata":{"execution":{"iopub.status.busy":"2023-10-10T09:36:47.578256Z","iopub.execute_input":"2023-10-10T09:36:47.578652Z","iopub.status.idle":"2023-10-10T09:36:47.585849Z","shell.execute_reply.started":"2023-10-10T09:36:47.578623Z","shell.execute_reply":"2023-10-10T09:36:47.5846Z"}}},{"cell_type":"code","source":"unique_characters = []\n\n# Iterate over the \"features\" and extract unique characters\nfor feature in full_train_smiles:\n    unique_characters.extend(set(feature))\n\n# Remove duplicates by converting the list to a set and then back to a list\nunique_characters = list(set(unique_characters))\n\n# Print the list of unique characters\nprint(unique_characters)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:11.040698Z","iopub.execute_input":"2023-11-12T23:59:11.041044Z","iopub.status.idle":"2023-11-12T23:59:11.048625Z","shell.execute_reply.started":"2023-11-12T23:59:11.040995Z","shell.execute_reply":"2023-11-12T23:59:11.047699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(unique_characters)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:11.049725Z","iopub.execute_input":"2023-11-12T23:59:11.049974Z","iopub.status.idle":"2023-11-12T23:59:11.058913Z","shell.execute_reply.started":"2023-11-12T23:59:11.04995Z","shell.execute_reply":"2023-11-12T23:59:11.05806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.layers.experimental.preprocessing import TextVectorization\n\n# Create char-level token vectorizer instance\nNUM_CHAR_TOKENS = len(unique_characters) + 2 \nchar_vectorizer_smiles = TextVectorization(max_tokens=NUM_CHAR_TOKENS,\n                                    output_sequence_length=output_seq_char_len,\n                                    standardize=None,\n                                    split=\"character\",\n                                    name=\"char_vectorizer_SMILES\")\n\n# Adapt character vectorizer to training characters\nchar_vectorizer_smiles.adapt(train_char_smiles)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:11.060049Z","iopub.execute_input":"2023-11-12T23:59:11.060412Z","iopub.status.idle":"2023-11-12T23:59:11.292966Z","shell.execute_reply.started":"2023-11-12T23:59:11.06038Z","shell.execute_reply":"2023-11-12T23:59:11.292221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the config of our char vectorizer\nchar_vectorizer_smiles.get_config()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:11.294112Z","iopub.execute_input":"2023-11-12T23:59:11.294403Z","iopub.status.idle":"2023-11-12T23:59:11.30091Z","shell.execute_reply.started":"2023-11-12T23:59:11.294378Z","shell.execute_reply":"2023-11-12T23:59:11.300076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\n\n# Test out character vectorizer\nrandom_train_feature = random.choice(train_char_smiles)\nprint(f\"Charified text:\\n{random_train_feature}\")\nprint(f\"\\nLength of feature: {len(random_train_feature.split())}\")\nvectorized_feature = char_vectorizer_smiles([random_train_feature])\nprint(f\"\\nVectorized feature:\\n{vectorized_feature}\")\nprint(f\"\\nLength of vectorized feature: {len(vectorized_feature[0])}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:11.301967Z","iopub.execute_input":"2023-11-12T23:59:11.302348Z","iopub.status.idle":"2023-11-12T23:59:12.265417Z","shell.execute_reply.started":"2023-11-12T23:59:11.302323Z","shell.execute_reply":"2023-11-12T23:59:12.264286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"You'll notice sequences with a length shorter than 193 (output_seq_char_length) get padded with zeros on the end, this ensures all sequences passed to our model are the same length.","metadata":{}},{"cell_type":"code","source":"# Check character vocabulary characteristics\nchar_vocab = char_vectorizer_smiles.get_vocabulary()\nprint(f\"Number of different characters in character vocab: {len(char_vocab)}\")\nprint(f\"5 most common characters: {char_vocab[:5]}\")\nprint(f\"5 least common characters: {char_vocab[-5:]}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:12.268561Z","iopub.execute_input":"2023-11-12T23:59:12.268894Z","iopub.status.idle":"2023-11-12T23:59:12.275764Z","shell.execute_reply.started":"2023-11-12T23:59:12.268867Z","shell.execute_reply":"2023-11-12T23:59:12.274841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Char embedding","metadata":{}},{"cell_type":"code","source":"# Create char embedding layer\nchar_embedding = tf.keras.layers.Embedding(input_dim=NUM_CHAR_TOKENS, # number of different characters\n                              output_dim=16, # embedding dimension of each character \n                              name=\"char_embedding_SMILES\")","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:12.276929Z","iopub.execute_input":"2023-11-12T23:59:12.277249Z","iopub.status.idle":"2023-11-12T23:59:12.287496Z","shell.execute_reply.started":"2023-11-12T23:59:12.277224Z","shell.execute_reply":"2023-11-12T23:59:12.286598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test out character embedding layer\nprint(f\"Charified text (before vectorization and embedding):\\n{random_train_feature}\\n\")\nchar_embed_example = char_embedding(char_vectorizer_smiles([random_train_feature]))\nprint(f\"Embedded chars (after vectorization and embedding):\\n{char_embed_example}\\n\")\nprint(f\"Character embedding shape: {char_embed_example.shape}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:12.288638Z","iopub.execute_input":"2023-11-12T23:59:12.288994Z","iopub.status.idle":"2023-11-12T23:59:12.318587Z","shell.execute_reply.started":"2023-11-12T23:59:12.28895Z","shell.execute_reply":"2023-11-12T23:59:12.317804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert list to array\ntrain_chars = np.array(train_char_smiles)\nval_chars = np.array(val_char_smiles)\ntest_chars = np.array(test_char_smiles)\nfull_train_chars = np.array(full_train_smiles)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:12.319674Z","iopub.execute_input":"2023-11-12T23:59:12.320135Z","iopub.status.idle":"2023-11-12T23:59:12.325675Z","shell.execute_reply.started":"2023-11-12T23:59:12.320102Z","shell.execute_reply":"2023-11-12T23:59:12.324771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Building models","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.utils import plot_model\nfrom tensorflow.keras import Model\nfrom tensorflow.keras.layers import LSTM, Dense, concatenate, Conv1D, GlobalMaxPooling1D, Input, Flatten, GaussianNoise, GlobalAveragePooling1D","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:12.326861Z","iopub.execute_input":"2023-11-12T23:59:12.327339Z","iopub.status.idle":"2023-11-12T23:59:12.336381Z","shell.execute_reply.started":"2023-11-12T23:59:12.327307Z","shell.execute_reply":"2023-11-12T23:59:12.335658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.layers import Input, Conv1D, Dense, GlobalMaxPooling1D, concatenate\nfrom tensorflow.keras.models import Model\n\ndef build_smiles_input(input_shape=(1,)):\n    # Define the SMILES input\n    smiles_input = Input(shape=input_shape, dtype=\"string\", name=\"smiles_input\")\n    char_vectors = char_vectorizer_smiles(smiles_input)\n    char_embeddings = char_embedding(char_vectors)\n    conv_layer = Conv1D(64, kernel_size=5, padding=\"same\", activation=\"relu\")(char_embeddings)\n    smiles_output = GlobalMaxPooling1D()(conv_layer)\n    return smiles_input, smiles_output\n\ndef build_cell_type_input(cell_type_shape=(6,)):\n    # Define the cell type input\n    cell_type_input = Input(shape=cell_type_shape, name=\"cell_type_input\")\n    cell_type_output = Dense(32, activation=\"relu\")(cell_type_input)\n    return cell_type_input, cell_type_output\n\ndef build_neural_network(concatenated):\n    # Define the neural network architecture\n    x = Dense(128, activation=\"relu\")(concatenated)\n    output = Dense(18211, activation=\"linear\")(x)\n    return output","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:12.37441Z","iopub.execute_input":"2023-11-12T23:59:12.374914Z","iopub.status.idle":"2023-11-12T23:59:12.382683Z","shell.execute_reply.started":"2023-11-12T23:59:12.37489Z","shell.execute_reply":"2023-11-12T23:59:12.381723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 2: Custom neural net ","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.layers import Input, Flatten, concatenate, GaussianNoise, Dense\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import EarlyStopping\n\ndef build_model_3(input_shape=(1,), cell_type_shape=(6,), optimizer=None, patience=10):\n    # Define the SMILES input\n    smiles_input = Input(shape=(1,), dtype=\"string\", name=\"smiles_input\")\n    char_vectors = char_vectorizer_smiles(smiles_input)\n    char_embeddings = char_embedding(char_vectors)\n    embed_flatten = Flatten()(char_embeddings)\n\n    # Define the cell type input\n    cell_type_input = Input(shape=cell_type_shape, dtype=tf.float32, name=\"cell_type_input\")\n\n    # Concatenate SMILES embedding and cell type data\n    concatenated_data = concatenate([embed_flatten, cell_type_input], name=\"concatenate\")\n\n    # Apply Gaussian Noise\n    x = GaussianNoise(0.09)(concatenated_data)\n\n    # Build the neural network layers\n    x = Dense(512, activation='elu')(x)\n    x = Dense(256, activation='elu')(x)\n    x = Dense(128, activation='elu')(x)\n    x = Dense(256, activation='elu')(x)\n    x = Dense(512, activation='elu')(x)\n    # Output layer\n    output = Dense(18211, activation='linear')(x)\n\n    # Create the model\n    model_3 = Model(inputs=[smiles_input, cell_type_input], outputs=output, name=\"model_3\")\n\n    # Compile the model\n    if optimizer:\n        model_3.compile(loss=\"mae\", optimizer=optimizer, metrics=[\"mae\"])\n    else:\n        model_3.compile(loss=\"mae\", optimizer=tf.keras.optimizers.Lion(\n    ), metrics=[\"mae\"])\n        \n\n    return model_3\n","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:01:17.766797Z","iopub.execute_input":"2023-11-13T00:01:17.767188Z","iopub.status.idle":"2023-11-13T00:01:17.778458Z","shell.execute_reply.started":"2023-11-13T00:01:17.767158Z","shell.execute_reply":"2023-11-13T00:01:17.7777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model(build_model_3(), show_shapes=True, show_layer_activations=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:59:13.030947Z","iopub.execute_input":"2023-11-12T23:59:13.03127Z","iopub.status.idle":"2023-11-12T23:59:13.59724Z","shell.execute_reply.started":"2023-11-12T23:59:13.031245Z","shell.execute_reply":"2023-11-12T23:59:13.596346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_2 = build_model_3()\nmodel_2_history = model_2.fit(\n    x=[train_chars, train_cell_type],\n    y=train_labels,\n    epochs=40,\n    verbose=0,\n    validation_data=([val_chars, val_cell_type], val_labels),\n    callbacks=[create_model_checkpoint(\"model_2\")])","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:01:23.180115Z","iopub.execute_input":"2023-11-13T00:01:23.1805Z","iopub.status.idle":"2023-11-13T00:01:39.996403Z","shell.execute_reply.started":"2023-11-13T00:01:23.18047Z","shell.execute_reply":"2023-11-13T00:01:39.995553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_2.load_weights(\"model_2\")\nprint(\"Scores on test:\\n\")\ncalculate_mae_and_mrrmse(model=model_2, data=[test_chars, test_cell_type], y_true=test_labels, scaler=scaler)\nprint(\"\\nScores on full data:\\n\")\ncalculate_mae_and_mrrmse(model=model_2, data=[full_train_chars, full_cell_type], y_true=full_labels, scaler=scaler)\nprint(\"\\nPlot training and validation curves:\\n\")\nplot_training_history(model_2_history, metrics=[\"mae\"])","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:01:39.998282Z","iopub.execute_input":"2023-11-13T00:01:39.998556Z","iopub.status.idle":"2023-11-13T00:01:43.313292Z","shell.execute_reply.started":"2023-11-13T00:01:39.99853Z","shell.execute_reply":"2023-11-13T00:01:43.312296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.__version__","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:01:43.314447Z","iopub.execute_input":"2023-11-13T00:01:43.314717Z","iopub.status.idle":"2023-11-13T00:01:43.320752Z","shell.execute_reply.started":"2023-11-13T00:01:43.314694Z","shell.execute_reply":"2023-11-13T00:01:43.319745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res = k_fold_predict(build_model_3, \n                     features_list=[full_train_chars, np.array(full_cell_type)], \n                     full_labels=full_labels, \n                     num_folds=10, \n                     random_state=RNDST1, \n                     epochs=300, \n                     scaler=scaler,\n                     test_data=[np.array(final_test_smiles), final_cell_type], \n                    early_stopping=100)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:16:08.946626Z","iopub.execute_input":"2023-11-13T00:16:08.946998Z","iopub.status.idle":"2023-11-13T00:18:56.004874Z","shell.execute_reply.started":"2023-11-13T00:16:08.946968Z","shell.execute_reply":"2023-11-13T00:18:56.004004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make Prediction","metadata":{}},{"cell_type":"markdown","source":"We'll train `Model_2` with full train data for final time to make prediction. ","metadata":{}},{"cell_type":"code","source":"smiles_input = Input(shape=(1,), dtype=\"string\", name=\"smiles_input\")\ncelltype_input = Input(shape=(6,), dtype=tf.float32, name=\"cell_type_input\")\n\nchar_vectors = char_vectorizer_smiles(smiles_input)\nchar_embeddings = char_embedding(char_vectors)\nembed_flatten = Flatten()(char_embeddings)\nconcatenated_data = concatenate([embed_flatten, celltype_input], name=\"concatenate\")\nx = GaussianNoise(0.09)(concatenated_data)\nx = Dense(512, activation='elu')(x)\nx = Dense(256, activation='elu')(x)\nx = Dense(128, activation='elu')(x)\nx = Dense(256, activation='elu')(x)\nx = Dense(512, activation='elu')(x)\noutput = Dense(18211, activation='linear')(x)\n\nmodel = Model(inputs=[smiles_input, celltype_input], outputs=output)\nmodel.compile(loss=\"mae\", optimizer=tf.keras.optimizers.Adam(learning_rate=0.00098), metrics=[\"mae\"])\n\nhistory = model.fit(x=[full_train_chars, np.array(full_cell_type)], y=full_labels,\n                              epochs=100,\n                              verbose=0,\n                              validation_data=([test_chars, test_cell_type], test_labels))\n\nprint(\"Scores on test:\\n\")\ncalculate_mae_and_mrrmse(model=model, data=[test_chars, test_cell_type], y_true=test_labels, scaler=scaler)\nprint(\"\\nScores on full data:\\n\")\ncalculate_mae_and_mrrmse(model=model, data=[full_train_chars, full_cell_type], y_true=full_labels, scaler=scaler)\nprint(\"\\nPlot training and validation curves:\\n\")\nplot_training_history(history, metrics=[\"mae\"])\n\nsample_columns = sample_submission.columns\nsample_columns = sample_columns[1:]\n\npreds = model.predict([np.array(final_test_smiles), final_cell_type], batch_size=1)\npreds = scaler.inverse_transform(preds)","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:02:52.771609Z","iopub.execute_input":"2023-11-13T00:02:52.772356Z","iopub.status.idle":"2023-11-13T00:03:14.891682Z","shell.execute_reply.started":"2023-11-13T00:02:52.772318Z","shell.execute_reply":"2023-11-13T00:03:14.890892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame(np.mean(res, axis = 0), columns=sample_columns)\ndf.insert(0, 'id', range(255))\ndf.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:03:14.892796Z","iopub.execute_input":"2023-11-13T00:03:14.893099Z","iopub.status.idle":"2023-11-13T00:03:22.790395Z","shell.execute_reply.started":"2023-11-13T00:03:14.893073Z","shell.execute_reply":"2023-11-13T00:03:22.789428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **Note:** The model is not at all tuned. So try tuning hyperparams and learning rate to get better LB score.  And let me know how it goes. ","metadata":{}},{"cell_type":"markdown","source":"# Improving Model\n* Experiment with different `SMILES` embedding dimensions, observe and find the best.\n* Explore various layer types such as `Attention`, `BatchNormalization` `Bidirectional`, `RNN`...\n* Use different model architecture.\n* Fine-tune model hyperparameters and learning rates for optimization.\n* Test out alternative loss functions to determine their impact on model performance.\n* Evaluate the use of `MinMax` Scaler instead of `StandardScaler` for scaling the labels\n* Try vectorization or embedding for the `cell_type`\n*  Increase the number of features and create a more complex model to capture additional patterns.","metadata":{}}]}