{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59094,"databundleVersionId":7010844,"sourceType":"competition"}],"dockerImageVersionId":30559,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# NLP Regression  \nThis is just a simple experiment with NLP and to get low MRRMSE(Mean Rowwise Root Mean Sqared Error)\n\nWe're going to use SMILES embedding.\n\nSteps to do above:\n1. Preprocess the data \n2. Make a TextVectorizer and embedding\n3. Build a Model\n4. Visualize, Evaluate and Repeat \n\nLet's go...\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import tensorflow as tf\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:01.163381Z","iopub.execute_input":"2023-11-12T23:26:01.16409Z","iopub.status.idle":"2023-11-12T23:26:09.217664Z","shell.execute_reply.started":"2023-11-12T23:26:01.164058Z","shell.execute_reply":"2023-11-12T23:26:09.216883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read data","metadata":{}},{"cell_type":"code","source":"data = pd.read_parquet(\"/kaggle/input/open-problems-single-cell-perturbations/de_train.parquet\")\nid_map = pd.read_csv(\"/kaggle/input/open-problems-single-cell-perturbations/id_map.csv\")\nsample_submission = pd.read_csv(\"/kaggle/input/open-problems-single-cell-perturbations/sample_submission.csv\")\nRNDST1=271828\nRNDST2=314159","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:09.219143Z","iopub.execute_input":"2023-11-12T23:26:09.219668Z","iopub.status.idle":"2023-11-12T23:26:15.546761Z","shell.execute_reply.started":"2023-11-12T23:26:09.219642Z","shell.execute_reply":"2023-11-12T23:26:15.545988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" **🔑Tip:** Stare at the data for a while to get some insights and ideas to run experiments.","metadata":{}},{"cell_type":"code","source":"data.head(20)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:15.547911Z","iopub.execute_input":"2023-11-12T23:26:15.54825Z","iopub.status.idle":"2023-11-12T23:26:15.59084Z","shell.execute_reply.started":"2023-11-12T23:26:15.548219Z","shell.execute_reply":"2023-11-12T23:26:15.589977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Aux functions ","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.callbacks import ModelCheckpoint\n\ndef create_model_checkpoint(filepath, monitor='val_mae', save_best_only=True,\n                            save_weights_only=True, mode='auto', verbose=0):\n    \"\"\"\n    Create a ModelCheckpoint callback for saving the best model weights during training.\n\n    Args:\n        filepath (str): Filepath to save the best weights.\n        monitor (str): Metric to monitor (e.g., 'val_loss' or 'val_mae').\n        save_best_only (bool): Save only the best weights.\n        save_weights_only (bool): Save only the model's weights, not the entire model.\n        mode (str): One of {'auto', 'min', 'max'}. In 'min' mode, it saves when the monitored metric decreases.\n        verbose (int): Verbosity mode. 0 = silent, 1 = progress bar, 2 = one line per epoch.\n\n    Returns:\n        keras.callbacks.ModelCheckpoint: ModelCheckpoint callback.\n    \"\"\"\n    checkpoint = ModelCheckpoint(\n        filepath=filepath,\n        monitor=monitor,\n        save_best_only=save_best_only,\n        save_weights_only=save_weights_only,\n        mode=mode,\n        verbose=verbose\n    )\n    return checkpoint\n","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:15.592777Z","iopub.execute_input":"2023-11-12T23:26:15.593043Z","iopub.status.idle":"2023-11-12T23:26:15.600202Z","shell.execute_reply.started":"2023-11-12T23:26:15.59302Z","shell.execute_reply":"2023-11-12T23:26:15.599374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_training_history(history, metrics):\n    \"\"\"\n    Plot training history curves for loss and evaluation metrics on the same line.\n\n    Args:\n        history (keras.callbacks.History): Training history object.\n        metrics (list): List of metric names to plot.\n\n    Returns:\n        None\n    \"\"\"\n    loss = history.history['loss']\n    val_loss = history.history['val_loss']\n\n    epochs = range(len(loss))\n\n    plt.figure(figsize=(12, 6))\n\n    # Plot loss\n    plt.subplot(1, 2, 1)\n    plt.plot(epochs, loss, label='Training Loss', color=\"blue\")\n    plt.plot(epochs, val_loss, label='Validation Loss', color=\"red\")\n    plt.title('Loss')\n    plt.xlabel('Epochs')\n    plt.legend()\n\n    # Plot specified evaluation metrics on the same line\n    for metric in metrics:\n        train_metric_name = f'Training {metric.capitalize()}'\n        val_metric_name = f'Validation {metric.capitalize()}'\n        train_metric = history.history[metric]\n        val_metric = history.history['val_' + metric]\n\n        plt.subplot(1, 2, 2)\n        plt.plot(epochs, train_metric, label=train_metric_name, color=\"green\")\n        plt.plot(epochs, val_metric, label=val_metric_name, color=\"orange\")\n\n    plt.title('Metrics')\n    plt.xlabel('Epochs')\n    plt.legend(loc='upper right')\n\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:15.601143Z","iopub.execute_input":"2023-11-12T23:26:15.601391Z","iopub.status.idle":"2023-11-12T23:26:15.61067Z","shell.execute_reply.started":"2023-11-12T23:26:15.601369Z","shell.execute_reply":"2023-11-12T23:26:15.609842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_columns(data, id_map):\n    sm_name_to_smiles = data.set_index('sm_name')['SMILES'].to_dict()\n    sm_lincs_id = data.set_index('sm_name')[\"sm_lincs_id\"].to_dict()\n\n    id_map['SMILES'] = id_map['sm_name'].map(sm_name_to_smiles)\n    id_map['sm_lincs_id'] = id_map['sm_name'].map(sm_lincs_id)\n\n    return id_map","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:15.611748Z","iopub.execute_input":"2023-11-12T23:26:15.612Z","iopub.status.idle":"2023-11-12T23:26:15.625081Z","shell.execute_reply.started":"2023-11-12T23:26:15.611978Z","shell.execute_reply":"2023-11-12T23:26:15.624313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_absolute_error\n\ndef calculate_mae_and_mrrmse(model, data, y_true, scaler=None):\n    \"\"\"\n    Calculate Mean Absolute Error (MAE) and Mean Rowwise Root Mean Squared Error (MRRMSE).\n\n    Parameters:\n    - model: The trained  model.\n    - data: The input data for prediction.\n    - y_true: The true target values.\n    - scaler: The scaler used for data normalization.\n\n    Returns:\n    - None\n    \"\"\"\n    # Predict using the model\n    y_pred_original = model.predict(data, batch_size=1)\n\n    if scaler:\n        \n        y_pred = scaler.inverse_transform(y_pred_original)\n        y_true = scaler.inverse_transform(y_true)\n        # Calculate Mean Absolute Error (MAE)\n        mae = mean_absolute_error(y_true , y_pred)\n\n        # Calculate Mean Rowwise Root Mean Squared Error (MRRMSE)\n        rowwise_rmse = np.sqrt(np.mean(np.square(y_true - y_pred), axis=1))\n        mrrmse_score = np.mean(rowwise_rmse)\n    else:\n       # Calculate Mean Absolute Error (MAE)\n        mae = mean_absolute_error(y_true , y_pred_original)\n\n        # Calculate Mean Rowwise Root Mean Squared Error (MRRMSE)\n        rowwise_rmse = np.sqrt(np.mean(np.square(y_true - y_pred_original), axis=1))\n        mrrmse_score = np.mean(rowwise_rmse)\n    # Print the results\n    print(f\"Mean Absolute Error (MAE): {mae}\")\n    print(f\"Mean Rowwise Root Mean Squared Error (MRRMSE): {mrrmse_score}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:15.626053Z","iopub.execute_input":"2023-11-12T23:26:15.6263Z","iopub.status.idle":"2023-11-12T23:26:15.967347Z","shell.execute_reply.started":"2023-11-12T23:26:15.626278Z","shell.execute_reply":"2023-11-12T23:26:15.966489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.model_selection import KFold\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import mean_absolute_error\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Dense, Flatten, concatenate, GaussianNoise\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import EarlyStopping\n\ndef build_model_4(input_shape, optimizer=Adam(learning_rate=0.001)):\n    # Define the SMILES input\n    smiles_input = Input(shape=input_shape[0], dtype=\"string\", name=\"smiles_input\")\n    char_vectors = char_vectorizer_smiles(smiles_input)\n    char_embeddings = char_embedding(char_vectors)\n    embed_flatten = Flatten()(char_embeddings)\n\n    # Define the cell type input\n    cell_type_input = Input(shape=input_shape[1], dtype=tf.float32, name=\"cell_type_input\")\n\n    # Concatenate SMILES embedding and cell type data\n    concatenated_data = concatenate([embed_flatten, cell_type_input], name=\"concatenate\")\n\n    # Apply Gaussian Noise\n    x = GaussianNoise(0.09)(concatenated_data)\n\n    # Build the neural network layers\n    x = Dense(512, activation='elu')(x)\n    x = Dense(256, activation='elu')(x)\n    x = Dense(128, activation='elu')(x)\n    x = Dense(256, activation='elu')(x)\n    x = Dense(512, activation='elu')(x)\n\n    # Output layer\n    output = Dense(18211, activation='linear')(x)\n\n    # Create the model\n    model = Model(inputs=[smiles_input, cell_type_input], outputs=output)\n\n    # Compile the model\n    model.compile(loss=\"mae\", optimizer=optimizer, metrics=[\"mae\"])\n    return model\n\ndef train_and_predict(model, X_train_list, y_train, X_val_list, y_val, X_test_list, scaler, epochs, early_stopping=10):\n    # Define Early Stopping callback\n    early_stopping_callback = EarlyStopping(monitor='val_loss', patience=early_stopping, restore_best_weights=True)\n\n    # Train the model with early stopping\n    history = model.fit(X_train_list, y_train, epochs=epochs, verbose=0, \n                        validation_data=(X_val_list, y_val), callbacks=[early_stopping_callback])\n\n    # Make predictions on the test set\n    preds = model.predict(X_test_list, batch_size=1)\n    preds = scaler.inverse_transform(preds)\n\n    return history, preds\n\n\ndef plot_training_history_kfold(histories_and_folds):\n    fig, axes = plt.subplots(nrows=2, ncols=1, figsize=(15, 10))\n\n    for history, fold in histories_and_folds:\n        color = plt.cm.jet(fold / len(histories_and_folds))\n\n        # Plot Training Loss\n        train_loss = history.history['loss']\n        min_train_loss_epoch = np.argmin(train_loss)\n        axes[0].plot(train_loss, label=f'Fold {fold + 1}', color=color)\n        axes[0].axvline(min_train_loss_epoch, linestyle='--', color=color, label=f'Min Train Loss (Fold {fold + 1}, Epoch {min_train_loss_epoch + 1})')\n\n        # Plot Validation Loss\n        val_loss = history.history['val_loss']\n        min_val_loss_epoch = np.argmin(val_loss)\n        axes[1].plot(val_loss, label=f'Fold {fold + 1}', color=color)\n        axes[1].axvline(min_val_loss_epoch, linestyle='--', color=color, label=f'Min Val Loss (Fold {fold + 1}, Epoch {min_val_loss_epoch + 1})')\n\n    axes[0].set_xlabel('Epochs')\n    axes[0].set_ylabel('Training Loss')\n    axes[0].set_title('Training Loss Across Folds')\n    axes[0].legend()\n\n    axes[1].set_xlabel('Epochs')\n    axes[1].set_ylabel('Validation Loss')\n    axes[1].set_title('Validation Loss Across Folds')\n    axes[1].legend()\n\n    plt.tight_layout()\n    plt.show()\n\ndef k_fold_predict(create_model_func, features_list=[], full_labels=None, test_data=None, num_folds=7, random_state=None, scaler=None, optimizer=None, epochs=100, early_stopping=10):\n    # Initialize lists to store the predictions\n    all_preds = []\n    all_histories = []\n    # Initialize the KFold object\n    kf = KFold(n_splits=num_folds, shuffle=True, random_state=random_state)\n\n    # Loop through the K folds\n    for fold, (train_index, val_index) in enumerate(kf.split(full_labels)):\n        # Convert indices to integers and split the data\n        train_index = train_index.astype(int)\n        val_index = val_index.astype(int)\n\n        X_train_list = [features[train_index] for features in features_list]\n        X_val_list = [features[val_index] for features in features_list]\n        y_train = full_labels[train_index]\n        y_val = full_labels[val_index]\n\n        # Create and compile the model\n        model = create_model_func(input_shape=[X_train.shape[1:] for X_train in X_train_list], optimizer=optimizer)\n\n        # Train the model and get predictions on the test set\n        history, preds = train_and_predict(model, X_train_list, y_train, X_val_list, y_val, test_data, scaler, epochs=100, early_stopping=early_stopping)\n\n        # Store predictions for this fold\n        all_preds.append(preds)\n        all_histories.append((history, fold))\n        \n    plot_training_history_kfold(all_histories)\n    return all_preds\n\n# Example usage:\n# final_test_data = [np.array(final_test_smiles), final_cell_type]\n# all_preds = k_fold_predict(build_model_4, features_list=[full_train_chars, full_cell_type], full_labels=full_labels,\n#                            test_data=final_test_data, num_folds=5, random_state=42, scaler=scaler, optimizer=Adam(learning_rate=0.001))\n","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:15.968621Z","iopub.execute_input":"2023-11-12T23:26:15.968978Z","iopub.status.idle":"2023-11-12T23:26:18.906589Z","shell.execute_reply.started":"2023-11-12T23:26:15.968942Z","shell.execute_reply":"2023-11-12T23:26:18.905804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing data ","metadata":{"execution":{"iopub.status.busy":"2023-10-10T09:17:28.606984Z","iopub.execute_input":"2023-10-10T09:17:28.607326Z","iopub.status.idle":"2023-10-10T09:17:28.611868Z","shell.execute_reply.started":"2023-10-10T09:17:28.607301Z","shell.execute_reply":"2023-10-10T09:17:28.610695Z"}}},{"cell_type":"code","source":"# Shuffle data because we use full features for final training \ndata = data.sample(frac=1.0, random_state=RNDST2)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:18.907921Z","iopub.execute_input":"2023-11-12T23:26:18.908285Z","iopub.status.idle":"2023-11-12T23:26:18.95718Z","shell.execute_reply.started":"2023-11-12T23:26:18.908253Z","shell.execute_reply":"2023-11-12T23:26:18.956091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_columns(data, id_map)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:18.961573Z","iopub.execute_input":"2023-11-12T23:26:18.96184Z","iopub.status.idle":"2023-11-12T23:26:19.05025Z","shell.execute_reply.started":"2023-11-12T23:26:18.961818Z","shell.execute_reply":"2023-11-12T23:26:19.04933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_type_feature = pd.DataFrame(data[\"cell_type\"], columns=[\"cell_type\"])\nsmiles_feature = pd.DataFrame(data[\"SMILES\"], columns=[\"SMILES\"])\nlabels = data.drop([\"cell_type\",\"sm_name\",\"sm_lincs_id\",\"SMILES\",\"control\"], axis=1)\n\n# for test\ntest_feature_smiles = pd.DataFrame(id_map[\"SMILES\"], columns=[\"SMILES\"])\ntest_feature_cell_type = pd.DataFrame(id_map[\"cell_type\"], columns=[\"cell_type\"])","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:19.05154Z","iopub.execute_input":"2023-11-12T23:26:19.052127Z","iopub.status.idle":"2023-11-12T23:26:19.093052Z","shell.execute_reply.started":"2023-11-12T23:26:19.052094Z","shell.execute_reply":"2023-11-12T23:26:19.092001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_type_feature.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:19.094359Z","iopub.execute_input":"2023-11-12T23:26:19.094726Z","iopub.status.idle":"2023-11-12T23:26:19.107011Z","shell.execute_reply.started":"2023-11-12T23:26:19.094686Z","shell.execute_reply":"2023-11-12T23:26:19.106162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"smiles_feature.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:19.108162Z","iopub.execute_input":"2023-11-12T23:26:19.108507Z","iopub.status.idle":"2023-11-12T23:26:19.120793Z","shell.execute_reply.started":"2023-11-12T23:26:19.108483Z","shell.execute_reply":"2023-11-12T23:26:19.120031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_feature_cell_type.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:19.121724Z","iopub.execute_input":"2023-11-12T23:26:19.121959Z","iopub.status.idle":"2023-11-12T23:26:19.132682Z","shell.execute_reply.started":"2023-11-12T23:26:19.121938Z","shell.execute_reply":"2023-11-12T23:26:19.131857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_feature_smiles.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:19.133639Z","iopub.execute_input":"2023-11-12T23:26:19.13387Z","iopub.status.idle":"2023-11-12T23:26:19.14516Z","shell.execute_reply.started":"2023-11-12T23:26:19.13385Z","shell.execute_reply":"2023-11-12T23:26:19.144367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:19.146243Z","iopub.execute_input":"2023-11-12T23:26:19.14655Z","iopub.status.idle":"2023-11-12T23:26:19.175334Z","shell.execute_reply.started":"2023-11-12T23:26:19.146527Z","shell.execute_reply":"2023-11-12T23:26:19.174539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Normalize labels\nfrom sklearn.preprocessing import StandardScaler\n\nscaler = StandardScaler()\nnorm_label = scaler.fit_transform(labels)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:19.176401Z","iopub.execute_input":"2023-11-12T23:26:19.176708Z","iopub.status.idle":"2023-11-12T23:26:19.710277Z","shell.execute_reply.started":"2023-11-12T23:26:19.176667Z","shell.execute_reply":"2023-11-12T23:26:19.709329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Onehotencode\nfrom sklearn.preprocessing import OneHotEncoder\n\nencoder = OneHotEncoder()\none_hot_celltype = encoder.fit_transform(cell_type_feature)\none_hot_test_cell_type = encoder.transform(test_feature_cell_type)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:19.711448Z","iopub.execute_input":"2023-11-12T23:26:19.711751Z","iopub.status.idle":"2023-11-12T23:26:19.719334Z","shell.execute_reply.started":"2023-11-12T23:26:19.711719Z","shell.execute_reply":"2023-11-12T23:26:19.718478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain_cell_type, temp_cell_type, train_labels, temp_labels = train_test_split(one_hot_celltype.toarray(),\n                                                                            norm_label,\n                                                                            test_size=0.3, random_state=RNDST2)\n\nval_cell_type, test_cell_type, val_labels , test_labels = train_test_split(temp_cell_type, temp_cell_type, test_size=0.6, random_state=RNDST2)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:19.72112Z","iopub.execute_input":"2023-11-12T23:26:19.721644Z","iopub.status.idle":"2023-11-12T23:26:19.848469Z","shell.execute_reply.started":"2023-11-12T23:26:19.721609Z","shell.execute_reply":"2023-11-12T23:26:19.847689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_smiles, temp_smiles, train_labels, temp_labels = train_test_split(smiles_feature[\"SMILES\"].to_numpy(),\n                                                                            norm_label,\n                                                                            test_size=0.3, random_state=RNDST2)\n\nval_smiles, test_smiles, val_labels , test_labels = train_test_split(temp_smiles, temp_labels, test_size=0.6, random_state=RNDST2)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:19.849469Z","iopub.execute_input":"2023-11-12T23:26:19.849711Z","iopub.status.idle":"2023-11-12T23:26:19.975625Z","shell.execute_reply.started":"2023-11-12T23:26:19.84969Z","shell.execute_reply":"2023-11-12T23:26:19.974792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_smiles = smiles_feature[\"SMILES\"].values \nfull_cell_type = one_hot_celltype.toarray()\nfull_labels = norm_label","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:19.97687Z","iopub.execute_input":"2023-11-12T23:26:19.97718Z","iopub.status.idle":"2023-11-12T23:26:19.981803Z","shell.execute_reply.started":"2023-11-12T23:26:19.977153Z","shell.execute_reply":"2023-11-12T23:26:19.980851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_smiles = test_feature_smiles[\"SMILES\"].values\nfinal_cell_type = one_hot_test_cell_type.toarray()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:19.982968Z","iopub.execute_input":"2023-11-12T23:26:19.983232Z","iopub.status.idle":"2023-11-12T23:26:19.993295Z","shell.execute_reply.started":"2023-11-12T23:26:19.983209Z","shell.execute_reply":"2023-11-12T23:26:19.992448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_smiles[:3]","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:19.994429Z","iopub.execute_input":"2023-11-12T23:26:19.994814Z","iopub.status.idle":"2023-11-12T23:26:20.005499Z","shell.execute_reply.started":"2023-11-12T23:26:19.99478Z","shell.execute_reply":"2023-11-12T23:26:20.004645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_cell_type","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.006843Z","iopub.execute_input":"2023-11-12T23:26:20.007111Z","iopub.status.idle":"2023-11-12T23:26:20.017462Z","shell.execute_reply.started":"2023-11-12T23:26:20.007089Z","shell.execute_reply":"2023-11-12T23:26:20.016408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_cell_type","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.018708Z","iopub.execute_input":"2023-11-12T23:26:20.019177Z","iopub.status.idle":"2023-11-12T23:26:20.027507Z","shell.execute_reply.started":"2023-11-12T23:26:20.019153Z","shell.execute_reply":"2023-11-12T23:26:20.026702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_smiles[:3]","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.02846Z","iopub.execute_input":"2023-11-12T23:26:20.028685Z","iopub.status.idle":"2023-11-12T23:26:20.037855Z","shell.execute_reply.started":"2023-11-12T23:26:20.028664Z","shell.execute_reply":"2023-11-12T23:26:20.036895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_smiles[:10]","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.038953Z","iopub.execute_input":"2023-11-12T23:26:20.039231Z","iopub.status.idle":"2023-11-12T23:26:20.048512Z","shell.execute_reply.started":"2023-11-12T23:26:20.039208Z","shell.execute_reply":"2023-11-12T23:26:20.047725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the lengths\nlen(train_smiles), len(train_smiles), len(val_smiles), len(val_smiles), len(test_smiles), len(test_smiles)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.054339Z","iopub.execute_input":"2023-11-12T23:26:20.054633Z","iopub.status.idle":"2023-11-12T23:26:20.0611Z","shell.execute_reply.started":"2023-11-12T23:26:20.054609Z","shell.execute_reply":"2023-11-12T23:26:20.060356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split chars \nWe will split the `SMILES` into chars and then vectorize.","metadata":{}},{"cell_type":"code","source":"# Function to split sentences into characters\ndef split_chars(text):\n    return \" \".join(list(text))","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.06205Z","iopub.execute_input":"2023-11-12T23:26:20.062284Z","iopub.status.idle":"2023-11-12T23:26:20.070396Z","shell.execute_reply.started":"2023-11-12T23:26:20.062263Z","shell.execute_reply":"2023-11-12T23:26:20.069716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = \"CC(C)c1cc(C(=O)N2Cc3ccc(CN4CCN(C)CC4)cc3C2)c(O)cc1O\"\nsplit_chars(sample)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.071425Z","iopub.execute_input":"2023-11-12T23:26:20.071691Z","iopub.status.idle":"2023-11-12T23:26:20.081237Z","shell.execute_reply.started":"2023-11-12T23:26:20.071651Z","shell.execute_reply":"2023-11-12T23:26:20.08045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Split the smiles","metadata":{}},{"cell_type":"code","source":"train_char_smiles = [split_chars(feature) for feature in train_smiles]\nval_char_smiles = [split_chars(feature) for feature in val_smiles]\ntest_char_smiles = [split_chars(feature) for feature in test_smiles]","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.082278Z","iopub.execute_input":"2023-11-12T23:26:20.08254Z","iopub.status.idle":"2023-11-12T23:26:20.092184Z","shell.execute_reply.started":"2023-11-12T23:26:20.082518Z","shell.execute_reply":"2023-11-12T23:26:20.091375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For final training\nfull_train_smiles = [split_chars(feature) for feature in full_smiles]","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.093278Z","iopub.execute_input":"2023-11-12T23:26:20.09357Z","iopub.status.idle":"2023-11-12T23:26:20.106488Z","shell.execute_reply.started":"2023-11-12T23:26:20.093536Z","shell.execute_reply":"2023-11-12T23:26:20.105609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For prediction\nfinal_test_smiles = [split_chars(feature) for feature in final_test_smiles]","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.107671Z","iopub.execute_input":"2023-11-12T23:26:20.108229Z","iopub.status.idle":"2023-11-12T23:26:20.116877Z","shell.execute_reply.started":"2023-11-12T23:26:20.108198Z","shell.execute_reply":"2023-11-12T23:26:20.11611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Char vectorizer","metadata":{}},{"cell_type":"code","source":"# What's the average character length?\nchar_len_smiles = [len(feature) for feature in full_smiles]\nmean_char_smiles = np.mean(char_len_smiles)\nmean_char_smiles","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.117962Z","iopub.execute_input":"2023-11-12T23:26:20.118217Z","iopub.status.idle":"2023-11-12T23:26:20.127933Z","shell.execute_reply.started":"2023-11-12T23:26:20.118195Z","shell.execute_reply":"2023-11-12T23:26:20.127196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  Find what character length covers 95% of sequences\noutput_seq_char_len = int(np.percentile(char_len_smiles, 95))\noutput_seq_char_len","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.129011Z","iopub.execute_input":"2023-11-12T23:26:20.129292Z","iopub.status.idle":"2023-11-12T23:26:20.140352Z","shell.execute_reply.started":"2023-11-12T23:26:20.129269Z","shell.execute_reply":"2023-11-12T23:26:20.139461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the distribution of our sequences at character-level\nplt.hist(char_len_smiles, bins=7);","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.141493Z","iopub.execute_input":"2023-11-12T23:26:20.141744Z","iopub.status.idle":"2023-11-12T23:26:20.352463Z","shell.execute_reply.started":"2023-11-12T23:26:20.141722Z","shell.execute_reply":"2023-11-12T23:26:20.351532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Get all the unique characters in smiles","metadata":{"execution":{"iopub.status.busy":"2023-10-10T09:36:47.578256Z","iopub.execute_input":"2023-10-10T09:36:47.578652Z","iopub.status.idle":"2023-10-10T09:36:47.585849Z","shell.execute_reply.started":"2023-10-10T09:36:47.578623Z","shell.execute_reply":"2023-10-10T09:36:47.5846Z"}}},{"cell_type":"code","source":"unique_characters = []\n\n# Iterate over the \"features\" and extract unique characters\nfor feature in full_train_smiles:\n    unique_characters.extend(set(feature))\n\n# Remove duplicates by converting the list to a set and then back to a list\nunique_characters = list(set(unique_characters))\n\n# Print the list of unique characters\nprint(unique_characters)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.353666Z","iopub.execute_input":"2023-11-12T23:26:20.354027Z","iopub.status.idle":"2023-11-12T23:26:20.362182Z","shell.execute_reply.started":"2023-11-12T23:26:20.353994Z","shell.execute_reply":"2023-11-12T23:26:20.361333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(unique_characters)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.363565Z","iopub.execute_input":"2023-11-12T23:26:20.363916Z","iopub.status.idle":"2023-11-12T23:26:20.373878Z","shell.execute_reply.started":"2023-11-12T23:26:20.363884Z","shell.execute_reply":"2023-11-12T23:26:20.37306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.layers.experimental.preprocessing import TextVectorization\n\n# Create char-level token vectorizer instance\nNUM_CHAR_TOKENS = len(unique_characters) + 2 \nchar_vectorizer_smiles = TextVectorization(max_tokens=NUM_CHAR_TOKENS,\n                                    output_sequence_length=output_seq_char_len,\n                                    standardize=None,\n                                    split=\"character\",\n                                    name=\"char_vectorizer_SMILES\")\n\n# Adapt character vectorizer to training characters\nchar_vectorizer_smiles.adapt(train_char_smiles)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.374953Z","iopub.execute_input":"2023-11-12T23:26:20.375193Z","iopub.status.idle":"2023-11-12T23:26:20.722325Z","shell.execute_reply.started":"2023-11-12T23:26:20.375171Z","shell.execute_reply":"2023-11-12T23:26:20.72136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the config of our char vectorizer\nchar_vectorizer_smiles.get_config()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.723722Z","iopub.execute_input":"2023-11-12T23:26:20.724393Z","iopub.status.idle":"2023-11-12T23:26:20.73094Z","shell.execute_reply.started":"2023-11-12T23:26:20.724359Z","shell.execute_reply":"2023-11-12T23:26:20.730112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\n\n# Test out character vectorizer\nrandom_train_feature = random.choice(train_char_smiles)\nprint(f\"Charified text:\\n{random_train_feature}\")\nprint(f\"\\nLength of feature: {len(random_train_feature.split())}\")\nvectorized_feature = char_vectorizer_smiles([random_train_feature])\nprint(f\"\\nVectorized feature:\\n{vectorized_feature}\")\nprint(f\"\\nLength of vectorized feature: {len(vectorized_feature[0])}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.732036Z","iopub.execute_input":"2023-11-12T23:26:20.732285Z","iopub.status.idle":"2023-11-12T23:26:20.778071Z","shell.execute_reply.started":"2023-11-12T23:26:20.732263Z","shell.execute_reply":"2023-11-12T23:26:20.777119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"You'll notice sequences with a length shorter than 193 (output_seq_char_length) get padded with zeros on the end, this ensures all sequences passed to our model are the same length.","metadata":{}},{"cell_type":"code","source":"# Check character vocabulary characteristics\nchar_vocab = char_vectorizer_smiles.get_vocabulary()\nprint(f\"Number of different characters in character vocab: {len(char_vocab)}\")\nprint(f\"5 most common characters: {char_vocab[:5]}\")\nprint(f\"5 least common characters: {char_vocab[-5:]}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.779244Z","iopub.execute_input":"2023-11-12T23:26:20.779757Z","iopub.status.idle":"2023-11-12T23:26:20.787409Z","shell.execute_reply.started":"2023-11-12T23:26:20.779727Z","shell.execute_reply":"2023-11-12T23:26:20.786549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Char embedding","metadata":{}},{"cell_type":"code","source":"# Create char embedding layer\nchar_embedding = tf.keras.layers.Embedding(input_dim=NUM_CHAR_TOKENS, # number of different characters\n                              output_dim=16, # embedding dimension of each character \n                              name=\"char_embedding_SMILES\")","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.788582Z","iopub.execute_input":"2023-11-12T23:26:20.788846Z","iopub.status.idle":"2023-11-12T23:26:20.797536Z","shell.execute_reply.started":"2023-11-12T23:26:20.788824Z","shell.execute_reply":"2023-11-12T23:26:20.796456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test out character embedding layer\nprint(f\"Charified text (before vectorization and embedding):\\n{random_train_feature}\\n\")\nchar_embed_example = char_embedding(char_vectorizer_smiles([random_train_feature]))\nprint(f\"Embedded chars (after vectorization and embedding):\\n{char_embed_example}\\n\")\nprint(f\"Character embedding shape: {char_embed_example.shape}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.79882Z","iopub.execute_input":"2023-11-12T23:26:20.79905Z","iopub.status.idle":"2023-11-12T23:26:20.843901Z","shell.execute_reply.started":"2023-11-12T23:26:20.79903Z","shell.execute_reply":"2023-11-12T23:26:20.842996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert list to array\ntrain_chars = np.array(train_char_smiles)\nval_chars = np.array(val_char_smiles)\ntest_chars = np.array(test_char_smiles)\nfull_train_chars = np.array(full_train_smiles)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.845125Z","iopub.execute_input":"2023-11-12T23:26:20.845496Z","iopub.status.idle":"2023-11-12T23:26:20.854379Z","shell.execute_reply.started":"2023-11-12T23:26:20.845462Z","shell.execute_reply":"2023-11-12T23:26:20.853441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Building models","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.utils import plot_model\nfrom tensorflow.keras import Model\nfrom tensorflow.keras.layers import LSTM, Dense, concatenate, Conv1D, GlobalMaxPooling1D, Input, Flatten, GaussianNoise, GlobalAveragePooling1D","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.855608Z","iopub.execute_input":"2023-11-12T23:26:20.856215Z","iopub.status.idle":"2023-11-12T23:26:20.861633Z","shell.execute_reply.started":"2023-11-12T23:26:20.856173Z","shell.execute_reply":"2023-11-12T23:26:20.860811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.layers import Input, Conv1D, Dense, GlobalMaxPooling1D, concatenate\nfrom tensorflow.keras.models import Model\n\ndef build_smiles_input(input_shape=(1,)):\n    # Define the SMILES input\n    smiles_input = Input(shape=input_shape, dtype=\"string\", name=\"smiles_input\")\n    char_vectors = char_vectorizer_smiles(smiles_input)\n    char_embeddings = char_embedding(char_vectors)\n    conv_layer = Conv1D(64, kernel_size=5, padding=\"same\", activation=\"relu\")(char_embeddings)\n    smiles_output = GlobalMaxPooling1D()(conv_layer)\n    return smiles_input, smiles_output\n\ndef build_cell_type_input(cell_type_shape=(6,)):\n    # Define the cell type input\n    cell_type_input = Input(shape=cell_type_shape, name=\"cell_type_input\")\n    cell_type_output = Dense(32, activation=\"relu\")(cell_type_input)\n    return cell_type_input, cell_type_output\n\ndef build_neural_network(concatenated):\n    # Define the neural network architecture\n    x = Dense(128, activation=\"relu\")(concatenated)\n    output = Dense(18211, activation=\"linear\")(x)\n    return output","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.862709Z","iopub.execute_input":"2023-11-12T23:26:20.86297Z","iopub.status.idle":"2023-11-12T23:26:20.871491Z","shell.execute_reply.started":"2023-11-12T23:26:20.862947Z","shell.execute_reply":"2023-11-12T23:26:20.870636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 2: Custom neural net ","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.layers import Input, Flatten, concatenate, GaussianNoise, Dense\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import EarlyStopping\n\ndef build_model_3(input_shape=(1,), cell_type_shape=(6,), optimizer=None, patience=10):\n    # Define the SMILES input\n    smiles_input = Input(shape=(1,), dtype=\"string\", name=\"smiles_input\")\n    char_vectors = char_vectorizer_smiles(smiles_input)\n    char_embeddings = char_embedding(char_vectors)\n    embed_flatten = Flatten()(char_embeddings)\n\n    # Define the cell type input\n    cell_type_input = Input(shape=cell_type_shape, dtype=tf.float32, name=\"cell_type_input\")\n\n    # Concatenate SMILES embedding and cell type data\n    concatenated_data = concatenate([embed_flatten, cell_type_input], name=\"concatenate\")\n\n    # Apply Gaussian Noise\n    x = GaussianNoise(0.09)(concatenated_data)\n\n    # Build the neural network layers\n    x = Dense(64, activation='sigmoid')(x)\n    \n    # Output layer\n    output = Dense(18211, activation='linear')(x)\n\n    # Create the model\n    model_3 = Model(inputs=[smiles_input, cell_type_input], outputs=output, name=\"model_3\")\n\n    # Compile the model\n    if optimizer:\n        model_3.compile(loss=\"mae\", optimizer=optimizer, metrics=[\"mae\"])\n    else:\n        model_3.compile(loss=\"mae\", optimizer=tf.keras.optimizers.Adam(learning_rate=0.00098), metrics=[\"mae\"])\n        \n    return model_3\n","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.872524Z","iopub.execute_input":"2023-11-12T23:26:20.872821Z","iopub.status.idle":"2023-11-12T23:26:20.887429Z","shell.execute_reply.started":"2023-11-12T23:26:20.872797Z","shell.execute_reply":"2023-11-12T23:26:20.886526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model(build_model_3(), show_shapes=True, show_layer_activations=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:20.888578Z","iopub.execute_input":"2023-11-12T23:26:20.888834Z","iopub.status.idle":"2023-11-12T23:26:21.504011Z","shell.execute_reply.started":"2023-11-12T23:26:20.888812Z","shell.execute_reply":"2023-11-12T23:26:21.503118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_2 = build_model_3()\nmodel_2_history = model_2.fit(\n    x=[train_chars, train_cell_type],\n    y=train_labels,\n    epochs=40,\n    verbose=0,\n    validation_data=([val_chars, val_cell_type], val_labels),\n    callbacks=[create_model_checkpoint(\"model_2\")])","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:26:21.505132Z","iopub.execute_input":"2023-11-12T23:26:21.505425Z","iopub.status.idle":"2023-11-12T23:27:28.995815Z","shell.execute_reply.started":"2023-11-12T23:26:21.505385Z","shell.execute_reply":"2023-11-12T23:27:28.994969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_2.load_weights(\"model_2\")\nprint(\"Scores on test:\\n\")\ncalculate_mae_and_mrrmse(model=model_2, data=[test_chars, test_cell_type], y_true=test_labels, scaler=scaler)\nprint(\"\\nScores on full data:\\n\")\ncalculate_mae_and_mrrmse(model=model_2, data=[full_train_chars, full_cell_type], y_true=full_labels, scaler=scaler)\nprint(\"\\nPlot training and validation curves:\\n\")\nplot_training_history(model_2_history, metrics=[\"mae\"])","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:27:28.997141Z","iopub.execute_input":"2023-11-12T23:27:28.997445Z","iopub.status.idle":"2023-11-12T23:27:33.730257Z","shell.execute_reply.started":"2023-11-12T23:27:28.997398Z","shell.execute_reply":"2023-11-12T23:27:33.72931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res = k_fold_predict(build_model_3, \n                     features_list=[full_train_chars, np.array(full_cell_type)], \n                     full_labels=full_labels, \n                     num_folds=10, \n                     random_state=RNDST1, \n                     epochs=10, \n                     scaler=scaler,\n                     test_data=[np.array(final_test_smiles), final_cell_type])\n","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:27:33.731407Z","iopub.execute_input":"2023-11-12T23:27:33.731696Z","iopub.status.idle":"2023-11-12T23:29:35.648321Z","shell.execute_reply.started":"2023-11-12T23:27:33.731671Z","shell.execute_reply":"2023-11-12T23:29:35.647274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make Prediction","metadata":{}},{"cell_type":"markdown","source":"We'll train `Model_2` with full train data for final time to make prediction. ","metadata":{}},{"cell_type":"code","source":"smiles_input = Input(shape=(1,), dtype=\"string\", name=\"smiles_input\")\ncelltype_input = Input(shape=(6,), dtype=tf.float32, name=\"cell_type_input\")\n\nchar_vectors = char_vectorizer_smiles(smiles_input)\nchar_embeddings = char_embedding(char_vectors)\nembed_flatten = Flatten()(char_embeddings)\nconcatenated_data = concatenate([embed_flatten, celltype_input], name=\"concatenate\")\nx = GaussianNoise(0.09)(concatenated_data)\nx = Dense(512, activation='elu')(x)\nx = Dense(256, activation='elu')(x)\nx = Dense(128, activation='elu')(x)\nx = Dense(256, activation='elu')(x)\nx = Dense(512, activation='elu')(x)\noutput = Dense(18211, activation='linear')(x)\n\nmodel = Model(inputs=[smiles_input, celltype_input], outputs=output)\nmodel.compile(loss=\"mae\", optimizer=tf.keras.optimizers.Adam(learning_rate=0.00098), metrics=[\"mae\"])\n\nhistory = model.fit(x=[full_train_chars, np.array(full_cell_type)], y=full_labels,\n                              epochs=100,\n                              verbose=0,\n                              validation_data=([test_chars, test_cell_type], test_labels))\n\nprint(\"Scores on test:\\n\")\ncalculate_mae_and_mrrmse(model=model, data=[test_chars, test_cell_type], y_true=test_labels, scaler=scaler)\nprint(\"\\nScores on full data:\\n\")\ncalculate_mae_and_mrrmse(model=model, data=[full_train_chars, full_cell_type], y_true=full_labels, scaler=scaler)\nprint(\"\\nPlot training and validation curves:\\n\")\nplot_training_history(history, metrics=[\"mae\"])\n\nsample_columns = sample_submission.columns\nsample_columns = sample_columns[1:]\n\npreds = model.predict([np.array(final_test_smiles), final_cell_type], batch_size=1)\npreds = scaler.inverse_transform(preds)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:29:35.649776Z","iopub.execute_input":"2023-11-12T23:29:35.650138Z","iopub.status.idle":"2023-11-12T23:29:58.72531Z","shell.execute_reply.started":"2023-11-12T23:29:35.650105Z","shell.execute_reply":"2023-11-12T23:29:58.724363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame(np.mean(res, axis = 0), columns=sample_columns)\ndf.insert(0, 'id', range(255))\ndf.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T23:29:58.726585Z","iopub.execute_input":"2023-11-12T23:29:58.726865Z","iopub.status.idle":"2023-11-12T23:30:06.894122Z","shell.execute_reply.started":"2023-11-12T23:29:58.726841Z","shell.execute_reply":"2023-11-12T23:30:06.893116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **Note:** The model is not at all tuned. So try tuning hyperparams and learning rate to get better LB score.  And let me know how it goes. ","metadata":{}},{"cell_type":"markdown","source":"# Improving Model\n* Experiment with different `SMILES` embedding dimensions, observe and find the best.\n* Explore various layer types such as `Attention`, `BatchNormalization` `Bidirectional`, `RNN`...\n* Use different model architecture.\n* Fine-tune model hyperparameters and learning rates for optimization.\n* Test out alternative loss functions to determine their impact on model performance.\n* Evaluate the use of `MinMax` Scaler instead of `StandardScaler` for scaling the labels\n* Try vectorization or embedding for the `cell_type`\n*  Increase the number of features and create a more complex model to capture additional patterns.","metadata":{}}]}