{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59094,"databundleVersionId":6541963,"sourceType":"competition"}],"dockerImageVersionId":30301,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Single-Cell Perturbation with Conv1D","metadata":{}},{"cell_type":"markdown","source":"## Import Packages","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nimport numpy as np\nimport pandas as pd\nfrom tensorflow import keras\nfrom sklearn.model_selection import RepeatedStratifiedKFold\nimport nltk\nimport gc","metadata":{"execution":{"iopub.status.busy":"2023-11-16T13:51:16.991203Z","iopub.execute_input":"2023-11-16T13:51:16.992206Z","iopub.status.idle":"2023-11-16T13:51:16.997688Z","shell.execute_reply.started":"2023-11-16T13:51:16.992165Z","shell.execute_reply":"2023-11-16T13:51:16.996631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Configuration","metadata":{}},{"cell_type":"code","source":"class Config:\n    batch_size = 32\n    vocab_size  = 0\n    epochs = 50\n    maxlen = 6\n    use_k_fold = True\n    \n    dataset_path = \"/kaggle/input/open-problems-single-cell-perturbations\"\n    \n    _target_columns = None\n    \n    def target_columns(self):\n        if self._target_columns == None:\n            submission = pd.read_csv(f\"{config.dataset_path}/sample_submission.csv\")\n            self._target_columns = list(submission.columns)\n            self._target_columns.remove(\"id\")\n        return self._target_columns\nconfig = Config() ","metadata":{"execution":{"iopub.status.busy":"2023-11-16T13:51:19.531109Z","iopub.execute_input":"2023-11-16T13:51:19.531507Z","iopub.status.idle":"2023-11-16T13:51:19.539915Z","shell.execute_reply.started":"2023-11-16T13:51:19.531476Z","shell.execute_reply":"2023-11-16T13:51:19.538524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading data","metadata":{}},{"cell_type":"code","source":"train = pd.read_parquet(f\"{config.dataset_path}/de_train.parquet\")\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T13:51:22.584953Z","iopub.execute_input":"2023-11-16T13:51:22.585751Z","iopub.status.idle":"2023-11-16T13:51:24.941286Z","shell.execute_reply.started":"2023-11-16T13:51:22.585707Z","shell.execute_reply":"2023-11-16T13:51:24.940095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(f\"{config.dataset_path}/id_map.csv\")\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T13:51:28.809590Z","iopub.execute_input":"2023-11-16T13:51:28.810005Z","iopub.status.idle":"2023-11-16T13:51:28.828566Z","shell.execute_reply.started":"2023-11-16T13:51:28.809971Z","shell.execute_reply":"2023-11-16T13:51:28.827387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocessing","metadata":{}},{"cell_type":"code","source":"train[\"text\"] = (train[\"cell_type\"] + \" \" + train[\"sm_name\"])#.apply(lambda sentence: \" \". join(nltk.word_tokenize(sentence.lower())))\ntest[\"text\"] = (test[\"cell_type\"] + \" \" + test[\"sm_name\"])#.apply(lambda sentence: \" \". join(nltk.word_tokenize(sentence.lower())))\nvectorizor = keras.layers.TextVectorization(output_sequence_length=config.maxlen)\nvectorizor.adapt(train[\"text\"])","metadata":{"execution":{"iopub.status.busy":"2023-11-16T13:51:33.009239Z","iopub.execute_input":"2023-11-16T13:51:33.010284Z","iopub.status.idle":"2023-11-16T13:51:34.281674Z","shell.execute_reply.started":"2023-11-16T13:51:33.010235Z","shell.execute_reply":"2023-11-16T13:51:34.280465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config.vcoab_size = len(vectorizor.get_vocabulary())\nconfig.vcoab_size","metadata":{"execution":{"iopub.status.busy":"2023-11-16T13:51:38.716308Z","iopub.execute_input":"2023-11-16T13:51:38.716714Z","iopub.status.idle":"2023-11-16T13:51:38.726722Z","shell.execute_reply.started":"2023-11-16T13:51:38.716680Z","shell.execute_reply":"2023-11-16T13:51:38.725189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Vocabulary\nvectorizor.get_vocabulary()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T13:51:40.778860Z","iopub.execute_input":"2023-11-16T13:51:40.779857Z","iopub.status.idle":"2023-11-16T13:51:40.791584Z","shell.execute_reply.started":"2023-11-16T13:51:40.779818Z","shell.execute_reply":"2023-11-16T13:51:40.790410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Building Model","metadata":{}},{"cell_type":"code","source":"def mean_rowwise_rmse(y_true, y_pred):\n    \"\"\"\n    Custom metric to calculate the Mean Rowwise Root Mean Squared Error (RMSE).\n\n    Parameters:\n    - y_true: The true target values.\n    - y_pred: The predicted values.\n\n    Returns:\n    - Mean Rowwise RMSE as a scalar tensor.\n    \"\"\"\n    return tf.reduce_mean(\n        tf.sqrt(tf.reduce_mean(tf.square(y_true - y_pred), axis=1))\n    )\n\ndef get_model():\n    inputs = keras.Input(shape=(), dtype=tf.string)\n    x = vectorizor(inputs)\n    x = keras.layers.Embedding(config.vcoab_size, 32, input_length=config.maxlen)(x)\n    x = keras.layers.Conv1D(filters=16, kernel_size=1, activation=\"swish\")(x)\n    x = keras.layers.SpatialDropout1D(0.3)(x)\n    x = keras.layers.Conv1D(filters=32, kernel_size=1, activation=\"swish\")(x)\n    x = keras.layers.SpatialDropout1D(0.3)(x)\n    x = keras.layers.Conv1D(filters=64, kernel_size=1, activation=\"swish\")(x)\n    x = keras.layers.SpatialDropout1D(0.3)(x)\n    x = keras.layers.Flatten()(x)\n    x = keras.layers.Dense(128, kernel_initializer='he_uniform', activation='swish')(x)\n    x = keras.layers.Dropout(0.3)(x)\n    x = keras.layers.Dense(256, kernel_initializer='he_uniform', activation='swish')(x)\n    x = keras.layers.Dropout(0.3)(x)\n    x = keras.layers.Dense(512, kernel_initializer='he_uniform', activation='swish')(x)\n    outputs = keras.layers.Dense(len(config.target_columns()))(x)\n    model = tf.keras.Model(inputs=inputs, outputs=outputs)\n    model.compile(loss=\"huber_loss\", optimizer=\"adam\", metrics=[mean_rowwise_rmse])\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-11-16T13:51:45.859184Z","iopub.execute_input":"2023-11-16T13:51:45.860054Z","iopub.status.idle":"2023-11-16T13:51:45.873298Z","shell.execute_reply.started":"2023-11-16T13:51:45.860009Z","shell.execute_reply":"2023-11-16T13:51:45.871993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = get_model()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T13:51:49.504131Z","iopub.execute_input":"2023-11-16T13:51:49.504565Z","iopub.status.idle":"2023-11-16T13:51:52.887845Z","shell.execute_reply.started":"2023-11-16T13:51:49.504529Z","shell.execute_reply":"2023-11-16T13:51:52.886618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keras.utils.plot_model(model, show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-16T13:51:56.066885Z","iopub.execute_input":"2023-11-16T13:51:56.067312Z","iopub.status.idle":"2023-11-16T13:51:57.183549Z","shell.execute_reply.started":"2023-11-16T13:51:56.067279Z","shell.execute_reply":"2023-11-16T13:51:57.182162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training models","metadata":{}},{"cell_type":"code","source":"keras.backend.clear_session()\nkfold =  RepeatedStratifiedKFold(n_splits=3, n_repeats=10, random_state=2024)\nmodels = []\nrmses = []\nfor i, (train_indices, valid_indices) in enumerate(kfold.split(train, train[\"sm_name\"])):\n    x_train = train.iloc[train_indices][\"text\"]\n    y_train = train.iloc[train_indices][config.target_columns()]\n    x_val = train.iloc[valid_indices][\"text\"]\n    y_val = train.iloc[valid_indices][config.target_columns()]\n    model_path = f\"model_{i}.tf\"\n    model = get_model()\n    checkpoint = keras.callbacks.ModelCheckpoint(model_path, monitor=\"val_mean_rowwise_rmse\", mode=\"min\", save_best_only=True, save_weights_only=True)\n    early_stop = keras.callbacks.EarlyStopping(monitor=\"val_loss\", mode=\"min\", patience=10)\n    history = model.fit(\n        x_train, y_train, \n        batch_size=config.batch_size, \n        epochs=config.epochs,\n        validation_data=(x_val, y_val),\n        callbacks=[checkpoint, early_stop],\n        verbose=2\n    )\n    model.load_weights(model_path)\n    result = model.evaluate(x_val, y_val)\n    print(\"Loss:\", result[0], \"RMSE:\", result[1])\n    rmses.append(result[1])\n    models.append(model) \n    if not config.use_k_fold:\n        break\nprint(f\"OOF:{np.mean(rmses)}\")","metadata":{"execution":{"iopub.status.busy":"2023-10-18T14:35:34.319207Z","iopub.execute_input":"2023-10-18T14:35:34.319656Z","iopub.status.idle":"2023-10-18T14:35:58.089427Z","shell.execute_reply.started":"2023-10-18T14:35:34.319622Z","shell.execute_reply":"2023-10-18T14:35:58.088569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"preds = np.median([model.predict(test[\"text\"]) for model in models], axis=0)\nsubmission_df = pd.DataFrame(preds, columns=config.target_columns())\nsubmission_df[\"id\"] = test[\"id\"]\nsubmission_df.to_csv(\"submission.csv\", index=False)\nsubmission_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-18T14:34:20.631285Z","iopub.execute_input":"2023-10-18T14:34:20.631749Z","iopub.status.idle":"2023-10-18T14:34:28.019083Z","shell.execute_reply.started":"2023-10-18T14:34:20.631716Z","shell.execute_reply":"2023-10-18T14:34:28.017859Z"},"trusted":true},"execution_count":null,"outputs":[]}]}