{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":51294,"databundleVersionId":6923401,"sourceType":"competition"},{"sourceId":6933839,"sourceType":"datasetVersion","datasetId":3981418}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"https://www.kaggle.com/code/mirenaborisova/srrnaf-2/edit","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.simplefilter('ignore')\n\nimport pandas as pd\npd.set_option('display.max_columns', 30)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-06T14:10:17.530422Z","iopub.execute_input":"2023-12-06T14:10:17.530746Z","iopub.status.idle":"2023-12-06T14:10:18.428451Z","shell.execute_reply.started":"2023-12-06T14:10:17.530716Z","shell.execute_reply":"2023-12-06T14:10:18.427648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\n\nresolver = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(resolver)\ntf.tpu.experimental.initialize_tpu_system(resolver)\nstrategy = tf.distribute.experimental.TPUStrategy(resolver)","metadata":{"execution":{"iopub.status.busy":"2023-12-06T14:10:18.429848Z","iopub.execute_input":"2023-12-06T14:10:18.430220Z","iopub.status.idle":"2023-12-06T14:10:39.565396Z","shell.execute_reply.started":"2023-12-06T14:10:18.430190Z","shell.execute_reply":"2023-12-06T14:10:39.564639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rmdb = pd.read_csv('/kaggle/input/rmdb-rna-mapping-database-2023-data/rmdb_data.v1.3.0.csv')\n# rmdb","metadata":{"execution":{"iopub.status.busy":"2023-12-06T14:10:39.566295Z","iopub.execute_input":"2023-12-06T14:10:39.566515Z","iopub.status.idle":"2023-12-06T14:10:51.783355Z","shell.execute_reply.started":"2023-12-06T14:10:39.566491Z","shell.execute_reply":"2023-12-06T14:10:51.782169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rmdb_SN_filter = rmdb[rmdb.SN_filter == 1]\nerror_feats = [feat for feat in rmdb_SN_filter.columns if 'error' in feat]\nrmdb_SN_filter_no_error = rmdb_SN_filter.drop(columns = error_feats)","metadata":{"execution":{"iopub.status.busy":"2023-12-06T14:10:51.785379Z","iopub.execute_input":"2023-12-06T14:10:51.785668Z","iopub.status.idle":"2023-12-06T14:10:52.158056Z","shell.execute_reply.started":"2023-12-06T14:10:51.785640Z","shell.execute_reply":"2023-12-06T14:10:52.156833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"experiment_type = rmdb_SN_filter_no_error.experiment_type.unique()\n\nrmdb_etype_2A3 = [experiment_type[0]]\nrmdb_etype_DMS = (experiment_type[1:]).tolist()","metadata":{"execution":{"iopub.status.busy":"2023-12-06T14:10:52.159248Z","iopub.execute_input":"2023-12-06T14:10:52.159529Z","iopub.status.idle":"2023-12-06T14:10:52.167158Z","shell.execute_reply.started":"2023-12-06T14:10:52.159501Z","shell.execute_reply":"2023-12-06T14:10:52.166408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_quick = pd.read_csv('/kaggle/input/stanford-ribonanza-rna-folding/train_data_QUICK_START.csv')\ntrain_quick_2A3 = train_quick[train_quick.experiment_type == '2A3_MaP'].reset_index(drop=True)\ntrain_quick_DMS = train_quick[train_quick.experiment_type == 'DMS_MaP'].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-06T14:10:52.168109Z","iopub.execute_input":"2023-12-06T14:10:52.168352Z","iopub.status.idle":"2023-12-06T14:11:08.533156Z","shell.execute_reply.started":"2023-12-06T14:10:52.168325Z","shell.execute_reply":"2023-12-06T14:11:08.531958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rmdb_2A3 = rmdb_SN_filter_no_error[rmdb_SN_filter_no_error.experiment_type.isin(rmdb_etype_2A3)].reset_index(drop=True)\nrmdb_2A3 = pd.concat([train_quick_2A3, rmdb_2A3]).reset_index(drop=True)\n\nrmdb_DMS = rmdb_SN_filter_no_error[rmdb_SN_filter_no_error.experiment_type.isin(rmdb_etype_DMS)].reset_index(drop=True)\nrmdb_DMS = pd.concat([train_quick_DMS, rmdb_DMS]).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-06T14:11:08.534393Z","iopub.execute_input":"2023-12-06T14:11:08.534690Z","iopub.status.idle":"2023-12-06T14:11:11.872558Z","shell.execute_reply.started":"2023-12-06T14:11:08.534657Z","shell.execute_reply":"2023-12-06T14:11:11.871404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_2A3 = rmdb_2A3.sequence\nrmdb_2A3 = rmdb_2A3.filter(regex='reactivity_[0-9]')\nrmdb_2A3 = rmdb_2A3.fillna(0)\nX_2A3 = X_2A3.iloc[rmdb_2A3.index].reset_index(drop=True)\nY_2A3 = rmdb_2A3.reset_index(drop=True)\n\nX_DMS = rmdb_DMS.sequence\nrmdb_DMS = rmdb_DMS.filter(regex='reactivity_[0-9]')\nrmdb_DMS = rmdb_DMS.fillna(0)\nX_DMS = X_DMS.iloc[rmdb_DMS.index].reset_index(drop=True)\nY_DMS = rmdb_DMS.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-06T14:11:11.873798Z","iopub.execute_input":"2023-12-06T14:11:11.874127Z","iopub.status.idle":"2023-12-06T14:11:13.768909Z","shell.execute_reply.started":"2023-12-06T14:11:11.874077Z","shell.execute_reply":"2023-12-06T14:11:13.767744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\n\ndel train_quick\ndel train_quick_2A3\ndel train_quick_DMS\n\ndel rmdb\ndel rmdb_SN_filter\ndel error_feats\ndel rmdb_SN_filter_no_error\ndel rmdb_etype_2A3\ndel rmdb_etype_DMS\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-12-06T14:11:13.770305Z","iopub.execute_input":"2023-12-06T14:11:13.770598Z","iopub.status.idle":"2023-12-06T14:11:14.064769Z","shell.execute_reply.started":"2023-12-06T14:11:13.770568Z","shell.execute_reply":"2023-12-06T14:11:14.063758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras import Model\nfrom transformers import TFAutoModel\n\nclass RNA(Model):\n    \n    def __init__(self):\n        \n        super().__init__()\n        self.encoder = TFAutoModel.from_pretrained('AmelieSchreiber/esm2_t6_8M_UR50D_rna_binding_site_predictor')\n        self.dropout = tf.keras.layers.Dropout(0.2)\n        self.dense = tf.keras.layers.Dense(1)\n        \n    def call(self, x):\n        \n        x = self.encoder(x).last_hidden_state\n        \n        return tf.squeeze(self.dense(x), -1)","metadata":{"execution":{"iopub.status.busy":"2023-12-06T14:11:14.067689Z","iopub.execute_input":"2023-12-06T14:11:14.067967Z","iopub.status.idle":"2023-12-06T14:11:36.512469Z","shell.execute_reply.started":"2023-12-06T14:11:14.067941Z","shell.execute_reply":"2023-12-06T14:11:36.511618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def loss(X, Y):\n    \n    X_mask = tf.math.is_nan(X)\n    X = tf.where(X_mask, tf.zeros_like(X), X)\n    sum_mask = tf.math.reduce_sum(tf.where(X_mask, tf.zeros_like(X), tf.ones_like(X)))\n    loss = tf.math.abs(X - Y)\n    loss = tf.where(X_mask, tf.zeros_like(loss), loss)\n    loss = tf.math.reduce_sum(loss) / (sum_mask if sum_mask != 0.0 else 1.0)\n    \n    return loss","metadata":{"execution":{"iopub.status.busy":"2023-12-06T14:11:36.513417Z","iopub.execute_input":"2023-12-06T14:11:36.513868Z","iopub.status.idle":"2023-12-06T14:11:36.518746Z","shell.execute_reply.started":"2023-12-06T14:11:36.513839Z","shell.execute_reply":"2023-12-06T14:11:36.518121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_vectorization = tf.keras.layers.TextVectorization(output_mode='int',\n                                                       ngrams=1,\n                                                       output_sequence_length=Y_DMS.shape[1],\n                                                       split='character',\n                                                       vocabulary=['a', 'c', 'g', 'u'])","metadata":{"execution":{"iopub.status.busy":"2023-12-06T14:11:36.519497Z","iopub.execute_input":"2023-12-06T14:11:36.519712Z","iopub.status.idle":"2023-12-06T14:11:36.553630Z","shell.execute_reply.started":"2023-12-06T14:11:36.519688Z","shell.execute_reply":"2023-12-06T14:11:36.552875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    \n    model_DMS = RNA()\n    model_DMS.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=5e-4), loss=loss)\n    \n    train_ds = tf.data.Dataset.from_tensor_slices((X_DMS.values, Y_DMS.values)) \\\n        .batch(128) \\\n        .map(lambda x, y: (text_vectorization(x), tf.clip_by_value(y, 0, 1)))\n    train_ds = train_ds.shuffle(train_ds.cardinality())\n    train_data = train_ds.take(int(len(train_ds) * 0.8))\n    validation_data = train_ds.skip(int(len(train_ds) * 0.8)).take(int(len(train_ds) * 0.2))\n\n    model_DMS.fit(train_data, validation_data=validation_data, epochs=25, batch_size=128)","metadata":{"execution":{"iopub.status.busy":"2023-12-06T14:11:36.554500Z","iopub.execute_input":"2023-12-06T14:11:36.554779Z","iopub.status.idle":"2023-12-06T15:31:46.135062Z","shell.execute_reply.started":"2023-12-06T14:11:36.554748Z","shell.execute_reply":"2023-12-06T15:31:46.133863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_vectorization_preds = tf.keras.layers.TextVectorization(output_mode='int',\n                                                             ngrams=1,\n                                                             output_sequence_length=457,\n                                                             split='character',\n                                                             vocabulary=['a', 'c', 'g', 'u'])","metadata":{"execution":{"iopub.status.busy":"2023-12-06T15:31:46.137595Z","iopub.execute_input":"2023-12-06T15:31:46.138234Z","iopub.status.idle":"2023-12-06T15:31:46.151353Z","shell.execute_reply.started":"2023-12-06T15:31:46.138199Z","shell.execute_reply":"2023-12-06T15:31:46.150448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/stanford-ribonanza-rna-folding/test_sequences.csv')","metadata":{"execution":{"iopub.status.busy":"2023-12-06T15:31:46.152417Z","iopub.execute_input":"2023-12-06T15:31:46.152674Z","iopub.status.idle":"2023-12-06T15:31:53.059317Z","shell.execute_reply.started":"2023-12-06T15:31:46.152648Z","shell.execute_reply":"2023-12-06T15:31:53.058155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    \n    preds_DMS = model_DMS.predict(tf.data.Dataset.from_tensor_slices((test.sequence)) \\\n        .batch(128) \\\n        .map(lambda x: text_vectorization_preds(x)))","metadata":{"execution":{"iopub.status.busy":"2023-12-06T15:31:53.060621Z","iopub.execute_input":"2023-12-06T15:31:53.060896Z","iopub.status.idle":"2023-12-06T15:39:18.666208Z","shell.execute_reply.started":"2023-12-06T15:31:53.060867Z","shell.execute_reply":"2023-12-06T15:39:18.664993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_DMS = pd.DataFrame(preds_DMS)\ndf_DMS.to_csv('preds_DMS.csv')","metadata":{"execution":{"iopub.status.busy":"2023-12-06T15:39:18.669915Z","iopub.execute_input":"2023-12-06T15:39:18.670193Z","iopub.status.idle":"2023-12-06T15:49:45.818473Z","shell.execute_reply.started":"2023-12-06T15:39:18.670167Z","shell.execute_reply":"2023-12-06T15:49:45.817392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    \n    model_2A3 = RNA()\n    model_2A3.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=5e-4), loss=loss)\n    \n    train_ds = tf.data.Dataset.from_tensor_slices((X_2A3.values, Y_2A3.values)) \\\n        .batch(128) \\\n        .map(lambda x, y: (text_vectorization(x), tf.clip_by_value(y, 0, 1)))\n    train_ds = train_ds.shuffle(train_ds.cardinality())\n    train_data = train_ds.take(int(len(train_ds) * 0.8))\n    validation_data = train_ds.skip(int(len(train_ds) * 0.8)).take(int(len(train_ds) * 0.2))\n\n    model_2A3.fit(train_data, validation_data=validation_data, epochs=25, batch_size=128)","metadata":{"execution":{"iopub.status.busy":"2023-12-06T15:49:45.819669Z","iopub.execute_input":"2023-12-06T15:49:45.819957Z","iopub.status.idle":"2023-12-06T17:12:57.166313Z","shell.execute_reply.started":"2023-12-06T15:49:45.819928Z","shell.execute_reply":"2023-12-06T17:12:57.165141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    \n    preds_2A3 = model_2A3.predict(tf.data.Dataset.from_tensor_slices((test.sequence)) \\\n        .batch(128) \\\n        .map(lambda x: text_vectorization_preds(x)))","metadata":{"execution":{"iopub.status.busy":"2023-12-06T17:12:57.168372Z","iopub.execute_input":"2023-12-06T17:12:57.168683Z","iopub.status.idle":"2023-12-06T17:20:22.650343Z","shell.execute_reply.started":"2023-12-06T17:12:57.168644Z","shell.execute_reply":"2023-12-06T17:20:22.649109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_2A3 = pd.DataFrame(preds_2A3)\n# df_2A3.to_csv('preds_2A3.csv')","metadata":{"execution":{"iopub.status.busy":"2023-12-06T17:20:22.652160Z","iopub.execute_input":"2023-12-06T17:20:22.652441Z","iopub.status.idle":"2023-12-06T17:20:22.658055Z","shell.execute_reply.started":"2023-12-06T17:20:22.652412Z","shell.execute_reply":"2023-12-06T17:20:22.657298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_seq_lengths = test.sequence.str.len()","metadata":{"execution":{"iopub.status.busy":"2023-12-06T17:20:22.658963Z","iopub.execute_input":"2023-12-06T17:20:22.659233Z","iopub.status.idle":"2023-12-06T17:20:23.132795Z","shell.execute_reply.started":"2023-12-06T17:20:22.659206Z","shell.execute_reply":"2023-12-06T17:20:23.131813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\npreds_2A3_bylength = []\n\nfor i in range(test_seq_lengths.size):\n    \n    x = np.reshape(preds_2A3[i, :test_seq_lengths[i]], (-1, 1))\n    x = np.clip(x, 0, 1)\n    \n    preds_2A3_bylength.append(x)","metadata":{"execution":{"iopub.status.busy":"2023-12-06T17:20:23.133775Z","iopub.execute_input":"2023-12-06T17:20:23.134037Z","iopub.status.idle":"2023-12-06T17:20:35.372858Z","shell.execute_reply.started":"2023-12-06T17:20:23.134010Z","shell.execute_reply":"2023-12-06T17:20:35.371875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_2A3 = np.concatenate(preds_2A3_bylength, 0)","metadata":{"execution":{"iopub.status.busy":"2023-12-06T17:20:35.373916Z","iopub.execute_input":"2023-12-06T17:20:35.374198Z","iopub.status.idle":"2023-12-06T17:20:36.315138Z","shell.execute_reply.started":"2023-12-06T17:20:35.374171Z","shell.execute_reply":"2023-12-06T17:20:36.314157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_DMS_bylength = []\n\nfor i in range(test_seq_lengths.size):\n    \n    x = np.reshape(preds_DMS[i, :test_seq_lengths[i]], (-1, 1))\n    x = np.clip(x, 0, 1)\n    \n    preds_DMS_bylength.append(x)","metadata":{"execution":{"iopub.status.busy":"2023-12-06T17:20:36.316226Z","iopub.execute_input":"2023-12-06T17:20:36.316520Z","iopub.status.idle":"2023-12-06T17:20:49.182253Z","shell.execute_reply.started":"2023-12-06T17:20:36.316490Z","shell.execute_reply":"2023-12-06T17:20:49.181250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_DMS = np.concatenate(preds_DMS_bylength, 0)","metadata":{"execution":{"iopub.status.busy":"2023-12-06T17:20:49.183225Z","iopub.execute_input":"2023-12-06T17:20:49.183473Z","iopub.status.idle":"2023-12-06T17:20:50.094184Z","shell.execute_reply.started":"2023-12-06T17:20:49.183448Z","shell.execute_reply":"2023-12-06T17:20:50.093109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_DMS.shape[0]","metadata":{"execution":{"iopub.status.busy":"2023-12-06T17:20:50.095211Z","iopub.execute_input":"2023-12-06T17:20:50.095483Z","iopub.status.idle":"2023-12-06T17:20:50.100549Z","shell.execute_reply.started":"2023-12-06T17:20:50.095454Z","shell.execute_reply":"2023-12-06T17:20:50.099756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({'id': np.arange(0, preds_DMS.shape[0], 1),\n                           'reactivity_DMS_MaP': preds_DMS[:, 0],\n                           'reactivity_2A3_MaP': preds_2A3[:, 0]})","metadata":{"execution":{"iopub.status.busy":"2023-12-06T17:20:50.101407Z","iopub.execute_input":"2023-12-06T17:20:50.101655Z","iopub.status.idle":"2023-12-06T17:20:52.124005Z","shell.execute_reply.started":"2023-12-06T17:20:50.101628Z","shell.execute_reply":"2023-12-06T17:20:52.123024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-06T17:20:52.127559Z","iopub.execute_input":"2023-12-06T17:20:52.127852Z","iopub.status.idle":"2023-12-06T17:20:52.140020Z","shell.execute_reply.started":"2023-12-06T17:20:52.127822Z","shell.execute_reply":"2023-12-06T17:20:52.139322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pyarrow","metadata":{"execution":{"iopub.status.busy":"2023-12-06T17:22:09.027215Z","iopub.execute_input":"2023-12-06T17:22:09.027582Z","iopub.status.idle":"2023-12-06T17:22:17.265419Z","shell.execute_reply.started":"2023-12-06T17:22:09.027552Z","shell.execute_reply":"2023-12-06T17:22:17.264184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_parquet('submission.parquet', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-12-06T17:22:17.267381Z","iopub.execute_input":"2023-12-06T17:22:17.267667Z","iopub.status.idle":"2023-12-06T17:23:20.685448Z","shell.execute_reply.started":"2023-12-06T17:22:17.267638Z","shell.execute_reply":"2023-12-06T17:23:20.684034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}