{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":51294,"databundleVersionId":7331882,"sourceType":"competition"},{"sourceId":6933839,"sourceType":"datasetVersion","datasetId":3981418}],"dockerImageVersionId":30580,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import tensorflow as tf\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nresolver = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(resolver)\ntf.tpu.experimental.initialize_tpu_system(resolver)\nprint(\"All devices: \", tf.config.list_logical_devices(':TPU'))\nstrategy = tf.distribute.experimental.TPUStrategy(resolver)\n\n#strategy = tf.distribute.MirroredStrategy()\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-22T15:11:32.770520Z","iopub.execute_input":"2023-12-22T15:11:32.770797Z","iopub.status.idle":"2023-12-22T15:11:54.383780Z","shell.execute_reply.started":"2023-12-22T15:11:32.770767Z","shell.execute_reply":"2023-12-22T15:11:54.382917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pandas import read_csv, concat\ndataset = read_csv('/kaggle/input/stanford-ribonanza-rna-folding/train_data_QUICK_START.csv', chunksize=100000)\ndataset = concat(dataset)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T15:11:54.385232Z","iopub.execute_input":"2023-12-22T15:11:54.385473Z","iopub.status.idle":"2023-12-22T15:12:12.352968Z","shell.execute_reply.started":"2023-12-22T15:11:54.385446Z","shell.execute_reply":"2023-12-22T15:12:12.351873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset.sequence.size","metadata":{"execution":{"iopub.status.busy":"2023-12-22T15:12:12.353960Z","iopub.execute_input":"2023-12-22T15:12:12.354289Z","iopub.status.idle":"2023-12-22T15:12:12.361671Z","shell.execute_reply.started":"2023-12-22T15:12:12.354251Z","shell.execute_reply":"2023-12-22T15:12:12.361054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dms, a3 = dataset[dataset.experiment_type=='DMS_MaP'].reset_index(drop=True), dataset[dataset.experiment_type=='2A3_MaP'].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T15:12:12.363186Z","iopub.execute_input":"2023-12-22T15:12:12.363435Z","iopub.status.idle":"2023-12-22T15:12:13.293731Z","shell.execute_reply.started":"2023-12-22T15:12:12.363410Z","shell.execute_reply":"2023-12-22T15:12:13.292895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rmdb = read_csv('/kaggle/input/rmdb-rna-mapping-database-2023-data/rmdb_data.v1.3.0.csv')","metadata":{"execution":{"iopub.status.busy":"2023-12-22T15:12:13.294753Z","iopub.execute_input":"2023-12-22T15:12:13.295041Z","iopub.status.idle":"2023-12-22T15:12:25.067520Z","shell.execute_reply.started":"2023-12-22T15:12:13.295008Z","shell.execute_reply":"2023-12-22T15:12:25.066570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a3_exp = ['1M7','NMIA','BzCN']\na3 = concat([a3, rmdb.query('experiment_type in @a3_exp and SN_filter==1')], axis=0).reset_index(drop=True)\n               \ndms_exp = ['BzCN_cotx', 'DMS_cotx', 'DMS_M2_seq', 'DMS']\ndms = concat([dms, rmdb.query('(experiment_type in @dms_exp) and SN_filter==1')], axis=0).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T15:12:25.068460Z","iopub.execute_input":"2023-12-22T15:12:25.068702Z","iopub.status.idle":"2023-12-22T15:12:30.924612Z","shell.execute_reply.started":"2023-12-22T15:12:25.068677Z","shell.execute_reply":"2023-12-22T15:12:30.923756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dms.experiment_type.unique()","metadata":{"execution":{"iopub.status.busy":"2023-12-22T15:12:30.925596Z","iopub.execute_input":"2023-12-22T15:12:30.925910Z","iopub.status.idle":"2023-12-22T15:12:30.938304Z","shell.execute_reply.started":"2023-12-22T15:12:30.925880Z","shell.execute_reply":"2023-12-22T15:12:30.937471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_a3 = a3.sequence\ny_a3 = a3.filter(regex='reactivity_[0-9]')#.values.where((a3.filter(regex='reactivity_error*')<1).values, a3.filter(regex='reactivity_[0-9]').values, np.nan )\ny_a3 = y_a3.query('@y_a3.fillna(0).sum(1)!=0')\nx_a3 = x_a3.iloc[y_a3.index].reset_index(drop=True)\ny_a3 = y_a3.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T15:12:30.939299Z","iopub.execute_input":"2023-12-22T15:12:30.939554Z","iopub.status.idle":"2023-12-22T15:12:32.494956Z","shell.execute_reply.started":"2023-12-22T15:12:30.939527Z","shell.execute_reply":"2023-12-22T15:12:32.494111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_dms = dms.sequence\ny_dms = dms.filter(regex='reactivity_[0-9]')#.values(.where(dms.filter(regex='reactivity_error*')<1)\ny_dms = y_dms.query('@y_dms.fillna(0).sum(1)!=0')\nx_dms = x_dms.iloc[y_dms.index].reset_index(drop=True)\ny_dms = y_dms.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T15:12:32.495897Z","iopub.execute_input":"2023-12-22T15:12:32.496177Z","iopub.status.idle":"2023-12-22T15:12:33.804814Z","shell.execute_reply.started":"2023-12-22T15:12:32.496148Z","shell.execute_reply":"2023-12-22T15:12:33.803926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!pip install keras_nlp","metadata":{"execution":{"iopub.status.busy":"2023-12-22T15:12:33.807616Z","iopub.execute_input":"2023-12-22T15:12:33.807929Z","iopub.status.idle":"2023-12-22T15:12:33.811306Z","shell.execute_reply.started":"2023-12-22T15:12:33.807900Z","shell.execute_reply":"2023-12-22T15:12:33.810617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras import Model\n#from keras_nlp.layers import TransformerDecoder, TransformerEncoder, PositionEmbedding\nfrom tensorflow.keras import Sequential\nfrom itertools import repeat\nfrom tensorflow.keras.layers import Dropout\nfrom transformers import TFAutoModel\n\nclass RNA(Model):\n    def __init__(self, hidden_dim):\n        super().__init__()\n        self.encoder = TFAutoModel.from_pretrained('AmelieSchreiber/esm2_t6_8M_UR50D_rna_binding_site_predictor')#Sequential([*repeat(TransformerEncoder(hidden_dim*4, 6, activation='gelu', dropout=0.1, normalize_first=True), 12)])\n        self.dp = Dropout(0.2)\n        self.fc = tf.keras.layers.Dense(1)\n    def call(self, x):\n        x = self.encoder(x).last_hidden_state\n        return tf.squeeze(self.fc(x), -1)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T15:12:33.812041Z","iopub.execute_input":"2023-12-22T15:12:33.812254Z","iopub.status.idle":"2023-12-22T15:12:56.295187Z","shell.execute_reply.started":"2023-12-22T15:12:33.812231Z","shell.execute_reply":"2023-12-22T15:12:56.294122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.layers import TextVectorization\nimport tensorflow as tf\n\ntokenizer = TextVectorization(\n    output_mode='int',\n    ngrams=1,\n    output_sequence_length=433,\n    split='character',\n    vocabulary=['a', 'c', 'g', 'u']\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T15:12:56.296423Z","iopub.execute_input":"2023-12-22T15:12:56.297409Z","iopub.status.idle":"2023-12-22T15:12:56.323723Z","shell.execute_reply.started":"2023-12-22T15:12:56.297373Z","shell.execute_reply":"2023-12-22T15:12:56.322903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\ndef loss_fn(labels, targets):\n    labels_mask = tf.math.is_nan(labels)\n    labels = tf.where(labels_mask, tf.zeros_like(labels), labels)\n    mask_count = tf.math.reduce_sum(tf.where(labels_mask, tf.zeros_like(labels), tf.ones_like(labels)))\n    loss = tf.math.abs(labels - targets)\n    loss = tf.where(labels_mask, tf.zeros_like(loss), loss)\n    loss = tf.math.reduce_sum(loss)/(mask_count if mask_count != 0.0 else 1.0)\n    return loss","metadata":{"execution":{"iopub.status.busy":"2023-12-22T15:12:56.324790Z","iopub.execute_input":"2023-12-22T15:12:56.325036Z","iopub.status.idle":"2023-12-22T15:12:56.330412Z","shell.execute_reply.started":"2023-12-22T15:12:56.325011Z","shell.execute_reply":"2023-12-22T15:12:56.329586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport tensorflow as tf\nfrom tensorflow.keras import Model, Sequential\nfrom tensorflow.keras.layers import Input, Embedding, Dense, ReLU, Flatten, Softmax, SimpleRNN, LSTM, MultiHeadAttention\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Layer, LayerNormalization, Dense, SimpleRNNCell, RNN, LSTM, Bidirectional, LSTMCell\nfrom sklearn.model_selection import KFold\nimport numpy as np\n\nwith strategy.scope():\n    model_a3 = RNA(192)\n    model_a3.compile(\n                optimizer=tf.keras.optimizers.Adam(learning_rate=5e-4),\n                loss=loss_fn,\n            )\n\n    train_dataset = tf.data.Dataset.from_tensor_slices((x_a3.values, y_a3.values)).batch(128).map(lambda x, y: (tokenizer(x), tf.clip_by_value(y, 0, 1)))\n    train_dataset = train_dataset.shuffle(train_dataset.cardinality())\n    train_split = train_dataset.take(int(len(train_dataset)*0.8))\n    val_split = train_dataset.skip(int(len(train_dataset)*0.8)).take(int(len(train_dataset)*0.2))\n    model_a3.fit(train_split, validation_data=val_split, epochs=25, batch_size=128)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T15:12:56.331333Z","iopub.execute_input":"2023-12-22T15:12:56.331579Z","iopub.status.idle":"2023-12-22T16:37:38.195163Z","shell.execute_reply.started":"2023-12-22T15:12:56.331554Z","shell.execute_reply":"2023-12-22T16:37:38.193735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport tensorflow as tf\nfrom tensorflow.keras import Model, Sequential\nfrom tensorflow.keras.layers import Input, Embedding, Dense, ReLU, Flatten, Softmax, SimpleRNN, LSTM, MultiHeadAttention\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Layer, LayerNormalization, Dense, SimpleRNNCell, RNN, LSTM, Bidirectional, LSTMCell\nfrom sklearn.model_selection import KFold\nimport numpy as np\n\nwith strategy.scope():\n    model_dms = RNA(192)\n    model_dms.compile(\n                optimizer=tf.keras.optimizers.Adam(learning_rate=5e-4),\n                loss=loss_fn,\n            )\n    train_dataset = tf.data.Dataset.from_tensor_slices((x_dms.values, y_dms.values)).batch(128).map(lambda x, y: (tokenizer(x), tf.clip_by_value(y, 0, 1)))\n    train_dataset = train_dataset.shuffle(train_dataset.cardinality())\n\n    train_split = train_dataset.take(int(len(train_dataset)*0.8))\n    val_split = train_dataset.skip(int(len(train_dataset)*0.8)).take(int(len(train_dataset)*0.2))\n    model_dms.fit(train_split, validation_data=val_split, epochs=25, batch_size=128)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T16:37:38.197650Z","iopub.execute_input":"2023-12-22T16:37:38.198311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.layers import TextVectorization\nimport tensorflow as tf\n\ntokenizer = TextVectorization(\n    output_mode='int',\n    ngrams=1,\n    output_sequence_length=457,\n    split='character',\n    vocabulary=['a', 'c', 'g', 'u']\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = read_csv('/kaggle/input/stanford-ribonanza-rna-folding/test_sequences.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    predictions_a3 = model_a3.predict(tf.data.Dataset.from_tensor_slices((test.sequence)).batch(128).map(lambda x: tokenizer(x)))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    predictions_dms = model_dms.predict(tf.data.Dataset.from_tensor_slices((test.sequence)).batch(128).map(lambda x: tokenizer(x)))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lens = test.sequence.str.len()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy\npredictions_a3_by_length = []\nfor i in range(lens.size):\n    x = numpy.reshape(predictions_a3[i, :lens[i]], (-1, 1))\n    x = numpy.clip(x, 0, 1)\n    #x[:26] = 0\n    #x[:-21] = 0\n    predictions_a3_by_length.append(x)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions_a3 = numpy.concatenate(predictions_a3_by_length, 0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy\npredictions_dms_by_length = []\nfor i in range(lens.size):\n    x = numpy.reshape(predictions_dms[i, :lens[i]], (-1, 1))\n    x = numpy.clip(x, 0, 1)\n    #x[:26] = 0\n    #x[:-21] = 0\n    predictions_dms_by_length.append(x)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions_dms = numpy.concatenate(predictions_dms_by_length, 0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions_dms.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pandas import DataFrame\nsubmission = DataFrame({'id':np.arange(0, 269796671, 1), 'reactivity_DMS_MaP': predictions_dms[:, 0], 'reactivity_2A3_MaP': predictions_a3[:, 0]})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.reactivity_2A3_MaP.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.reactivity_DMS_MaP.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pyarrow","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_parquet('submission.parquet', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}