{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":51294,"databundleVersionId":6923401,"sourceType":"competition"},{"sourceId":7147371,"sourceType":"datasetVersion","datasetId":4126190},{"sourceId":147571371,"sourceType":"kernelVersion"}],"dockerImageVersionId":30559,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"!pip install fastparquet","metadata":{"execution":{"iopub.status.busy":"2023-12-07T14:35:57.031171Z","iopub.execute_input":"2023-12-07T14:35:57.031722Z","iopub.status.idle":"2023-12-07T14:36:08.539307Z","shell.execute_reply.started":"2023-12-07T14:35:57.031682Z","shell.execute_reply":"2023-12-07T14:36:08.538184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport pickle\nimport shutil\nimport math\nimport pandas as pd\nimport gc\nimport os","metadata":{"execution":{"iopub.status.busy":"2023-12-07T14:36:08.540759Z","iopub.execute_input":"2023-12-07T14:36:08.541162Z","iopub.status.idle":"2023-12-07T14:36:12.121135Z","shell.execute_reply.started":"2023-12-07T14:36:08.541121Z","shell.execute_reply":"2023-12-07T14:36:12.119677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"DEBUG = False","metadata":{"execution":{"iopub.status.busy":"2023-12-07T14:36:12.123317Z","iopub.execute_input":"2023-12-07T14:36:12.124670Z","iopub.status.idle":"2023-12-07T14:36:12.129952Z","shell.execute_reply.started":"2023-12-07T14:36:12.124618Z","shell.execute_reply":"2023-12-07T14:36:12.128840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class transformer_block(tf.keras.layers.Layer):\n    def __init__(self, dim, num_heads, feed_forward_dim, rate=0.1):\n        super().__init__()\n        self.att = tf.keras.layers.MultiHeadAttention(num_heads=num_heads, key_dim=dim//num_heads)\n        self.ffn = tf.keras.Sequential(\n            [\n                tf.keras.layers.Dense(feed_forward_dim, activation=\"relu\"),\n                tf.keras.layers.Dense(dim),\n            ]\n        )\n        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n        self.dropout1 = tf.keras.layers.Dropout(rate)\n        self.dropout2 = tf.keras.layers.Dropout(rate)\n        self.supports_masking = True\n\n    def call(self, inputs, training, mask):\n        att_mask = tf.expand_dims(mask, axis=-1)\n        att_mask = tf.repeat(att_mask, repeats=tf.shape(att_mask)[1], axis=-1)\n\n        attn_output = self.att(inputs, inputs, attention_mask = att_mask)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(inputs + attn_output)\n        ffn_output = self.ffn(out1)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        return self.layernorm2(out1 + ffn_output)\n\n\nclass positional_encoding_layer(tf.keras.layers.Layer):\n    def __init__(self, num_vocab=5, maxlen=500, hidden_dim=384):\n        super().__init__()\n        self.hidden_dim = hidden_dim\n        self.pos_emb = self.positional_encoding(maxlen-1, hidden_dim)\n        self.supports_masking = True\n\n    def call(self, x):\n        maxlen = tf.shape(x)[-2]\n        x = tf.math.multiply(x, tf.math.sqrt(tf.cast(self.hidden_dim, tf.float32)))\n        return x + self.pos_emb[:maxlen, :]\n\n    def positional_encoding(self, maxlen, hidden_dim):\n        depth = hidden_dim/2\n        positions = tf.range(maxlen, dtype = tf.float32)[..., tf.newaxis]\n        depths = tf.range(depth, dtype = tf.float32)[np.newaxis, :]/depth\n        angle_rates = tf.math.divide(1, tf.math.pow(tf.cast(10000, tf.float32), depths))\n        angle_rads = tf.linalg.matmul(positions, angle_rates)\n        pos_encoding = tf.concat(\n          [tf.math.sin(angle_rads), tf.math.cos(angle_rads)],\n          axis=-1)\n        return pos_encoding\n    \nclass global_positional_encoding_layer(tf.keras.layers.Layer):\n    def __init__(self, num_vocab=5, maxlen=13, hidden_dim=24):\n        super().__init__()\n        self.hidden_dim = hidden_dim\n        self.pos_emb = self.positional_encoding(maxlen-1, hidden_dim)\n        self.supports_masking = True\n\n    def call(self, x):\n        maxlen = tf.shape(x)[-2]\n        expanded_pos_emb = tf.tile(self.pos_emb, multiples=[tf.shape(x)[0], 1, 1])\n        return tf.concat((x, expanded_pos_emb[:,:maxlen, :]), axis=-1)\n\n    def positional_encoding(self, maxlen, hidden_dim):\n        depth = hidden_dim/2\n        positions = tf.range(maxlen, dtype = tf.float32)[..., tf.newaxis]\n        depths = tf.range(depth, dtype = tf.float32)[np.newaxis, :]/depth\n        angle_rates = tf.math.divide(1, tf.math.pow(tf.cast(10000, tf.float32), depths))\n        angle_rads = tf.linalg.matmul(positions, angle_rates)\n        pos_encoding = tf.concat(\n          [tf.math.sin(angle_rads), tf.math.cos(angle_rads)],\n          axis=-1)\n        pos_encoding = tf.repeat(pos_encoding, repeats=[60,40,40,40,40,40,40,40,40,40,40,40], axis=-2)\n        \n        return pos_encoding[None,...]","metadata":{"execution":{"iopub.status.busy":"2023-12-07T14:36:12.134629Z","iopub.execute_input":"2023-12-07T14:36:12.135467Z","iopub.status.idle":"2023-12-07T14:36:12.160444Z","shell.execute_reply.started":"2023-12-07T14:36:12.135423Z","shell.execute_reply":"2023-12-07T14:36:12.159616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model_slide(hidden_dim = 384, max_len = 206):\n    inp = tf.keras.Input([None])\n\n    x = inp\n\n    x = tf.keras.layers.Embedding(num_vocab, hidden_dim, mask_zero=True)(x)\n    x = positional_encoding_layer(num_vocab=num_vocab, maxlen=81, hidden_dim=hidden_dim)(x)\n\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n\n    model = tf.keras.Model(inp, x)\n\n    return model   ","metadata":{"execution":{"iopub.status.busy":"2023-12-07T14:36:12.161557Z","iopub.execute_input":"2023-12-07T14:36:12.161833Z","iopub.status.idle":"2023-12-07T14:36:12.173229Z","shell.execute_reply.started":"2023-12-07T14:36:12.161810Z","shell.execute_reply":"2023-12-07T14:36:12.172504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_sliding_model(hidden_dim = 384, max_len = 240):\n    inp = tf.keras.Input([max_len])\n\n    model1 = get_model(hidden_dim = 192, max_len = 80)\n\n    inp1 = inp[...,:80]\n    inp2 = inp[...,40:120]\n    inp3 = inp[...,80:160]\n    inp4 = inp[...,120:200]\n    inp5 = inp[...,160:]\n\n    out1 = model1(inp1)\n    out2 = model1(inp2)\n    out3 = model1(inp3)\n    out4 = model1(inp4)\n    out5 = model1(inp5)\n\n    out = tf.concat((out1[:,:60],\n                     out2[:,20:60],\n                     out3[:,20:60],\n                     out4[:,20:60],\n                     out5[:,20:]), axis=-2)\n\n    dr = tf.keras.layers.Dropout(0.5)(out)\n    dn = tf.keras.layers.Dense(2)(dr)\n\n    model = tf.keras.Model(inp, dn)\n    return model      \n","metadata":{"execution":{"iopub.status.busy":"2023-12-07T14:36:12.174259Z","iopub.execute_input":"2023-12-07T14:36:12.174601Z","iopub.status.idle":"2023-12-07T14:36:12.186540Z","shell.execute_reply.started":"2023-12-07T14:36:12.174577Z","shell.execute_reply":"2023-12-07T14:36:12.185817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_max_len = 457\nnum_vocab = 5\n\ndef get_model_orig(hidden_dim = 384, max_len = 206):\n    inp = tf.keras.Input([None])\n    x = inp\n\n    x = tf.keras.layers.Embedding(num_vocab, hidden_dim, mask_zero=True)(x)\n    x = positional_encoding_layer(num_vocab=num_vocab, maxlen=500, hidden_dim=hidden_dim)(x)\n\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n    x = transformer_block(hidden_dim, 6, hidden_dim*4)(x)\n\n    x = tf.keras.layers.Dropout(0.5)(x)\n    x = tf.keras.layers.Dense(2)(x)\n\n    model = tf.keras.Model(inp, x)\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-12-07T14:36:12.187641Z","iopub.execute_input":"2023-12-07T14:36:12.188192Z","iopub.status.idle":"2023-12-07T14:36:12.197857Z","shell.execute_reply.started":"2023-12-07T14:36:12.188160Z","shell.execute_reply":"2023-12-07T14:36:12.196953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"test_sequences_df = pd.read_csv('/kaggle/input/stanford-ribonanza-rna-folding/test_sequences.csv')\ntest_sequences_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-07T14:36:12.198883Z","iopub.execute_input":"2023-12-07T14:36:12.199195Z","iopub.status.idle":"2023-12-07T14:36:16.620216Z","shell.execute_reply.started":"2023-12-07T14:36:12.199172Z","shell.execute_reply":"2023-12-07T14:36:16.619217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sequences = test_sequences_df.sequence.to_numpy()\nencoding_dict = {'A':1, 'C': 2, 'G': 3, 'U': 4}\nencoding_dict","metadata":{"execution":{"iopub.status.busy":"2023-12-07T14:36:16.621585Z","iopub.execute_input":"2023-12-07T14:36:16.621977Z","iopub.status.idle":"2023-12-07T14:36:16.629073Z","shell.execute_reply.started":"2023-12-07T14:36:16.621943Z","shell.execute_reply":"2023-12-07T14:36:16.628209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_len = 240\ntest_sequences_encoded_207 = []\nfor seq in test_sequences[:1335823]:\n    test_sequences_encoded_207.append(\n        np.concatenate([np.asarray([encoding_dict[x] for x in seq]), np.zeros((max_len - len(seq)))]).astype(np.float32))\n    \nmax_len = 307\ntest_sequences_encoded_307 = []\nfor seq in test_sequences[1335823:1337823]:\n    test_sequences_encoded_307.append(\n        np.asarray([encoding_dict[x] for x in seq]).astype(np.float32))\n    \nmax_len = 457\ntest_sequences_encoded_457 = []\nfor seq in test_sequences[1337823:]:\n    test_sequences_encoded_457.append(\n        np.asarray([encoding_dict[x] for x in seq]).astype(np.float32))","metadata":{"execution":{"iopub.status.busy":"2023-12-07T14:36:16.630315Z","iopub.execute_input":"2023-12-07T14:36:16.630881Z","iopub.status.idle":"2023-12-07T14:37:19.927890Z","shell.execute_reply.started":"2023-12-07T14:36:16.630852Z","shell.execute_reply":"2023-12-07T14:37:19.927005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(encodedseq, model):\n    test_ds = tf.data.Dataset.from_tensor_slices(encodedseq)\n    batch_size = 256\n    test_ds = test_ds.batch(batch_size, drop_remainder=False)\n    test_ds = test_ds.prefetch(tf.data.AUTOTUNE)\n    preds = model.predict(test_ds)\n    predlist = [p for p in preds]\n    return predlist","metadata":{"execution":{"iopub.status.busy":"2023-12-07T14:37:19.929300Z","iopub.execute_input":"2023-12-07T14:37:19.929669Z","iopub.status.idle":"2023-12-07T14:37:19.935900Z","shell.execute_reply.started":"2023-12-07T14:37:19.929637Z","shell.execute_reply":"2023-12-07T14:37:19.934838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_len = 240\nmodel_orig = get_sliding_model(hidden_dim = 192,max_len = max_len)\nmodel_orig.load_weights('/kaggle/input/slidingmodelepoch300/model_epoch_299.h5')","metadata":{"execution":{"iopub.status.busy":"2023-12-07T14:37:19.937014Z","iopub.execute_input":"2023-12-07T14:37:19.937266Z","iopub.status.idle":"2023-12-07T14:37:28.842395Z","shell.execute_reply.started":"2023-12-07T14:37:19.937243Z","shell.execute_reply":"2023-12-07T14:37:28.841406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_len = 457\nmodel_flip = get_model_orig(hidden_dim = 192,max_len = max_len)\nmodel_flip.load_weights('/kaggle/input/srrf-transformer-tpu-training/weights/model_epoch_199.h5')","metadata":{"execution":{"iopub.status.busy":"2023-12-07T14:37:28.845906Z","iopub.execute_input":"2023-12-07T14:37:28.846184Z","iopub.status.idle":"2023-12-07T14:37:31.279504Z","shell.execute_reply.started":"2023-12-07T14:37:28.846161Z","shell.execute_reply":"2023-12-07T14:37:31.278597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds207 = predict(test_sequences_encoded_207,model_orig)","metadata":{"execution":{"iopub.status.busy":"2023-12-07T14:37:31.280594Z","iopub.execute_input":"2023-12-07T14:37:31.280866Z","iopub.status.idle":"2023-12-07T14:39:37.356380Z","shell.execute_reply.started":"2023-12-07T14:37:31.280841Z","shell.execute_reply":"2023-12-07T14:39:37.354963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds307 = predict(test_sequences_encoded_307,model_flip)","metadata":{"execution":{"iopub.status.busy":"2023-12-07T14:39:37.357353Z","iopub.status.idle":"2023-12-07T14:39:37.357702Z","shell.execute_reply.started":"2023-12-07T14:39:37.357539Z","shell.execute_reply":"2023-12-07T14:39:37.357554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds457 = predict(test_sequences_encoded_457,model_flip)","metadata":{"execution":{"iopub.status.busy":"2023-12-07T14:39:37.359750Z","iopub.status.idle":"2023-12-07T14:39:37.360543Z","shell.execute_reply.started":"2023-12-07T14:39:37.360260Z","shell.execute_reply":"2023-12-07T14:39:37.360285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = preds207 + preds307 + preds457 ","metadata":{"execution":{"iopub.status.busy":"2023-12-07T14:39:37.362146Z","iopub.status.idle":"2023-12-07T14:39:37.362822Z","shell.execute_reply.started":"2023-12-07T14:39:37.362549Z","shell.execute_reply":"2023-12-07T14:39:37.362572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_processed = []\nfor i, pred in enumerate(preds):\n    preds_processed.append(pred[:len(test_sequences[i])])\nconcat_preds = np.concatenate(preds_processed)","metadata":{"execution":{"iopub.status.busy":"2023-12-07T14:39:37.364104Z","iopub.status.idle":"2023-12-07T14:39:37.364568Z","shell.execute_reply.started":"2023-12-07T14:39:37.364322Z","shell.execute_reply":"2023-12-07T14:39:37.364344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({'id':np.arange(0, len(concat_preds), 1), 'reactivity_DMS_MaP':concat_preds[:,1], 'reactivity_2A3_MaP':concat_preds[:,0]})\nsubmission.to_parquet('submission.parquet', engine = 'fastparquet', index=False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-07T14:39:37.366117Z","iopub.status.idle":"2023-12-07T14:39:37.366428Z","shell.execute_reply.started":"2023-12-07T14:39:37.366275Z","shell.execute_reply":"2023-12-07T14:39:37.366289Z"},"trusted":true},"execution_count":null,"outputs":[]}]}