{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":51294,"databundleVersionId":6923401,"sourceType":"competition"}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-28T16:37:31.945172Z","iopub.execute_input":"2023-11-28T16:37:31.945784Z","iopub.status.idle":"2023-11-28T16:37:32.331084Z","shell.execute_reply.started":"2023-11-28T16:37:31.945753Z","shell.execute_reply":"2023-11-28T16:37:32.330257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install transformers","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:37:48.223521Z","iopub.execute_input":"2023-11-28T16:37:48.224201Z","iopub.status.idle":"2023-11-28T16:38:00.543086Z","shell.execute_reply.started":"2023-11-28T16:37:48.224162Z","shell.execute_reply":"2023-11-28T16:38:00.541961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install -U transformers","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:38:00.545546Z","iopub.execute_input":"2023-11-28T16:38:00.546101Z","iopub.status.idle":"2023-11-28T16:38:22.770344Z","shell.execute_reply.started":"2023-11-28T16:38:00.546055Z","shell.execute_reply":"2023-11-28T16:38:22.768784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install transformers[torch]","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:38:22.77288Z","iopub.execute_input":"2023-11-28T16:38:22.773363Z","iopub.status.idle":"2023-11-28T16:38:34.144746Z","shell.execute_reply.started":"2023-11-28T16:38:22.773318Z","shell.execute_reply":"2023-11-28T16:38:34.143087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install memory_profiler","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:38:34.147406Z","iopub.execute_input":"2023-11-28T16:38:34.147908Z","iopub.status.idle":"2023-11-28T16:38:45.531479Z","shell.execute_reply.started":"2023-11-28T16:38:34.147869Z","shell.execute_reply":"2023-11-28T16:38:45.529815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch, gc, random\nfrom transformers.file_utils import is_tf_available, is_torch_available, is_torch_tpu_available\nfrom transformers import AutoTokenizer, AutoModelForSequenceClassification, Trainer, TrainingArguments\n%load_ext memory_profiler\nfrom sklearn.model_selection import train_test_split\nimport pandas as pd\nimport numpy as np\nfrom torch import nn\nfrom torch.utils.data import DataLoader\n","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:38:45.533498Z","iopub.execute_input":"2023-11-28T16:38:45.533835Z","iopub.status.idle":"2023-11-28T16:39:02.641014Z","shell.execute_reply.started":"2023-11-28T16:38:45.533807Z","shell.execute_reply":"2023-11-28T16:39:02.639925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install einops","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:39:25.487061Z","iopub.execute_input":"2023-11-28T16:39:25.48739Z","iopub.status.idle":"2023-11-28T16:39:36.704767Z","shell.execute_reply.started":"2023-11-28T16:39:25.487362Z","shell.execute_reply":"2023-11-28T16:39:36.703674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pytorch_lamb","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:39:51.789294Z","iopub.execute_input":"2023-11-28T16:39:51.789686Z","iopub.status.idle":"2023-11-28T16:40:03.314658Z","shell.execute_reply.started":"2023-11-28T16:39:51.789655Z","shell.execute_reply":"2023-11-28T16:40:03.313565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pytorch_lamb import Lamb","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:40:03.316687Z","iopub.execute_input":"2023-11-28T16:40:03.31705Z","iopub.status.idle":"2023-11-28T16:40:16.469146Z","shell.execute_reply.started":"2023-11-28T16:40:03.31702Z","shell.execute_reply":"2023-11-28T16:40:16.468106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed: int):\n  random.seed(seed)\n  np.random.seed(seed)\n  if is_torch_available():\n      torch.manual_seed(seed)\n      torch.cuda.manual_seed_all(seed)\n\n  if is_tf_available():\n      import tensorflow as tf\n\n      tf.random.set_seed(seed)\n\nset_seed(42)","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:40:20.223255Z","iopub.execute_input":"2023-11-28T16:40:20.223893Z","iopub.status.idle":"2023-11-28T16:40:20.233755Z","shell.execute_reply.started":"2023-11-28T16:40:20.223861Z","shell.execute_reply":"2023-11-28T16:40:20.232913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device ('cuda' if torch.cuda.is_available () else 'cpu')\ndevice","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:40:24.393436Z","iopub.execute_input":"2023-11-28T16:40:24.393839Z","iopub.status.idle":"2023-11-28T16:40:24.402127Z","shell.execute_reply.started":"2023-11-28T16:40:24.393809Z","shell.execute_reply":"2023-11-28T16:40:24.401409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir('/kaggle/input/')","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:40:27.99821Z","iopub.execute_input":"2023-11-28T16:40:27.999129Z","iopub.status.idle":"2023-11-28T16:40:28.005733Z","shell.execute_reply.started":"2023-11-28T16:40:27.99909Z","shell.execute_reply":"2023-11-28T16:40:28.004654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading Data","metadata":{}},{"cell_type":"code","source":"train=pd.read_csv('/kaggle/input/stanford-ribonanza-rna-folding/train_data.csv')","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:40:31.946907Z","iopub.execute_input":"2023-11-28T16:40:31.947282Z","iopub.status.idle":"2023-11-28T16:42:30.100813Z","shell.execute_reply.started":"2023-11-28T16:40:31.947254Z","shell.execute_reply":"2023-11-28T16:42:30.099075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train2=train[train['SN_filter']==1]","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:42:59.194092Z","iopub.execute_input":"2023-11-28T16:42:59.194488Z","iopub.status.idle":"2023-11-28T16:43:00.22354Z","shell.execute_reply.started":"2023-11-28T16:42:59.194451Z","shell.execute_reply":"2023-11-28T16:43:00.222072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_2A3_MaP=train2[train2['experiment_type'] == '2A3_MaP']\nreactivity_columns = df_2A3_MaP.columns[df_2A3_MaP.columns.str.startswith('reactivity_0')]\ndf_2A3_MaP['react_2A3']=df_2A3_MaP.apply(lambda row: row.loc[reactivity_columns].values.astype('float32'), axis=1)\ndf_2A3_MaP = df_2A3_MaP.drop(columns=reactivity_columns).reset_index(drop=True)\n\ndf_DMS_MaP=train2[train2['experiment_type'] == 'DMS_MaP']\nreactivity_columns2 = df_DMS_MaP.columns[df_DMS_MaP.columns.str.startswith('reactivity_0')]\ndf_DMS_MaP['react_DMS']=df_DMS_MaP.apply(lambda row: row.loc[reactivity_columns2].values.astype('float32'), axis=1)\ndf_DMS_MaP = df_DMS_MaP.drop(columns=reactivity_columns2).reset_index(drop=True)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:43:08.421354Z","iopub.execute_input":"2023-11-28T16:43:08.421782Z","iopub.status.idle":"2023-11-28T16:45:45.054513Z","shell.execute_reply.started":"2023-11-28T16:43:08.421747Z","shell.execute_reply":"2023-11-28T16:45:45.053495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def filter_col(data):\n  error_columns = [col for col in data.columns if 'error' in col]\n  data = data.drop(columns=error_columns).reset_index(drop=True)\n  data = data.drop(columns=['reads', \t'signal_to_noise', \t'SN_filter','dataset_name']).reset_index(drop=True)\n  return data","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:45:51.956008Z","iopub.execute_input":"2023-11-28T16:45:51.956468Z","iopub.status.idle":"2023-11-28T16:45:51.962459Z","shell.execute_reply.started":"2023-11-28T16:45:51.956433Z","shell.execute_reply":"2023-11-28T16:45:51.961544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def filter_col2(data, cols_to_filter):\n  data = data.drop(columns=cols_to_filter).reset_index(drop=True)\n  return data","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:45:58.281299Z","iopub.execute_input":"2023-11-28T16:45:58.281734Z","iopub.status.idle":"2023-11-28T16:45:58.28738Z","shell.execute_reply.started":"2023-11-28T16:45:58.281702Z","shell.execute_reply":"2023-11-28T16:45:58.286228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_DMS_MaP=filter_col(df_DMS_MaP)\ndf_2A3_MaP=filter_col(df_2A3_MaP)","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:46:01.311989Z","iopub.execute_input":"2023-11-28T16:46:01.312401Z","iopub.status.idle":"2023-11-28T16:46:01.542285Z","shell.execute_reply.started":"2023-11-28T16:46:01.312371Z","shell.execute_reply":"2023-11-28T16:46:01.540823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merge_df=df_2A3_MaP.merge(df_DMS_MaP, on='sequence_id', how='inner')","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:46:04.250933Z","iopub.execute_input":"2023-11-28T16:46:04.251746Z","iopub.status.idle":"2023-11-28T16:46:04.539986Z","shell.execute_reply.started":"2023-11-28T16:46:04.251712Z","shell.execute_reply":"2023-11-28T16:46:04.539098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merge_df.columns","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:46:09.558361Z","iopub.execute_input":"2023-11-28T16:46:09.558891Z","iopub.status.idle":"2023-11-28T16:46:09.569743Z","shell.execute_reply.started":"2023-11-28T16:46:09.558844Z","shell.execute_reply":"2023-11-28T16:46:09.568734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1= filter_col2(merge_df,['experiment_type_x', 'sequence_y', 'experiment_type_y'])","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:46:14.760737Z","iopub.execute_input":"2023-11-28T16:46:14.761167Z","iopub.status.idle":"2023-11-28T16:46:14.856279Z","shell.execute_reply.started":"2023-11-28T16:46:14.761129Z","shell.execute_reply":"2023-11-28T16:46:14.855282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Specify the number of rows you want in your random chunk\nchunk_size = 5000  # Change this to the desired chunk size\n# Randomly select a chunk of the DataFrame\ndf = df1.sample(n=chunk_size)\n# Specify the file pathway where you want to save the CSV file\n#file_pathway = '/content/drive/MyDrive/Kaggle_competition/chunk3.csv'\ndf=df.reset_index(drop=True)\n\n# Save the DataFrame to a CSV file\n#df.to_csv(file_pathway, index=False)  # Set index=False to exclude the index column from the CSV","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:50:10.74252Z","iopub.execute_input":"2023-11-28T16:50:10.74297Z","iopub.status.idle":"2023-11-28T16:50:10.766725Z","shell.execute_reply.started":"2023-11-28T16:50:10.74292Z","shell.execute_reply":"2023-11-28T16:50:10.765801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# need to replace U with T as we are going to use DNABERT model\ndef replace_U_with_T(s):\n    # s is a string that contains U\n    # return a new string that replaces U with T\n    return s.replace('U', 'T')\n","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:50:15.108551Z","iopub.execute_input":"2023-11-28T16:50:15.109806Z","iopub.status.idle":"2023-11-28T16:50:15.115515Z","shell.execute_reply.started":"2023-11-28T16:50:15.109759Z","shell.execute_reply":"2023-11-28T16:50:15.114468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['sequence']=df['sequence_x'].apply(lambda x: replace_U_with_T(x))","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:50:17.842393Z","iopub.execute_input":"2023-11-28T16:50:17.842775Z","iopub.status.idle":"2023-11-28T16:50:17.855776Z","shell.execute_reply.started":"2023-11-28T16:50:17.842746Z","shell.execute_reply":"2023-11-28T16:50:17.854881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def padding_nan(arr, pad_len):\n  pad_width = (0, pad_len - len(arr)) # The number of values to add before and after the array\n  pad_value = np.nan # The value to use for padding\n  padded_arr = np.pad(arr, pad_width, mode=\"constant\", constant_values=pad_value)\n  return(padded_arr)","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:50:20.405851Z","iopub.execute_input":"2023-11-28T16:50:20.406872Z","iopub.status.idle":"2023-11-28T16:50:20.412192Z","shell.execute_reply.started":"2023-11-28T16:50:20.406835Z","shell.execute_reply":"2023-11-28T16:50:20.411133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# need to do padding to 457 as it is a max length of the RNA in test file see below\ndf['react_2A3_pad']=df['react_2A3'].apply(lambda x: padding_nan(x,457))","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:50:23.871157Z","iopub.execute_input":"2023-11-28T16:50:23.871567Z","iopub.status.idle":"2023-11-28T16:50:24.026088Z","shell.execute_reply.started":"2023-11-28T16:50:23.871535Z","shell.execute_reply":"2023-11-28T16:50:24.025191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['react_DMS_pad']=df['react_DMS'].apply(lambda x: padding_nan(x,457))","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:50:27.762132Z","iopub.execute_input":"2023-11-28T16:50:27.762555Z","iopub.status.idle":"2023-11-28T16:50:27.926986Z","shell.execute_reply.started":"2023-11-28T16:50:27.762515Z","shell.execute_reply":"2023-11-28T16:50:27.925956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X1 = df.sequence.astype(str)\ny1 = df.react_2A3_pad\ny2 = df.react_DMS_pad\n\n\n# Split Data\nX_train2A3, X_test2A3, y_train2A3, y_test2A3 = train_test_split(X1.tolist(), y1, test_size=0.15)\nX_trainDMS, X_testDMS, y_trainDMS, y_testDMS = train_test_split(X1.tolist(), y2, test_size=0.15)\n\n\nx_train2A3 = np.array(X_train2A3)\nx_test2A3 = np.array(X_test2A3)\n\nx_trainDMS = np.array(X_trainDMS)\nx_testDMS = np.array(X_testDMS)\n\n\n\nprint(x_trainDMS.shape)\nprint(y_trainDMS.shape)\nprint(x_trainDMS[0][:50])\n\nprint(x_train2A3.shape)\nprint(y_train2A3.shape)\nprint(x_train2A3[0][:50])","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:50:31.94179Z","iopub.execute_input":"2023-11-28T16:50:31.942223Z","iopub.status.idle":"2023-11-28T16:50:31.96472Z","shell.execute_reply.started":"2023-11-28T16:50:31.942192Z","shell.execute_reply":"2023-11-28T16:50:31.963047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_2A3_train = pd.DataFrame({\"sequence\": x_train2A3, \"react\": y_train2A3}).reset_index(drop=True)\ndf_2A3_test = pd.DataFrame({\"sequence\": x_test2A3, \"react\": y_test2A3}).reset_index(drop=True)\n\ndf_DMS_train = pd.DataFrame({\"sequence\": x_trainDMS, \"react\": y_trainDMS}).reset_index(drop=True)\ndf_DMS_test = pd.DataFrame({\"sequence\": x_testDMS, \"react\": y_testDMS}).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:50:38.735337Z","iopub.execute_input":"2023-11-28T16:50:38.735771Z","iopub.status.idle":"2023-11-28T16:50:38.748873Z","shell.execute_reply.started":"2023-11-28T16:50:38.735735Z","shell.execute_reply":"2023-11-28T16:50:38.747784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test set","metadata":{}},{"cell_type":"code","source":"test_df=pd.read_csv('/kaggle/input/stanford-ribonanza-rna-folding/test_sequences.csv')","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:47:15.966551Z","iopub.execute_input":"2023-11-28T16:47:15.966985Z","iopub.status.idle":"2023-11-28T16:47:24.153473Z","shell.execute_reply.started":"2023-11-28T16:47:15.966935Z","shell.execute_reply":"2023-11-28T16:47:24.152518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['len']=test_df['sequence'].apply(lambda x: len(x))","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:47:25.724223Z","iopub.execute_input":"2023-11-28T16:47:25.725154Z","iopub.status.idle":"2023-11-28T16:47:26.276981Z","shell.execute_reply.started":"2023-11-28T16:47:25.725109Z","shell.execute_reply":"2023-11-28T16:47:26.275792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max(test_df['len']) # as I understand the max len output should be 457","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:47:29.964922Z","iopub.execute_input":"2023-11-28T16:47:29.965355Z","iopub.status.idle":"2023-11-28T16:47:30.091412Z","shell.execute_reply.started":"2023-11-28T16:47:29.965319Z","shell.execute_reply":"2023-11-28T16:47:30.090294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading DNABERT","metadata":{}},{"cell_type":"code","source":"from transformers import AutoTokenizer, AutoModel, AutoModelForPreTraining, BertConfig, BertForPreTraining\n\ntokenizer = AutoTokenizer.from_pretrained(\"zhihan1996/DNABERT-2-117M\", trust_remote_code=True)\nmodel = AutoModel.from_pretrained(\"zhihan1996/DNABERT-2-117M\")","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:47:33.563586Z","iopub.execute_input":"2023-11-28T16:47:33.564109Z","iopub.status.idle":"2023-11-28T16:48:04.729755Z","shell.execute_reply.started":"2023-11-28T16:47:33.564063Z","shell.execute_reply":"2023-11-28T16:48:04.728997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#testing model\nrna=df['sequence'][0]\ninputs = tokenizer(rna, return_tensors = 'pt',padding=True, truncation=True, max_length=38)[\"input_ids\"]\ninputs","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:51:11.82361Z","iopub.execute_input":"2023-11-28T16:51:11.824336Z","iopub.status.idle":"2023-11-28T16:51:11.847794Z","shell.execute_reply.started":"2023-11-28T16:51:11.824296Z","shell.execute_reply":"2023-11-28T16:51:11.847014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#according to vocabulary https://huggingface.co/zhihan1996/DNABERT-2-117M/blob/main/tokenizer.json token for \"[CLS]\": 1 which has index 0\nhidden_state = model(inputs)[0] # pretrained model\nhidden_state[:, 0, :] # or replace by mean pooling","metadata":{"execution":{"iopub.status.busy":"2023-11-28T16:51:30.096563Z","iopub.execute_input":"2023-11-28T16:51:30.097025Z","iopub.status.idle":"2023-11-28T16:51:30.208988Z","shell.execute_reply.started":"2023-11-28T16:51:30.096984Z","shell.execute_reply":"2023-11-28T16:51:30.208004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class RNA_Dataset:\n    def __init__(self, data):\n        self.data = data\n\n    def __len__(self):\n        return self.data.shape[0]\n\n    def __getitem__(self, idx):\n        inputs = tokenizer(self.data.sequence.iloc[idx], return_tensors = 'pt', padding=True, truncation=True, max_length=38)[\"input_ids\"]\n        hidden_state = model(inputs)[0] # pretrained model\n        X = hidden_state[:, 0, :] # or replace by mean pooling\n        y = self.data.react.iloc[idx]\n        return X, y","metadata":{"execution":{"iopub.status.busy":"2023-11-28T13:14:45.3787Z","iopub.execute_input":"2023-11-28T13:14:45.37992Z","iopub.status.idle":"2023-11-28T13:14:45.3863Z","shell.execute_reply.started":"2023-11-28T13:14:45.379834Z","shell.execute_reply":"2023-11-28T13:14:45.38521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Dataset_2A3train=RNA_Dataset(df_2A3_train)\nDataset_2A3test=RNA_Dataset(df_2A3_test)","metadata":{"execution":{"iopub.status.busy":"2023-11-28T13:15:17.561468Z","iopub.execute_input":"2023-11-28T13:15:17.561852Z","iopub.status.idle":"2023-11-28T13:15:17.567085Z","shell.execute_reply.started":"2023-11-28T13:15:17.561821Z","shell.execute_reply":"2023-11-28T13:15:17.565678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Dataset_DMStrain=RNA_Dataset(df_DMS_train)\nDataset_DMStest=RNA_Dataset(df_DMS_test)","metadata":{"execution":{"iopub.status.busy":"2023-11-28T13:15:31.665534Z","iopub.execute_input":"2023-11-28T13:15:31.665899Z","iopub.status.idle":"2023-11-28T13:15:31.670257Z","shell.execute_reply.started":"2023-11-28T13:15:31.665852Z","shell.execute_reply":"2023-11-28T13:15:31.669365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 10\n\n# Create data loaders.\ntrain_dataloader_DMS = DataLoader(Dataset_DMStrain, batch_size=batch_size)\ntest_dataloader_DMS = DataLoader(Dataset_DMStest, batch_size=batch_size)\n\nfor X, y in train_dataloader_DMS:\n    print(X.shape)\n    print(y.shape, y.dtype)\n    break","metadata":{"execution":{"iopub.status.busy":"2023-11-28T13:15:46.830453Z","iopub.execute_input":"2023-11-28T13:15:46.830785Z","iopub.status.idle":"2023-11-28T13:15:47.980942Z","shell.execute_reply.started":"2023-11-28T13:15:46.830761Z","shell.execute_reply":"2023-11-28T13:15:47.980053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 10\n\n# Create data loaders.\ntrain_dataloader = DataLoader(Dataset_2A3train, batch_size=batch_size)\ntest_dataloader = DataLoader(Dataset_2A3test, batch_size=batch_size)\n\nfor X, y in train_dataloader:\n    print(X.shape)\n    print(y.shape, y.dtype)\n    break","metadata":{"execution":{"iopub.status.busy":"2023-11-28T13:16:06.31713Z","iopub.execute_input":"2023-11-28T13:16:06.318077Z","iopub.status.idle":"2023-11-28T13:16:07.332319Z","shell.execute_reply.started":"2023-11-28T13:16:06.318043Z","shell.execute_reply":"2023-11-28T13:16:07.330858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class BertRegressor(torch.nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.layers = nn.Sequential(nn.Linear(768, 512), nn.GELU(),\n                                    nn.Linear(512, 128), nn.GELU(),\n                                    nn.Linear(128, 128), nn.GELU(),\n                                    nn.Linear(128, 457))\n\n    def forward(self, cls_embedding):\n        y = self.layers(cls_embedding)\n        return y","metadata":{"execution":{"iopub.status.busy":"2023-11-28T13:17:17.603971Z","iopub.execute_input":"2023-11-28T13:17:17.60433Z","iopub.status.idle":"2023-11-28T13:17:17.611439Z","shell.execute_reply.started":"2023-11-28T13:17:17.604305Z","shell.execute_reply":"2023-11-28T13:17:17.610158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomMSELoss(nn.Module):\n  def __init__(self):\n    super(CustomMSELoss, self).__init__()\n\n  def forward(self, y_pred, y_true):\n    # Check if the target values are nan\n    nan_mask = torch.isnan(y_true)\n    # Replace the nan values with zeros\n    y_true = torch.where(nan_mask, torch.zeros_like(y_true), y_true)\n    # Calculate the absolute differences between the predicted and true values\n    diff = torch.square(y_pred - y_true)\n    # Mask out the nan values from the differences\n    diff = torch.where(nan_mask, torch.zeros_like(diff), diff)\n    # Calculate the mean absolute error while ignoring the nan values\n    mae = torch.mean(diff)\n    return mae\n","metadata":{"execution":{"iopub.status.busy":"2023-11-28T21:02:15.579604Z","iopub.execute_input":"2023-11-28T21:02:15.579904Z","iopub.status.idle":"2023-11-28T21:02:15.912927Z","shell.execute_reply.started":"2023-11-28T21:02:15.579878Z","shell.execute_reply":"2023-11-28T21:02:15.911707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(dataloader, model, loss_fn, optimizer):\n    size = len(dataloader.dataset)\n    model.train()\n    for batch, (X, y) in enumerate(dataloader):\n        X, y = X.to(device), y.to(device)\n\n        # Compute prediction error\n        pred = model(X)\n        loss = loss_fn(pred, y)\n\n        # Backpropagation\n        loss.backward()\n        optimizer.step()\n        optimizer.zero_grad()\n\n        if batch % 100 == 0:\n            loss, current = loss.item(), (batch + 1) * len(X)\n            print(f\"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]\")","metadata":{"execution":{"iopub.status.busy":"2023-11-28T13:18:00.579951Z","iopub.execute_input":"2023-11-28T13:18:00.580324Z","iopub.status.idle":"2023-11-28T13:18:01.336405Z","shell.execute_reply.started":"2023-11-28T13:18:00.580296Z","shell.execute_reply":"2023-11-28T13:18:01.335538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test(dataloader, model, loss_fn):\n    size = len(dataloader.dataset)\n    num_batches = len(dataloader)\n    model.eval()\n    test_loss, correct = 0, 0\n    with torch.no_grad():\n        for batch, (X, y) in enumerate(dataloader):\n            X, y = X.to(device), y.to(device)\n            pred = model(X)\n            test_loss += loss_fn(pred, y).item()\n\n    test_loss /= num_batches\n\n    print(f\" Avg loss: {test_loss:>8f} \\n\")","metadata":{"execution":{"iopub.status.busy":"2023-11-28T13:18:20.630734Z","iopub.execute_input":"2023-11-28T13:18:20.631152Z","iopub.status.idle":"2023-11-28T13:18:20.637788Z","shell.execute_reply.started":"2023-11-28T13:18:20.631116Z","shell.execute_reply":"2023-11-28T13:18:20.63635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reg_head_2A3= BertRegressor().to(device)","metadata":{"execution":{"iopub.status.busy":"2023-11-28T13:19:26.73859Z","iopub.execute_input":"2023-11-28T13:19:26.739097Z","iopub.status.idle":"2023-11-28T13:19:26.75064Z","shell.execute_reply.started":"2023-11-28T13:19:26.739064Z","shell.execute_reply":"2023-11-28T13:19:26.749352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_fn=CustomMSELoss()\n#optimizer = torch.optim.SGD(reg_head_2A3.parameters(), lr=0.001, momentum=0.9)\noptimizer = Lamb(reg_head_2A3.parameters(), lr=0.001)\noptimizer_bert = Lamb(model.parameters(), lr=0.001)","metadata":{"execution":{"iopub.status.busy":"2023-11-28T13:19:44.453979Z","iopub.execute_input":"2023-11-28T13:19:44.454353Z","iopub.status.idle":"2023-11-28T13:19:44.460019Z","shell.execute_reply.started":"2023-11-28T13:19:44.454326Z","shell.execute_reply":"2023-11-28T13:19:44.458864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 1\nfor t in range(epochs):\n    print(f\"Epoch {t+1}\\n-------------------------------\")\n    train(train_dataloader,reg_head_2A3, loss_fn, optimizer)\n    test(test_dataloader, reg_head_2A3, loss_fn)\n    #test(test_dataloader, model, loss_fn)\nprint(\"Done!\")","metadata":{"execution":{"iopub.status.busy":"2023-11-28T13:20:02.627446Z","iopub.execute_input":"2023-11-28T13:20:02.627805Z","iopub.status.idle":"2023-11-28T13:20:10.645465Z","shell.execute_reply.started":"2023-11-28T13:20:02.627778Z","shell.execute_reply":"2023-11-28T13:20:10.644084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 2\nfor t in range(epochs):\n    print(f\"Epoch {t+1}\\n-------------------------------\")\n    train(train_dataloader,reg_head_2A3, loss_fn, optimizer_bert)\n    test(test_dataloader, reg_head_2A3, loss_fn)\n    #test(test_dataloader, model, loss_fn)\nprint(\"Done!\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 4\nfor t in range(epochs):\n    print(f\"Epoch {t+1}\\n-------------------------------\")\n    train(train_dataloader,reg_head_2A3, loss_fn, optimizer)\n    test(test_dataloader, reg_head_2A3, loss_fn)\n    #test(test_dataloader, model, loss_fn)\nprint(\"Done!\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reg_head_DMS= BertRegressor().to(device)\n#optimizer2=torch.optim.SGD(reg_head_DMS.parameters(), lr=0.001, momentum=0.9)\noptimizer2 = Lamb(reg_head_DMS.parameters(), lr=0.001)","metadata":{"execution":{"iopub.status.busy":"2023-11-28T13:20:39.276949Z","iopub.execute_input":"2023-11-28T13:20:39.277356Z","iopub.status.idle":"2023-11-28T13:20:39.288252Z","shell.execute_reply.started":"2023-11-28T13:20:39.277326Z","shell.execute_reply":"2023-11-28T13:20:39.286589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 5\nfor t in range(epochs):\n    print(f\"Epoch {t+1}\\n-------------------------------\")\n    train(train_dataloader_DMS,reg_head_DMS, loss_fn, optimizer2)\n    test(test_dataloader_DMS, reg_head_DMS, loss_fn)\n    #test(test_dataloader, model, loss_fn)\nprint(\"Done!\")","metadata":{"execution":{"iopub.status.busy":"2023-11-28T13:20:59.187861Z","iopub.execute_input":"2023-11-28T13:20:59.188284Z","iopub.status.idle":"2023-11-28T13:21:06.03201Z","shell.execute_reply.started":"2023-11-28T13:20:59.188251Z","shell.execute_reply":"2023-11-28T13:21:06.030532Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission file","metadata":{}},{"cell_type":"code","source":"#test file was loaded before\ntest_df.head()\ntest_df['sequence']=test_df['sequence'].dropna().astype('str')\ntest_df['sequence']=test_df['sequence'].apply(lambda x: replace_U_with_T(x))","metadata":{"execution":{"iopub.status.busy":"2023-11-28T13:22:50.107387Z","iopub.execute_input":"2023-11-28T13:22:50.107739Z","iopub.status.idle":"2023-11-28T13:22:51.445056Z","shell.execute_reply.started":"2023-11-28T13:22:50.107713Z","shell.execute_reply":"2023-11-28T13:22:51.4441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class test_Dataset:\n    def __init__(self, data):\n        self.data = data\n\n    def __len__(self):\n        return self.data.shape[0]\n\n    def __getitem__(self, idx):\n        inputs = tokenizer(self.data.sequence.iloc[idx], return_tensors = 'pt', padding=True, truncation=True, max_length=38)[\"input_ids\"]\n        hidden_state = model(inputs)[0] # pretrained model\n        X = hidden_state[:, 0, :] # or replace by mean pooling\n        return X\n","metadata":{"execution":{"iopub.status.busy":"2023-11-28T13:23:43.155453Z","iopub.execute_input":"2023-11-28T13:23:43.155815Z","iopub.status.idle":"2023-11-28T13:23:43.162971Z","shell.execute_reply.started":"2023-11-28T13:23:43.155788Z","shell.execute_reply":"2023-11-28T13:23:43.161774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_Data=test_Dataset(test_df)\ntest_loader = DataLoader(test_Data, batch_size=32,num_workers=8, pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-28T13:23:51.604527Z","iopub.execute_input":"2023-11-28T13:23:51.604937Z","iopub.status.idle":"2023-11-28T13:23:51.611392Z","shell.execute_reply.started":"2023-11-28T13:23:51.604904Z","shell.execute_reply":"2023-11-28T13:23:51.60992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# I calculated this in chunks, model is heavy pruning of model needed or acsess to several GPU\nreg_head_2A3.eval()\nreg_head_DMS.eval()\npredict_2A3 = []\npredict_DMS = []\nwith torch.no_grad():\n  for batch,X in enumerate(test_loader):\n    X=X.to(device)\n    predict_2A3.append(reg_head_2A3(X))\n    predict_DMS.append(reg_head_DMS(X))","metadata":{"execution":{"iopub.status.busy":"2023-11-28T13:24:25.227373Z","iopub.execute_input":"2023-11-28T13:24:25.227754Z","iopub.status.idle":"2023-11-28T13:24:35.300596Z","shell.execute_reply.started":"2023-11-28T13:24:25.227726Z","shell.execute_reply":"2023-11-28T13:24:35.299176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"react=predict_DMS[0].squeeze().cpu().numpy()\nfor i in range(1,len(predict_DMS)):\n  react_i=predict_DMS[i].squeeze().cpu().numpy()\n  react=np.concatenate((react_i, react), axis=0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_DMS=[]\nfor i in range(react.shape[0]):\n  pred_DMS.append(react[i][:test_df['len'][i]])\npred_DMS_by_id=np.concatenate(pred_DMS)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"react2=predict_2A3[0].squeeze().cpu().numpy()\nfor i in range(1,len(predict_2A3)):\n  react_i=predict_2A3[i].squeeze().cpu().numpy()\n  react2=np.concatenate((react_i, react2), axis=0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_2A3=[]\nfor i in range(react2.shape[0]):\n  pred_2A3.append(react2[i][:test_df['len'][i]])\npred_2A3_by_id=np.concatenate(pred_2A3)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub=pd.DataFrame({\n        \"id\": np.arange(0, len(pred_DMS_by_id), 1),\n        \"reactivity_DMS_MaP\": pred_DMS_by_id, \"reactivity_2A3_MaP\": pred_2A3_by_id\n    })","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv('/kaggle/working/submission.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]}]}