{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":51294,"databundleVersionId":6923401,"sourceType":"competition"}],"dockerImageVersionId":30588,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\n\ndf = pd.read_csv('/kaggle/input/stanford-ribonanza-rna-folding/train_data.csv')\n\ndf_2A3 = df[df['experiment_type'] == '2A3_MaP']\nprint(len(df_2A3))\ndf_DMS = df[df['experiment_type'] == 'DMS_MaP']\nprint(len(df_DMS))\n\n# df_2A3 = df_2A3.sample(n=int(len(df_2A3)/4), replace=False)\n# df_2A3 = df_2A3.sample(n=15000, replace=False)\n# df_DMS = df_DMS.sample(n=15000, replace=False)\n# print(len(df_2A3))\n# print(len(df_DMS))\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T16:05:41.773424Z","iopub.execute_input":"2023-12-04T16:05:41.773832Z","iopub.status.idle":"2023-12-04T16:07:25.598423Z","shell.execute_reply.started":"2023-12-04T16:05:41.773785Z","shell.execute_reply":"2023-12-04T16:07:25.597237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch import nn, optim\nfrom torch.nn import TransformerEncoder, TransformerEncoderLayer\nimport time\nimport math\nfrom sklearn.model_selection import KFold\nfrom torch.nn.utils.rnn import pad_sequence\nimport numpy as np\nfrom tqdm import tqdm\nfrom torch.optim import lr_scheduler\n\n\n# 辞書の定義：RNAのヌクレオチドを整数にマッピング\nbase_to_int = {'A': 0, 'C': 1, 'G': 2, 'U': 3, 'PAD': 4}\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:30:54.850535Z","iopub.execute_input":"2023-12-04T13:30:54.851506Z","iopub.status.idle":"2023-12-04T13:30:56.87879Z","shell.execute_reply.started":"2023-12-04T13:30:54.851465Z","shell.execute_reply":"2023-12-04T13:30:56.877588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RNADataset(Dataset):\n    def __init__(self, dataframe, max_length, context_size=3):\n        self.dataframe = dataframe\n        self.max_length = max_length\n        self.context_size = context_size\n        self.pad_id = base_to_int['PAD']\n    \n    def __len__(self):\n        return len(self.dataframe)\n    \n    def __getitem__(self, idx):\n        sequence = self.dataframe.iloc[idx, 1]\n\n        padded_sequence = self.encode_seq(sequence)\n        \n        # reactivity_0001 -> reactivity_0206\n        reactivity = self.dataframe.iloc[idx, 6:212].fillna(0).values.astype(float)\n        reactivity_padded = list(reactivity) + [0]*(self.max_length - len(reactivity))\n        reactivity_padded = reactivity_padded[:self.max_length]\n        \n        mask = [1 if i < len(padded_sequence) else 0 for i in range(self.max_length)]\n        \n        return padded_sequence, torch.tensor(reactivity_padded, dtype=torch.float), torch.tensor(mask, dtype=torch.float)\n        \n    \n    def encode_seq(self, sequence):\n        # Convert sequence to list of integers using the dictionary\n        encoded_sequence = [base_to_int[base] for base in sequence]\n        \n        # Pad the sequence to the specified max_length\n        padded_sequence = encoded_sequence + [base_to_int['PAD']] * (self.max_length - len(encoded_sequence))\n        padded_sequence = padded_sequence[:self.max_length]\n\n        return torch.tensor(padded_sequence, dtype=torch.long)\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:30:56.881872Z","iopub.execute_input":"2023-12-04T13:30:56.88237Z","iopub.status.idle":"2023-12-04T13:30:56.898179Z","shell.execute_reply.started":"2023-12-04T13:30:56.882339Z","shell.execute_reply":"2023-12-04T13:30:56.897065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_length = max(df_2A3['sequence'].apply(len))\ndataset = RNADataset(df_2A3, max_length)\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:30:56.899636Z","iopub.execute_input":"2023-12-04T13:30:56.900049Z","iopub.status.idle":"2023-12-04T13:30:56.92815Z","shell.execute_reply.started":"2023-12-04T13:30:56.899996Z","shell.execute_reply":"2023-12-04T13:30:56.927098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset[0]","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:30:56.929416Z","iopub.execute_input":"2023-12-04T13:30:56.929807Z","iopub.status.idle":"2023-12-04T13:30:56.952036Z","shell.execute_reply.started":"2023-12-04T13:30:56.929774Z","shell.execute_reply":"2023-12-04T13:30:56.951071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"padded_sequence, reactivity_padded, mask_padded = dataset[0]\npadded_sequence.shape, reactivity_padded.shape, mask_padded.shape","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:30:56.974545Z","iopub.execute_input":"2023-12-04T13:30:56.975099Z","iopub.status.idle":"2023-12-04T13:30:56.986892Z","shell.execute_reply.started":"2023-12-04T13:30:56.975049Z","shell.execute_reply":"2023-12-04T13:30:56.985647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# 位置エンコーディングのクラス\nclass PositionalEncoding(nn.Module):\n    def __init__(self, d_model, dropout=0.6, max_len=5000):\n        super(PositionalEncoding, self).__init__()\n        self.dropout = nn.Dropout(p=dropout)\n        position = torch.arange(max_len).unsqueeze(1)\n        div_term = torch.exp(torch.arange(0, d_model, 2) * -(math.log(10000.0) / d_model))\n        pe = torch.zeros(max_len, 1, d_model)\n        pe[:, 0, 0::2] = torch.sin(position * div_term)\n        pe[:, 0, 1::2] = torch.cos(position * div_term)\n        self.register_buffer('pe', pe)\n\n    def forward(self, x):\n        x = x + self.pe[:x.size(0)]\n        return self.dropout(x)\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:30:56.998412Z","iopub.execute_input":"2023-12-04T13:30:56.998943Z","iopub.status.idle":"2023-12-04T13:30:57.010818Z","shell.execute_reply.started":"2023-12-04T13:30:56.998907Z","shell.execute_reply":"2023-12-04T13:30:57.009888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# トランスフォーマーモデルのクラス\nclass TransformerModel(nn.Module):\n    def __init__(self, ntoken, ninp, nhead, nhid, nlayers, dropout=0.6):\n        super(TransformerModel, self).__init__()\n        self.model_type = 'Transformer'\n        self.pos_encoder = PositionalEncoding(ninp, dropout)\n\n        # 隠れ層のサイズ、マルチヘッドアテンションのヘッド数、層の数を増やす\n        encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout)\n        self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)\n\n        self.encoder = nn.Embedding(ntoken, ninp)\n        self.ninp = ninp\n        self.decoder = nn.Linear(ninp, 1)\n        self.init_weights()\n\n    def init_weights(self):\n        initrange = 0.1\n        self.encoder.weight.data.uniform_(-initrange, initrange)\n        self.decoder.bias.data.zero_()\n        self.decoder.weight.data.uniform_(-initrange, initrange)\n\n    def forward(self, src):\n        src = self.encoder(src) * math.sqrt(self.ninp)\n        src = self.pos_encoder(src)\n        output = self.transformer_encoder(src)\n        output = self.decoder(output)\n        return output.squeeze(-1)\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:30:57.012142Z","iopub.execute_input":"2023-12-04T13:30:57.013254Z","iopub.status.idle":"2023-12-04T13:30:57.026084Z","shell.execute_reply.started":"2023-12-04T13:30:57.013205Z","shell.execute_reply":"2023-12-04T13:30:57.02517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 損失関数：マスク付きMSE\nclass MaskedHuberLoss(nn.Module):\n    def __init__(self, delta=1.0):\n        super(MaskedHuberLoss, self).__init__()\n        self.delta = delta\n\n    def forward(self, input, target, mask):\n        clipped_target = torch.clip(target, min=0, max=1)\n\n        loss = torch.where(torch.abs(input - clipped_target) < self.delta,\n                           0.5 * (input - clipped_target) ** 2,\n                           self.delta * (torch.abs(input - clipped_target) - 0.5 * self.delta))\n        loss = loss * mask\n        return loss.sum() / mask.sum()\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:30:57.027497Z","iopub.execute_input":"2023-12-04T13:30:57.027921Z","iopub.status.idle":"2023-12-04T13:30:57.038399Z","shell.execute_reply.started":"2023-12-04T13:30:57.027887Z","shell.execute_reply":"2023-12-04T13:30:57.037403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ハイパーパラメータ\nntokens = len(base_to_int)  # ボキャブラリサイズ\nemsize = 300  # 埋め込み次元を300に増やす\nnhid = 300   # 隠れ層のサイズを300に増やす\nnlayers = 2  # トランスフォーマー内の層の数を6に増やす\nnhead = 100    # マルチヘッドアテンションのヘッド数を6に増やす\ndropout = 0.6  # ドロップアウト値を0.6に増やす\nweight_decay = 0.0001\n\n# データの読み込みと前処理\nmax_length = max(df_2A3['sequence'].apply(len))\ndataset = RNADataset(df_2A3, max_length)\n\n# クロスバリデーションの設定\nkf = KFold(n_splits=3)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:30:57.043184Z","iopub.execute_input":"2023-12-04T13:30:57.043534Z","iopub.status.idle":"2023-12-04T13:30:57.067085Z","shell.execute_reply.started":"2023-12-04T13:30:57.043508Z","shell.execute_reply":"2023-12-04T13:30:57.065946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataloader = DataLoader(dataset, batch_size=128, shuffle=True)\nfor sequences, reactivities, masks in dataloader:\n    print(sequences.shape, reactivities.shape, masks.shape)\n    break\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:30:57.068622Z","iopub.execute_input":"2023-12-04T13:30:57.069636Z","iopub.status.idle":"2023-12-04T13:30:57.274834Z","shell.execute_reply.started":"2023-12-04T13:30:57.069596Z","shell.execute_reply":"2023-12-04T13:30:57.273736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold, (train_index, test_index) in enumerate(kf.split(dataset)):\n    print(f\"Fold {fold + 1}\")\n    train_dataset = torch.utils.data.Subset(dataset, train_index)\n    test_dataset = torch.utils.data.Subset(dataset, test_index)\n    \n    train_dataloader = DataLoader(train_dataset, batch_size=160, shuffle=True)\n    test_dataloader = DataLoader(test_dataset, batch_size=160, shuffle=False)\n\n\n    # モデルと損失関数の初期化\n    model = TransformerModel(ntokens, emsize, nhead, nhid, nlayers, dropout).to(\"cuda\")\n    loss_fn = MaskedHuberLoss()\n    optimizer = optim.Adam(model.parameters(), weight_decay=0.0001, lr=5e-3)\n    \n    # スケジューラーの設定\n    def lambda_epoch(epoch):\n        max_epoch = 3\n        return math.pow((1-epoch/max_epoch), 0.9)\n\n    scheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda_epoch)\n\n\n    # トレーニングループ\n    num_epochs = 3\n    \n    for epoch in range(num_epochs):\n        start_time = time.time()\n        model.train()\n        total_loss = 0.0\n        for batch, (seq, target, mask) in enumerate(train_dataloader):\n            seq = seq.to(\"cuda\")\n            target = target.to(\"cuda\")\n            mask = mask.to(\"cuda\")\n            \n            optimizer.zero_grad()\n            output = model(seq)\n            loss = loss_fn(output, target, mask)\n            loss.backward()\n            optimizer.step()\n            total_loss += loss.item()\n            \n\n            if batch % 300 == 0 and batch > 0:\n                current = batch * len(seq)\n                total = len(train_dataloader.dataset)\n                elapsed = time.time() \n                loss_per_batch = total_loss / batch\n                print(f'| Epoch {epoch+1:3d}/{num_epochs:3d} | {current:5d}/{total:5d} sequences | Mean Loss Batch: {loss_per_batch} | Elapsed time {elapsed - start_time:.2f}s')\n        \n        # 学習率スケジューラーを1回実行\n        scheduler.step()\n    \n    \n        elapsed_time = time.time() - start_time\n        print(f'Epoch {epoch+1:3d} completed in {elapsed_time:.2f}s, Total loss {total_loss:.4f}')\n            \n    torch.save(model.state_dict(), '/kaggle/working/model_state_dict_2A3.pth')\n\n\n    # テストデータでの評価\n    model.eval()\n    total_test_loss = 0\n    with torch.no_grad():\n        for batch, (data, target, mask) in enumerate(test_dataloader):  # 4つの要素をアンパック\n            data = data.to(\"cuda\")\n            target = target.to(\"cuda\")\n            mask = mask.to(\"cuda\")\n            \n            output = model(data)\n            loss = loss_fn(output, target, mask)\n            total_test_loss += loss.item()\n\n        print(f\"Fold {fold + 1} Test Loss: {total_test_loss / len(test_dataloader)}\")\n\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:30:57.276442Z","iopub.execute_input":"2023-12-04T13:30:57.276891Z","iopub.status.idle":"2023-12-04T13:32:58.66334Z","shell.execute_reply.started":"2023-12-04T13:30:57.276853Z","shell.execute_reply":"2023-12-04T13:32:58.662305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:32:58.664666Z","iopub.execute_input":"2023-12-04T13:32:58.664964Z","iopub.status.idle":"2023-12-04T13:32:58.730764Z","shell.execute_reply.started":"2023-12-04T13:32:58.664938Z","shell.execute_reply":"2023-12-04T13:32:58.729619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ハイパーパラメータ\nntokens = len(base_to_int)  # ボキャブラリサイズ\nemsize = 300  # 埋め込み次元を300に増やす\nnhid = 300   # 隠れ層のサイズを300に増やす\nnlayers = 2  # トランスフォーマー内の層の数を6に増やす\nnhead = 100    # マルチヘッドアテンションのヘッド数を6に増やす\ndropout = 0.6  # ドロップアウト値を0.6に増やす\nweight_decay = 0.0001\n\n# データの読み込みと前処理\nmax_length = max(df_DMS['sequence'].apply(len))\ndataset = RNADataset(df_DMS, max_length)\n\n# クロスバリデーションの設定\nkf = KFold(n_splits=3)\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:32:58.732321Z","iopub.execute_input":"2023-12-04T13:32:58.732717Z","iopub.status.idle":"2023-12-04T13:32:58.75527Z","shell.execute_reply.started":"2023-12-04T13:32:58.732685Z","shell.execute_reply":"2023-12-04T13:32:58.754094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold, (train_index, test_index) in enumerate(kf.split(dataset)):\n    print(f\"Fold {fold + 1}\")\n    train_dataset = torch.utils.data.Subset(dataset, train_index)\n    test_dataset = torch.utils.data.Subset(dataset, test_index)\n    \n    train_dataloader = DataLoader(train_dataset, batch_size=160, shuffle=True)\n    test_dataloader = DataLoader(test_dataset, batch_size=160, shuffle=False)\n\n\n    # モデルと損失関数の初期化\n    model = TransformerModel(ntokens, emsize, nhead, nhid, nlayers, dropout).to(\"cuda\")\n    loss_fn = MaskedHuberLoss()\n    optimizer = optim.Adam(model.parameters(), weight_decay=0.0001, lr=5e-3)\n    \n    # スケジューラーの設定\n    def lambda_epoch(epoch):\n        max_epoch = 3\n        return math.pow((1-epoch/max_epoch), 0.9)\n\n    scheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda_epoch)\n    \n    # トレーニングループ\n    num_epochs = 3\n    \n    for epoch in range(num_epochs):\n        start_time = time.time()\n        model.train()\n        total_loss = 0.0\n        for batch, (seq, target, mask) in enumerate(train_dataloader):\n            seq = seq.to(\"cuda\")\n            target = target.to(\"cuda\")\n            mask = mask.to(\"cuda\")\n            \n            optimizer.zero_grad()\n            output = model(seq)\n            loss = loss_fn(output, target, mask)\n            loss.backward()\n            optimizer.step()\n            total_loss += loss.item()\n            \n\n            if batch % 300 == 0 and batch > 0:\n                current = batch * len(seq)\n                total = len(train_dataloader.dataset)\n                elapsed = time.time() \n                loss_per_batch = total_loss / batch\n                print(f'| Epoch {epoch+1:3d}/{num_epochs:3d} | {current:5d}/{total:5d} sequences | Mean Loss Batch: {loss_per_batch} | Elapsed time {elapsed - start_time:.2f}s')\n        \n        # 学習率スケジューラーを1回実行\n        scheduler.step()\n        \n        elapsed_time = time.time() - start_time\n        print(f'Epoch {epoch+1:3d} completed in {elapsed_time:.2f}s, Total loss {total_loss:.4f}')\n            \n    torch.save(model.state_dict(), '/kaggle/working/model_state_dict_DMS.pth')\n\n\n    # テストデータでの評価\n    model.eval()\n    total_test_loss = 0\n    with torch.no_grad():\n        for batch, (data, target, mask) in enumerate(test_dataloader):  # 4つの要素をアンパック\n            data = data.to(\"cuda\")\n            target = target.to(\"cuda\")\n            mask = mask.to(\"cuda\")\n            \n            output = model(data)\n            loss = loss_fn(output, target, mask)\n            total_test_loss += loss.item()\n\n        print(f\"Fold {fold + 1} Test Loss: {total_test_loss / len(test_dataloader)}\")\n\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:32:58.756886Z","iopub.execute_input":"2023-12-04T13:32:58.757338Z","iopub.status.idle":"2023-12-04T13:34:54.869032Z","shell.execute_reply.started":"2023-12-04T13:32:58.757289Z","shell.execute_reply":"2023-12-04T13:34:54.868037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:34:54.870412Z","iopub.execute_input":"2023-12-04T13:34:54.87072Z","iopub.status.idle":"2023-12-04T13:34:54.947105Z","shell.execute_reply.started":"2023-12-04T13:34:54.870687Z","shell.execute_reply":"2023-12-04T13:34:54.945904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"# # データの読み込み\n# df_test = pd.read_csv('/kaggle/input/stanford-ribonanza-rna-folding/test_sequences.csv')\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:34:54.948711Z","iopub.execute_input":"2023-12-04T13:34:54.949086Z","iopub.status.idle":"2023-12-04T13:34:55.027736Z","shell.execute_reply.started":"2023-12-04T13:34:54.949055Z","shell.execute_reply":"2023-12-04T13:34:55.026884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_test.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:34:55.029077Z","iopub.execute_input":"2023-12-04T13:34:55.029853Z","iopub.status.idle":"2023-12-04T13:34:55.041465Z","shell.execute_reply.started":"2023-12-04T13:34:55.029811Z","shell.execute_reply":"2023-12-04T13:34:55.040411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class RNADataset_Inference(Dataset):\n#     def __init__(self, dataframe, max_length, context_size=3):\n#         self.dataframe = dataframe\n#         self.max_length = max_length\n#         self.context_size = context_size\n#         self.pad_id = base_to_int['PAD']\n\n#     def __len__(self):\n#         return len(self.dataframe)\n\n#     def __getitem__(self, idx):\n#         sequence = self.dataframe.iloc[idx, 3]\n\n#         padded_sequence = self.encode_seq(sequence)\n\n#         return padded_sequence\n    \n#     def encode_seq(self, sequence):\n#         # Convert sequence to list of integers using the dictionary\n#         encoded_sequence = [base_to_int[base] for base in sequence]\n        \n#         # Pad the sequence to the specified max_length\n#         padded_sequence = encoded_sequence + [base_to_int['PAD']] * (self.max_length - len(encoded_sequence))\n#         padded_sequence = padded_sequence[:self.max_length]\n\n#         return torch.tensor(padded_sequence, dtype=torch.long)\n    ","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:34:55.042912Z","iopub.execute_input":"2023-12-04T13:34:55.043263Z","iopub.status.idle":"2023-12-04T13:34:55.055048Z","shell.execute_reply.started":"2023-12-04T13:34:55.043236Z","shell.execute_reply":"2023-12-04T13:34:55.054055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# max_length = max(df_test['sequence'].apply(len))\n# dataset_infer = RNADataset_Inference(df_test, max_length)\n# infer_dataloader = DataLoader(dataset_infer, batch_size=512, shuffle=False)\n\n# ntokens = len(base_to_int)  # ボキャブラリのサイズ\n# emsize = 300  # 埋め込みの次元\n# nhid = 300    # 隠れ層のサイズ\n# nlayers = 2   # トランスフォーマー内の層の数\n# nhead = 100   # マルチヘッドアテンションのヘッド数\n# dropout = 0.6 # ドロップアウト率\n\n# # モデルの初期化\n# model_2A3 = TransformerModel(ntokens, emsize, nhead, nhid, nlayers, dropout)\n# model_2A3.load_state_dict(torch.load('/kaggle/working/model_state_dict_2A3.pth'))\n# model_2A3.eval().to(\"cuda\")\n\n# model_DMS = TransformerModel(ntokens, emsize, nhead, nhid, nlayers, dropout)\n# model_DMS.load_state_dict(torch.load('/kaggle/working/model_state_dict_DMS.pth'))\n# model_DMS.eval().to(\"cuda\")","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:37:26.734193Z","iopub.execute_input":"2023-12-04T13:37:26.734622Z","iopub.status.idle":"2023-12-04T13:37:26.813965Z","shell.execute_reply.started":"2023-12-04T13:37:26.734563Z","shell.execute_reply":"2023-12-04T13:37:26.81294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dataset_infer[0]","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:37:29.608916Z","iopub.execute_input":"2023-12-04T13:37:29.609326Z","iopub.status.idle":"2023-12-04T13:37:29.617894Z","shell.execute_reply.started":"2023-12-04T13:37:29.609291Z","shell.execute_reply":"2023-12-04T13:37:29.616914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# batch = next(iter(infer_dataloader))\n# batch","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:36:20.266687Z","iopub.execute_input":"2023-12-04T13:36:20.267074Z","iopub.status.idle":"2023-12-04T13:36:20.334676Z","shell.execute_reply.started":"2023-12-04T13:36:20.267044Z","shell.execute_reply":"2023-12-04T13:36:20.333632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # ids = np.empty(shape = (0,1),dtype=int)\n# preds_2A3 = np.empty(shape = (0,1),dtype=np.float32)\n# preds_DMS = np.empty(shape = (0,1),dtype=np.float32)\n\n\n# for seq in tqdm(infer_dataloader):\n# #     print(seq)\n#     seq = seq.to(\"cuda\")\n#     out_2A3 = model_2A3(seq).detach().cpu().numpy()\n#     out_DMS = model_DMS(seq).detach().cpu().numpy()\n    \n#     clipped_out_2A3 = np.clip(out_2A3, a_min=0,  a_max=1)\n#     clipped_out_DMS = np.clip(out_DMS, a_min=0, a_max=1)\n\n\n#     preds_2A3 = np.append(preds_2A3, clipped_out_2A3)\n#     preds_DMS = np.append(preds_DMS, clipped_out_DMS)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:37:51.150335Z","iopub.execute_input":"2023-12-04T13:37:51.150743Z","iopub.status.idle":"2023-12-04T13:38:36.107825Z","shell.execute_reply.started":"2023-12-04T13:37:51.15071Z","shell.execute_reply":"2023-12-04T13:38:36.106885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission_df = pd.DataFrame({'id':np.arange(0, len(preds_2A3), 1),\"reactivity_2A3_MaP\": preds_2A3,\"reactivity_DMS_MaP\": preds_DMS})\n# submission_df.to_csv('submission.csv', index=False)\n\n# submission_df.head(20)","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:38:36.109706Z","iopub.execute_input":"2023-12-04T13:38:36.110418Z","iopub.status.idle":"2023-12-04T13:38:36.135901Z","shell.execute_reply.started":"2023-12-04T13:38:36.110379Z","shell.execute_reply":"2023-12-04T13:38:36.134943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# start_time = time.time()\n# predictions = []\n\n# for sequence in tqdm(df_test['sequence'], desc=f'Processing split {i}'):\n#     # Preprocess sequence\n#     context_padded_sequence = get_context_padded_sequence(sequence)\n#     padded_sequence = context_padded_sequence + [base_to_int['PAD']] * (max_length - len(context_padded_sequence))\n#     sequence_tensor = torch.tensor(padded_sequence, dtype=torch.long).unsqueeze(0)\n\n#     # Prediction\n#     with torch.no_grad():\n#         prediction = model(sequence_tensor)\n\n#     # Get and process prediction results\n#     predicted_reactivity = prediction.squeeze(0).tolist()\n#     predicted_reactivity_trimmed = predicted_reactivity[:len(sequence)]\n#     predictions.append(predicted_reactivity_trimmed)\n    \n#     elapsed_time = time.time() - start_time\n#     estimated_remaining_time = (elapsed_time / (i + 1)) * (num_splits - i - 1)\n#     print(f\"Completed split {i}. Estimated remaining time: {estimated_remaining_time:.2f} seconds\")\n\n\n    \n# submission = pd.DataFrame({'id':np.arange(0, len(concat_preds), 1), 'reactivity_DMS_MaP':concat_preds[:,1], 'reactivity_2A3_MaP':concat_preds[:,0]})\n# submission.to_csv('submission.csv', index=False)\n# submission.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-04T13:35:02.007849Z","iopub.status.idle":"2023-12-04T13:35:02.008284Z","shell.execute_reply.started":"2023-12-04T13:35:02.008087Z","shell.execute_reply":"2023-12-04T13:35:02.008108Z"},"trusted":true},"execution_count":null,"outputs":[]}]}