{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\nfrom torch.nn.utils.rnn import pad_sequence\nimport torch\nimport pandas as pd\nimport numpy as np\nimport sys\nimport pprint\n\nimport os\n# os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\nfrom pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping, TQDMProgressBar\nimport torch\nfrom torch.utils.data import DataLoader,Subset\n\nimport yaml\nimport pytorch_lightning as pl\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data import random_split\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torchmetrics.regression import MeanAbsoluteError\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nimport pytorch_lightning as pl\nfrom torch.optim import AdamW\n\n# from rnadataset import RiboDataset,CustomCollate\n# from model import LightningModel","metadata":{"_uuid":"84db2287-0e98-4016-926b-9be0093bee19","_cell_guid":"9f633373-0b7f-42b5-9459-416c3d07e4b2","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-16T19:56:38.483796Z","iopub.execute_input":"2023-10-16T19:56:38.484204Z","iopub.status.idle":"2023-10-16T19:56:52.938293Z","shell.execute_reply.started":"2023-10-16T19:56:38.484169Z","shell.execute_reply":"2023-10-16T19:56:52.937431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DMS_MaP \nclass RiboDataset(torch.utils.data.Dataset):\n    experiments = ['DMS_MaP','2A3_MaP','2A3_MaP_DMS_MaP']\n    seq_elements = ['A','C','G','U']\n    nanValue=-1000\n    def __init__(self, data_path, exp_type ='2A3_MaP',maxSeqLen=206,test=False,min_max=[0,1]):\n        self.test=test\n        self.maxSeqLen=maxSeqLen\n        self.min_max=min_max\n        assert exp_type in self.experiments, f'error experiment_type must be in {self.experiments}'\n        self.exp_type = exp_type\n        self.df_keys=['sequence','experiment_type']\n        self.df_keys.extend([f'reactivity_{str(i).zfill(4)}' for i in range(1,maxSeqLen+1)])\n#         print(self.reactivity_keys)\n        if not self.test:\n            self.df = pd.read_csv(data_path,usecols=self.df_keys)\n            if exp_type!=self.experiments[2]:\n                self.df = self.df[self.df['experiment_type']==exp_type]\n            self.df[self.df_keys[2:]] =self.df[self.df_keys[2:]].fillna(self.nanValue)\n        else:\n            self.df = pd.read_csv(data_path,usecols=['id_min','id_max','sequence'])\n\n        # if test:\n        #     self.df = self.df.iloc[:10000]\n#         print(self.df.drop_duplicates(subset=self.reactivity_keys[2:]))\n    \n    \n    def get_unique(self):\n        unique=[]\n        for i in range(1,self.maxSeqLen+1):    \n             unique.append(self.df[f'reactivity_{str(i).zfill(4)}'].unique())\n        return unique\n    def get_min_max(self):\n        min_max=[]\n        for i in range(1,self.maxSeqLen+1):\n            x =self.df[f'reactivity_{str(i).zfill(4)}']\n            min =  x.min()  \n            max =  x.max()\n            min_max.append((min,max))\n            \n        min_max =np.array(min_max)\n        print(min_max.min(),min_max.max())\n        return min_max\n    def __getitem__(self, index):\n        sample = self.df.iloc[index]\n        if not self.test: \n            input_seq = sample['sequence']\n            \n            \n            input_vector = self.process_seq(input_seq)\n        \n            exp_type=sample['experiment_type']\n            if exp_type==self.exp_type[0]:\n                exp_type = 1 \n            else:\n                exp_type = -1 \n            reactivity_vector =np.array(sample[self.df_keys[2:]],dtype=float).squeeze()\n            # reactivity_vector=np.expand_dims(reactivity_vector,axis=0)\n            valid_indx= reactivity_vector!=self.nanValue\n            reactivity_vector=reactivity_vector[valid_indx]\n            reactivity_vector =reactivity_vector.clip(self.min_max[0],self.min_max[1])\n            input_vector=input_vector[valid_indx]\n        \n            assert input_vector.shape==reactivity_vector.shape, f'Error the input vector and reactivity vector should have the same shape got{input_vector.shape},{reactivity_vector.shape}'\n            reactivity_vector = torch.from_numpy(reactivity_vector).to(torch.float32)\n            input_vector = torch.from_numpy(input_vector).to(torch.float32)*exp_type      \n            \n            return input_vector, reactivity_vector, len(input_vector)\n        else:\n            input_seq = sample['sequence']\n            id_min = sample['id_min']\n            id_max = sample['id_max']\n            input_vector = self.process_seq(input_seq)\n            input_vector = torch.from_numpy(input_vector).to(torch.float32)   \n            return input_vector, len(input_vector)\n\n    def process_seq(self,input_seq):\n        # exp_type = 1 if exp_type== self.experiments[0] else -1\n        seq_len=len(input_seq)\n        input_seq = np.array(list(input_seq))\n        for i, c in enumerate(self.seq_elements):\n            input_seq[input_seq==c]=i+1\n        \n        # input_seq= np.expand_dims(input_seq,axis=0).astype('uint8')\n        if not self.test:\n            input_seq=np.pad(input_seq.astype('uint8'),(0,self.maxSeqLen-seq_len))\n        else:\n            input_seq = input_seq.astype('uint8')\n        \n        return input_seq\n    def __len__(self):\n        return len(self.df)\n\n\n\n    def get_samples(self, num_samples):\n        samples = []\n        for i in range(num_samples):\n            samples.append(self.__getitem__(i))\n        return samples\n\nclass CustomCollate:\n    def __init__(self, seq_pad_idx,reac_pad_idx):\n        self.seq_pad_idx = seq_pad_idx\n        self.reac_pad_idx = reac_pad_idx\n\n    def __call__(self, batch):\n        # Sort the batch w.r.t source length in decreasing order as sorted batch\n        # is required in packed padded sequence function.\n        batch = sorted(batch, key=lambda x: -x[-1])\n        src = [item[0] for item in batch]\n        src = pad_sequence(src, batch_first=True, padding_value=self.seq_pad_idx)\n        src=src.unsqueeze(1)\n        targets = [item[1] for item in batch]\n        # exp_type = [item[2] for item in batch]\n\n        targets = pad_sequence(targets, batch_first=True, padding_value=self.reac_pad_idx)\n        targets=targets.unsqueeze(1)\n        masks=[]\n        max_len=batch[0][-1]\n        masks= np.zeros((len(batch),max_len))\n        for i, item in enumerate(batch):\n            masks[i,0:item[-1]]=1\n\n        masks = torch.tensor(masks)\n        masks=masks.unsqueeze(1).to(torch.float32)\n        # exp_type = torch.tensor(exp_type).unsqueeze(1).unsqueeze(1).to(torch.float32)\n        \n       \n        return src, targets, masks\n\nclass CustomCollateTest:\n    def __init__(self, seq_pad_idx):\n        self.seq_pad_idx = seq_pad_idx\n\n\n    def __call__(self, batch):\n        # Sort the batch w.r.t source length in decreasing order as sorted batch\n        # is required in packed padded sequence function.\n        batch = sorted(batch, key=lambda x: -x[-1])\n        src = [item[0] for item in batch]\n        src = pad_sequence(src, batch_first=True, padding_value=self.seq_pad_idx)\n        src=src.unsqueeze(1)\n        masks=[]\n        max_len=batch[0][-1]\n        masks= np.zeros((len(batch),max_len))\n        for i, item in enumerate(batch):\n            masks[i,0:item[-1]]=1\n\n        masks = torch.tensor(masks)\n        masks=masks.unsqueeze(1).to(torch.float32)\n        xlen = [item[-1] for item in batch]\n        # exp_type = torch.tensor(exp_type).unsqueeze(1).unsqueeze(1).to(torch.float32)\n        \n       \n        return src, masks,xlen","metadata":{"_uuid":"c41a878c-fd44-4b2e-a0f7-b0bdd0e966c6","_cell_guid":"765ea1ca-2f80-4cf8-9cf5-530b6bb8d702","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-16T19:56:52.940374Z","iopub.execute_input":"2023-10-16T19:56:52.940712Z","iopub.status.idle":"2023-10-16T19:56:52.961399Z","shell.execute_reply.started":"2023-10-16T19:56:52.940681Z","shell.execute_reply":"2023-10-16T19:56:52.960572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Conv1DBlock(torch.nn.Module):\n    def __init__(self,layers=[],useRes=True,masked=True):\n        \"\"\"\n        In the constructor we instantiate four parameters and assign them as\n        member parameters.\n        \"\"\"\n        super().__init__()\n        self.useRes=useRes\n        self.masked = masked\n        self.conv1Dlayers = nn.ModuleList()\n        for idx , layer in enumerate(layers):\n            con1DLayer=[]\n            if idx%2==0 and idx!=0:\n                con1DLayer.append(nn.BatchNorm1d(layer[0]))\n            con1DLayer.append(nn.Conv1d(*layer, stride=1,padding='same'))\n            con1DLayer.append(nn.ReLU())\n            self.conv1Dlayers.append(nn.Sequential(*con1DLayer))\n            \n            \n        # self.conv1DBolk =nn.Sequential(*conv1Dlayers)\n\n    def forward(self, input):\n        \"\"\"\n        In the forward function we accept a Tensor of input data and we must return\n        a Tensor of output data. We can use Modules defined in the constructor as\n        well as arbitrary operators on Tensors.\n        \"\"\"\n        x,mask = input\n        out=x\n        for layer in self.conv1Dlayers:\n            # if self.masked :\n            #     out=out*mask\n            out=layer(out)\n            if self.useRes:\n                out=out+x\n            if self.masked :\n                out = out*mask\n        return out\n# layers = [[in_ch,out_ch,kernal_size],[]]\nclass Conv1DPyramid(torch.nn.Module):\n    def __init__(self,branches=[],masked=True,min_max=[0,1]):\n        \"\"\"\n        In the constructor we instantiate four parameters and assign them as\n        member parameters.\n        \"\"\"\n        super().__init__()\n        self.min_max=min_max\n        self.masked=masked\n        self.normLayer = nn.BatchNorm1d(1)\n        self.conv1DBranches = nn.ModuleList()\n\n        for idx , branch in enumerate(branches):\n            layers = [[64,64,branch] for _ in range(10)]\n            layers[0][0]=1\n            self.conv1DBranches.append(Conv1DBlock(layers))\n    \n        self.head = nn.ModuleList()\n        layer1 = [[32,32,5] for _ in range(5)]\n        layer1[0][0]=int(len(branches)*64)\n        self.head.append(Conv1DBlock(layer1,useRes=False))\n        layer1 = [[32,32,5] for _ in range(5)]\n        self.head.append(Conv1DBlock(layer1,useRes=False))\n        layer1 = [[1,1,3] for _ in range(2)]\n        layer1[0][0]=32\n        self.head.append(Conv1DBlock(layer1,useRes=False))\n        self.outLayer =nn.Hardtanh(min_val=self.min_max[0], max_val=self.min_max[1])\n    def forward(self, input):\n        \"\"\"\n        In the forward function we accept a Tensor of input data and we must return\n        a Tensor of output data. We can use Modules defined in the constructor as\n        well as arbitrary operators on Tensors.\n        \"\"\"\n        x,mask=input\n        \n\n        x=self.normLayer(x)\n        if self.masked:\n            x=x*mask\n        feature_maps=[]\n        for branch in self.conv1DBranches:\n            out = branch((x,mask))\n            # if self.masked:\n            #     out=out*mask\n            feature_maps.append(out)\n        out = torch.cat(feature_maps,dim=1)\n        for i,layer in enumerate(self.head):\n            out=layer((out,mask))\n            if self.masked:\n                out=out*mask \n            #adding skipp connection \n            if i in [0,1]:\n                out=out+x\n            if self.masked:\n                out=out*mask \n        out = self.outLayer(out)\n        if self.masked:\n                out=out*mask\n        return out\n\n\nclass LightningModel(pl.LightningModule):\n    def __init__(self, config):\n        super().__init__()\n        self.config =config\n\n        self.model = Conv1DPyramid(config['kernels'],min_max=config['min_max'])\n        self.loss_module = nn.MSELoss(reduction = 'mean')\n        self.val_mae = MeanAbsoluteError()\n\n    def forward(self, x):\n        return self.model(x)\n\n    def training_step(self, batch, batch_idx):\n        features, true_labels ,masks = batch\n        logits = self((features,masks))\n        # print(logits[0])\n\n        true_labels=true_labels*masks\n        logits=logits*masks\n        loss = self.loss_module(logits, true_labels)\n        self.log(\"train_loss\", loss, prog_bar=True)\n        # logits=F.hardtanh(logits, min_val=0., max_val=1.)*masks\n        # print(logits[0])\n        return loss  # this is passed to the optimizer for training\n\n    def validation_step(self, batch, batch_idx):\n        features, true_labels ,masks = batch\n        logits = self((features,masks))\n        # print(logits[0])\n        true_labels=F.hardtanh(true_labels, min_val=0., max_val=1.)*masks\n        logits=F.hardtanh(logits, min_val=0., max_val=1.)*masks\n        # print(logits[0])\n\n        loss  = self.loss_module(logits, true_labels)\n        # loss_mae = F.l1_loss(logits, true_labels)\n        self.val_mae.update(logits, true_labels)\n        self.log(\"val_loss\", loss, prog_bar=True)\n        self.log(\"val_mae\", self.val_mae, prog_bar=True)\n    def configure_optimizers(self):\n        optimizer = AdamW(self.parameters(), **self.config[\"optimizer_params\"])\n\n        if self.config[\"scheduler\"][\"name\"] == \"CosineAnnealingLR\":\n            scheduler = CosineAnnealingLR(\n                optimizer,\n                **self.config[\"scheduler\"][\"params\"][\"CosineAnnealingLR\"],\n            )\n            lr_scheduler_dict = {\"scheduler\": scheduler, \"interval\": \"step\"}\n            return {\"optimizer\": optimizer, \"lr_scheduler\": lr_scheduler_dict}\n        return optimizer","metadata":{"_uuid":"e944b9da-226c-40b0-8a3c-1eebf664d4aa","_cell_guid":"56403982-370b-42eb-899f-7e38c2bfc6a7","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-16T19:56:52.962965Z","iopub.execute_input":"2023-10-16T19:56:52.963266Z","iopub.status.idle":"2023-10-16T19:56:52.986222Z","shell.execute_reply.started":"2023-10-16T19:56:52.963239Z","shell.execute_reply":"2023-10-16T19:56:52.985254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile config.yaml\nbase_dir: \"./\"\ndata_path: \"./train_data.csv\"\noutput_dir: \"models\"\nsaved_model_path: \"\"\n\nseed: 400\ntrain_bs: 8\nvalid_bs: 16\nworkers: 1\nmin_max: [0,1] # -130,130\n\nprogress_bar_refresh_rate: 1\n\nearly_stop:\n    monitor: \"val_loss\"\n    mode: \"min\"\n    patience: 10\n    verbose: 1\n\ntrainer:\n    max_epochs: 200 #26\n    min_epochs: 200 #26\n    enable_progress_bar: True\n    precision: \"16-mixed\"\n    devices: 1\n\n\noptimizer_params:\n    lr: 0.0001 #5\n    weight_decay: 0.03 # #0.01, 0.02\nscheduler:\n    name: \"CosineAnnealingLR\"\n    params:\n        CosineAnnealingLR:\n            T_max: 2\n            eta_min: 1.0e-6\n            last_epoch: -1\n        ReduceLROnPlateau:\n            mode: \"min\"\n            factor: 0.31622776602\n            patience: 4\n            verbose: True","metadata":{"_uuid":"5fa27ae7-47d6-48df-9bc7-506de6163d5e","_cell_guid":"9f6cfa73-1b19-448f-bba6-c07aa277f9c7","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-16T19:56:52.988637Z","iopub.execute_input":"2023-10-16T19:56:52.989543Z","iopub.status.idle":"2023-10-16T19:56:53.003101Z","shell.execute_reply.started":"2023-10-16T19:56:52.989513Z","shell.execute_reply":"2023-10-16T19:56:53.002208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\ndef main():\n    \n    with open(\"config.yaml\", \"r\") as file_obj:\n        config = yaml.safe_load(file_obj)\n    # print ('sys.argv[1]',sys.argv[1])\n    pl.seed_everything(config['seed'])\n    kernels = [i for  i in range(1,10,2)]+[i for i in range(21,50,4)]+[i for i in range(65,100,8)]+[i for i in range(129,256,16)]\n    # print(kernels)\n\n    config['kernels'] = kernels\n\n    data_path='/kaggle/input/stanford-ribonanza-rna-folding/test_sequences.csv'\n    ds= RiboDataset(data_path=data_path,exp_type='2A3_MaP_DMS_MaP',min_max=config['min_max'],test=True)\n\n    test_dl = DataLoader(\n        ds, \n        batch_size=2048,\n        shuffle=True,\n        num_workers=2,\n        collate_fn=CustomCollateTest(seq_pad_idx=0))\n    path_2A3_MaP = \"/kaggle/input/model-check-points/2A3_MaP-val_mae0.1724.ckpt\"\n    path_DMS_MaP = \"/kaggle/input/model-check-points/DMS_MaP-val_mae0.1393.ckpt\"\n    model_2A3_MaP = LightningModel.load_from_checkpoint(path_2A3_MaP,config=config)\n    model_DMS_MaP = LightningModel.load_from_checkpoint(path_DMS_MaP,config=config)\n    model_2A3_MaP.to('cuda:1')\n    model_2A3_MaP.freeze()\n    model_2A3_MaP.eval()\n    model_2A3_MaP.zero_grad()\n    model_DMS_MaP.freeze()\n    model_DMS_MaP.eval()\n    model_DMS_MaP.zero_grad()\n    sub_path = 'submission.csv'\n    # df = pd.DataFrame(columns=['id','reactivity_DMS_MaP','reactivity_2A3_MaP'])\n    pred_DMS_list=[]\n    pred_2A3_list=[]\n    for sample in tqdm(iter(test_dl)):\n        x,mask,xLen=sample\n        # print('.')\n        mask_1=mask.to('cuda:0')\n        x_1=x.to('cuda:0')\n        mask_2=mask.to('cuda:1')\n        x_2=x.to('cuda:1')*-1\n        with torch.no_grad():\n            pred_DMS_MaP=model_DMS_MaP((x_1,mask_1)).squeeze().cpu().numpy()\n            pred_2A3_MaP=model_2A3_MaP((x_2,mask_2)).squeeze().cpu().numpy()        \n        for i, pred_DMS,pred_2A3 in zip(xLen,pred_DMS_MaP,pred_2A3_MaP):\n            \n            \n            pred_DMS=pred_DMS[:i].tolist()\n            pred_2A3=pred_2A3[:i].tolist()\n            pred_DMS_list.extend(pred_DMS)\n            pred_2A3_list.extend(pred_2A3)\n    out={'id':np.arange(len(pred_DMS_list)),'reactivity_DMS_MaP':pred_DMS_list,'reactivity_2A3_MaP':pred_2A3_list}\n    df = pd.DataFrame(out)\n    # df = pd.concat([df,df2 ],axis=0, ignore_index=True)\n    df.to_csv('submission.csv',index=False)\n\n\n\n\nmain()\n","metadata":{"_uuid":"15c7c10b-6c74-4f50-a670-a6df593fcd2f","_cell_guid":"e40fdeba-f2ff-4506-a547-d48ac0454bef","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-16T19:56:53.004389Z","iopub.execute_input":"2023-10-16T19:56:53.005017Z","iopub.status.idle":"2023-10-16T20:04:55.410162Z","shell.execute_reply.started":"2023-10-16T19:56:53.004989Z","shell.execute_reply":"2023-10-16T20:04:55.408495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}