{"metadata":{"kernelspec":{"display_name":"py37torch112","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.7.10"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":51294,"databundleVersionId":6923401,"sourceType":"competition"},{"sourceId":6900449,"sourceType":"datasetVersion","datasetId":3963795}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# With noise or without, that is the question [0.15462 LB]?\n\n### The idea was to check with which data set the best training result would be obtained. The signal-to-noise ratio was chosen as the variable component. The result was obtained for four variants of the signal-to-noise ratio [1.0, 0.75, 0.50, 0.25].\n\n### The wonderful notebook by `Iafoss` [RNA starter [0.186 LB]](https://www.kaggle.com/code/iafoss/rna-starter-0-186-lb) was used as a starting point.\n\n### The data set used is taken from here: [Generating a Dataset with structure field](https://www.kaggle.com/code/konstantinboyko/generating-a-dataset-with-structure-field)\n\n### The execution results are given at the end of the notebook.\n\n### To receive submission in this notebook, you need to assign the CFG.train variable to False, and write the path to the file with the trained model into the CFG.MODELS array.","metadata":{}},{"cell_type":"code","source":"import os, gc, sys\nimport datetime as dt\nimport numpy as np\nimport sklearn.model_selection as skms\nfrom pathlib import Path\nimport pandas as pd\nimport polars as pl\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data as tud\nimport torch.cuda.amp as tca\nfrom torch.cuda.amp import GradScaler, autocast\nimport fastai\nimport fastai.vision.all as fva\nfrom fastai.vision.all import *","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def print_os_environ(names):\n    for env_name in names:\n        try:\n            print(env_name, '=', os.environ[env_name])\n        except Exception:\n            pass\n\nprint_os_environ(names=['CONTAINER_NAME','NVIDIA_PRODUCT_NAME', 'BUILD_DATE', 'CUDA_VERSION', 'NV_CUDNN_PACKAGE', 'CUBLAS_VERSION', 'CUTENSOR_VERSION',\n      'NVIDIA_TENSORFLOW_VERSION', 'TENSORFLOW_VERSION', 'TENSORBOARD_PORT', 'TENSORBOARD_DEBUGGER_PORT', 'NVIDIA_PYTORCH_VERSION', 'TRT_VERSION'])\n\nprint(f\"python={sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}\")\n\n'''\ntry:\n      os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'  #tf.get_logger().setLevel('FATAL')\n      import tensorflow as tf; \n      print(f\"tensorflow={tf.__version__}\", tf.config.list_physical_devices('GPU'))\n      os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'  #tf.get_logger().setLevel('ERROR')\nexcept Exception:\n      pass\n'''\n\ntry:\n      print('PyTorch Version:', torch.__version__, ', CUDA is available:', torch.cuda.is_available(), ', Version CUDA:', torch.version.cuda)\n      print(torch.cuda.get_device_capability(), torch.cuda.get_arch_list())\n      print('CuDNN Enabled:', torch.backends.cudnn.enabled, ', Version:', torch.backends.cudnn.version())\nexcept Exception:\n      pass\n\ntry:\n      print('FastAI Version:', fastai.__version__)\nexcept Exception:\n      pass","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    train = True\n    debug = False\n    jupyter = False\n    kaggle = os.environ.get('KAGGLE_KERNEL_RUN_TYPE', '')\n    PATH_APP = Path('/kaggle/code/My v16. Iafoss - Structure v2 0.15679-0.15462')\n    PATH_SRC = Path('/kaggle/input/stanford-ribonanza-rna-folding')\n    PATH_PREP = Path('/kaggle/input/stanford-ribonanza-rna-folding-prepare/')\n    PATH_OUT = Path('/kaggle/working')\n    fname_src_csv = 'train_data_struct_ext.csv'\n    fname_src_parquet = 'train_data_struct_ext.parquet'\n    fname_test_parquet = 'test_sequences_struct_ext.parquet'\n    fname_log = 'log_data'\n    pre_train_model = \"\"\n    MODELS = []  # [PATH_APP / \"pass_2 model_122 (0.148919 0.130181 0.131064).pth\"]\n    SEED = 2023\n    device = 'cuda'\n    react_min = 206  # Number of reactivity values\n    seq_max = 457\n    pad_react_len = seq_max - react_min\n    vocab_struc = True\n    if vocab_struc:\n        fldname_seq='sequence_ext'\n        seq_map = {\n            'A': 1,  # G + (\n            'B': 2,  # G + .\n            'C': 3,  # G + )\n            'D': 4,  # A + (\n            'E': 5,  # A + .\n            'F': 6,  # A + )\n            'G': 7,  # C + (\n            'H': 8,  # C + .\n            'I': 9,  # C + )\n            'J': 10, # U + (\n            'K': 11, # U + .\n            'L': 12  # U + )\n        }\n    else:\n        fldname_seq='sequence'\n        seq_map = {'A':0,'C':1,'G':2,'U':3}\n    vocab = len(seq_map) + 1\n    if debug:\n        count_records = 1000\n        num_workers = 1\n    else:\n        num_workers = 14","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HYPER:\n    signal_to_noise = 1.0\n    test_size = 0.2\n    norm_first = False\n    dim = 192\n    depth = 12\n    head_size = 32\n    batch_size = 256\n    dropout=0.1\n    gradient_clip = 3.0\n    lr_max = 5e-4\n    wd = 0.05\n    pct_start = 0.02\n    if CFG.debug:\n        n_epoch = 1\n    else:\n        n_epoch = 200","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class logger:\n    data = []\n    def __init__(self, max_count, filename):\n        self.max_count = max_count\n        self.filename = filename\n        self.finish = False\n        self.cur_date_time = dt.datetime.now()\n    def add(self, place, **kwargs):\n        if self.max_count > len(self.data):\n            str=\"place=\" + place\n            for key,value in kwargs.items():\n                str += f\", {key}={value}\"\n            self.data.append(str)\n        elif not self.finish:\n            self.save()\n            self.finish = True\n    def save(self):\n        with open(self.filename + cur_date_time.strftime(\"%Y.%m.%d_%H-%M-%S\") + \".csv\",'w') as file_out:\n            file_out.writelines(\"%s\\n\" % line for line in self.data)\n    def get_data(self):\n        str=\"\"\n        for line in self.data:\n            str+=line+\"\\n\"\n        return str\n\nlog = logger(max_count = 100, filename=CFG.PATH_OUT / CFG.fname_log)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\n# Convert nested array data to a flat one-dimensional array.\ndef flatten(o):\n    \"Concatenate all collections and items as a generator\"\n    for item in o:\n        if isinstance(o, dict):      # If instance dictionary\n            yield o[item]; \n            continue\n        elif isinstance(item, str):  # If the instance is string\n            yield item; \n            continue\n        try: \n            yield from flatten(item)\n        except TypeError: \n            yield item","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fix fastai bug to enable fp16 training with dictionaries\n@fva.delegates(GradScaler)\nclass MixedPrecision(Callback):\n    \"Mixed precision training using Pytorch's `autocast` and `GradScaler`\"\n    order = 10\n    \n    def __init__(self, **kwargs): \n        self.kwargs = kwargs\n    \n    def before_fit(self): \n        self.autocast,self.learn.scaler,self.scales = autocast(),GradScaler(**self.kwargs),L()\n    \n    def before_batch(self): \n        self.autocast.__enter__()\n    \n    def after_pred(self):\n        if next(flatten(self.pred)).dtype==torch.float16: \n            self.learn.pred = to_float(self.pred)\n    \n    def after_loss(self):  # Event after loss calculation\n        self.autocast.__exit__(None, None, None)\n\n    def before_backward(self):  # Event before backpropagation\n        self.learn.loss_grad = self.scaler.scale(self.loss_grad)\n\n    def before_step(self):  # Event before step\n        \"Use `self` as a fake optimizer. `self.skipped` will be set to True `after_step` if gradients overflow. \"\n        self.skipped=True\n        self.scaler.step(self)\n        if self.skipped: \n            raise CancelStepException()\n        self.scales.append(self.scaler.get_scale())\n\n    def after_step(self):   # Event after step\n        self.learn.scaler.update()\n\n    @property \n    def param_groups(self): \n        \"Pretend to be an optimizer for `GradScaler`\"\n        return self.opt.param_groups\n    \n    def step(self, *args, **kwargs): \n        \"Fake optimizer step to detect whether this batch was skipped from `GradScaler`\"\n        self.skipped=False\n    \n    def after_fit(self): \n        self.autocast,self.learn.scaler,self.scales = None,None,None\n        \nfastai.callback.fp16.MixedPrecision = MixedPrecision","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RNA_Dataset(tud.Dataset):\n    def __init__(self, df_2A3, df_DMS, seq_max_len, **kwargs):\n        self.seq_max_len = seq_max_len\n        # Dividing columns into numpy arrays\n        self.seq = df_2A3[CFG.fldname_seq].values\n        self.seq_len = df_2A3['seq_len'].values\n        self.react_2A3 = df_2A3[[c for c in df_2A3.columns if 'reactivity_0' in c]].values\n        self.react_DMS = df_DMS[[c for c in df_DMS.columns if 'reactivity_0' in c]].values\n\n    def __len__(self):\n        return len(self.seq)\n\n    # Enter index data\n    def __getitem__(self, idx):\n        seq = np.array([CFG.seq_map[s] for s in self.seq[idx]])  # Map the characters of the string into an array of numbers\n        mask = torch.zeros(self.seq_max_len, dtype=torch.bool)\n        mask[:len(seq)] = True  # We set True for the initial elements along the length of the sequence\n        seq = np.pad(array=seq, pad_width=(0, self.seq_max_len - len(seq)))  # Right align the sequence with zeros\n        # Let's combine the sequence of arrays along a new axis.\n        react = torch.from_numpy(np.stack(arrays=[self.react_2A3[idx], self.react_DMS[idx]], axis=-1))  # Shape(206,2)\n        return {'seq': torch.from_numpy(seq), 'mask': mask}, {'react': react, 'mask': mask}\n\n# Preparing training packages\nclass LenMatchBatchSampler(tud.BatchSampler):\n    def __iter__(self):\n        buckets = [[]] * 100\n        yielded = 0\n        # Grouping records into batches with length multiples of 16\n        for idx in self.sampler:\n            s = self.sampler.data_source[idx]\n            if isinstance(s, tuple):  # If instance s represents a tuple\n                L = s[0][\"mask\"].sum()  # the sum of the true values in the mask\n            else:\n                L = s[\"mask\"].sum()\n            L = max(1, L // 16)  # Determining the package group\n            if len(buckets[L]) == 0:\n                buckets[L] = []\n            buckets[L].append(idx)\n\n            if len(buckets[L]) == self.batch_size:  # If the package is formed\n                batch = list(buckets[L])\n                yield batch\n                yielded += 1\n                buckets[L] = []\n\n        # We will collect the incomplete balances of the packages together\n        batch = []\n        leftover = [idx for bucket in buckets for idx in bucket]\n\n        for idx in leftover:\n            batch.append(idx)\n            if len(batch) == self.batch_size:\n                yielded += 1\n                yield batch\n                batch = []\n\n        # If there are records left in the last packet and the packet discard flag is not set, then we return the remainder\n        if len(batch) > 0 and not self.drop_last:\n            yielded += 1\n            yield batch","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dict_to(x, device='cuda'):\n    return {k: x[k].to(device) for k in x}\n\ndef to_device(x, device='cuda'):\n    return tuple(dict_to(e, device) for e in x)\n\nclass DeviceDataLoader:\n    def __init__(self, dataloader, device='cuda'):\n        self.dataloader = dataloader\n        self.device = device\n\n    def __len__(self):\n        return len(self.dataloader)\n\n    def __iter__(self):\n        for batch in self.dataloader:\n            yield tuple(dict_to(x, self.device) for x in batch)\n\nclass RNA_Data_Loaders():\n    def __init__(self, seed=CFG.SEED):\n        self.seed = seed\n        df = pd.read_parquet(CFG.PATH_PREP / CFG.fname_src_parquet )\n        df['seq_len'] = df.sequence_ext.apply(len)  # Length of the 'sequence_ext' field\n        self.seq_max_len = max(df['seq_len'].max(), CFG.react_min)  # Number of reactivity values\n        df_2A3 = df.loc[df.experiment_type == '2A3_MaP']\n        df_DMS = df.loc[df.experiment_type == 'DMS_MaP']\n        \n        filter_base = (df_2A3['signal_to_noise'].values >= HYPER.signal_to_noise) & \\\n                    (df_DMS['signal_to_noise'].values >= HYPER.signal_to_noise)\n        df_base_2A3, df_base_DMS = self.reset_indexes(filter_base, df_2A3, df_DMS)\n        splits = skms.train_test_split(df_base_2A3, df_base_DMS, test_size=HYPER.test_size, random_state=seed, shuffle=True)\n        for split in splits:\n            split = split.reset_index(drop=True)\n        df_2A3_train, df_DMS_train, self.df_2A3_valid, self.df_DMS_valid = splits[0], splits[2], splits[1], splits[3]\n\n        filter_noise = (df_2A3['signal_to_noise'].values >= 0.5) & (df_DMS['signal_to_noise'].values >= 0.5)\n        df_2A3_noise_train, df_DMS_noise_train = self.get_train_single(~filter_base & filter_noise, df_2A3, df_DMS)\n        self.df_2A3_train = pd.concat([df_2A3_noise_train, df_2A3_train], axis=0).reset_index()\n        self.df_DMS_train = pd.concat([df_DMS_noise_train, df_DMS_train], axis=0).reset_index()\n\n    def get_train_single(self, filter, df_2A3, df_DMS):\n        df_filter_2A3, df_filter_DMS = self.reset_indexes(filter, df_2A3, df_DMS)\n        splits = skms.train_test_split(df_filter_2A3, df_filter_DMS, test_size=0.5, random_state=self.seed, shuffle=True)\n        df_2A3_train = pd.concat([splits[0], splits[1]], ignore_index=True)\n        df_DMS_train = pd.concat([splits[2], splits[3]], ignore_index=True)\n        return df_2A3_train, df_DMS_train\n\n    def reset_indexes(self, filter_mask, df_2A3, df_DMS):\n        df_2A3 = df_2A3.loc[filter_mask].reset_index(drop=True)\n        df_DMS = df_DMS.loc[filter_mask].reset_index(drop=True)\n        for rec_2A3,rec_DMS in zip(df_2A3.itertuples(index=False), df_DMS.itertuples(index=False)):\n            assert rec_2A3[0] == rec_DMS[0]\n        return df_2A3, df_DMS\n    \n    def get_device_loader(self, df_2A3, df_DMS, seq_max_len, persistent_workers=False):\n        ds_base = RNA_Dataset(df_2A3, df_DMS, seq_max_len)\n        sampler_mask = tud.RandomSampler(ds_base)\n        len_sampler = LenMatchBatchSampler(sampler_mask, batch_size=HYPER.batch_size, drop_last=True)\n        data_loader = tud.DataLoader(ds_base, batch_sampler=len_sampler, num_workers=CFG.num_workers, persistent_workers=persistent_workers)\n        return DeviceDataLoader(data_loader, CFG.device)\n\n    def conv_ds_to_data_loaders(self, df_2A3_train, df_DMS_train, df_2A3_valid, df_DMS_valid):\n        loader_train = self.get_device_loader(df_2A3_train, df_DMS_train, self.seq_max_len, persistent_workers=True)\n        loader_valid = self.get_device_loader(df_2A3_valid, df_DMS_valid, self.seq_max_len)\n        return DataLoaders(loader_train, loader_valid)\n\n    def get_base_loaders(self):\n        return self.conv_ds_to_data_loaders(self.df_2A3_train, self.df_DMS_train, self.df_2A3_valid, self.df_DMS_valid)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SinusoidalPosEmb(nn.Module):\n    def __init__(self, dim=16, M=10000):\n        super().__init__()\n        self.dim = dim\n        self.M = M    \n\n    def forward(self, x):\n        device = x.device\n        half_dim = self.dim // 2\n        emb = math.log(self.M) / half_dim\n        emb = torch.exp(torch.arange(half_dim, device=device) * (-emb))  # Tensor with exponential decay from [1, .. 1.1007e-04]\n        emb = x[...,None] * emb[None,...]\n        emb = torch.cat((emb.sin(), emb.cos()), dim=-1)\n        return emb\n\nclass RNA_Model(nn.Module):\n    def __init__(self, dim=192, depth=12, head_size=32, **kwargs):\n        super().__init__()\n        self.emb = nn.Embedding(CFG.vocab,dim)\n        self.pos_enc = SinusoidalPosEmb(dim)\n        self.transformer = nn.TransformerEncoder(\n            nn.TransformerEncoderLayer(d_model=dim, nhead=dim//head_size, dim_feedforward=4*dim,\n                dropout=HYPER.dropout, activation=nn.GELU(), batch_first=True, norm_first=HYPER.norm_first), depth)\n        self.proj_out = nn.Linear(dim,2)\n    \n    def forward(self, x0):\n        mask = x0['mask']\n        Lmax = mask.sum(-1).max()  # Let's determine the maximum sequence length in a packet\n        mask = mask[:,:Lmax]       # Let's cut the arrays of elements to the maximum length in the package\n        x = x0['seq'][:,:Lmax]     # Let's cut the arrays of elements to the maximum length in the package\n        \n        pos = torch.arange(Lmax, device=x.device).unsqueeze(0)  # Let's create an array with the maximum length dimension Tensor.Shape(1,177)\n        pos = self.pos_enc(pos)\n        x = self.emb(x)        \n        x = x + pos            \n        \n        x = self.transformer(x, src_key_padding_mask=~mask)\n        x = self.proj_out(x)\n        return x","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def loss(pred,target):\n    p = pred[target['mask'][:,:pred.shape[1]]]\n    y = target['react'][target['mask']].clip(0,1)\n    loss = F.l1_loss(p, y, reduction='none')\n    loss = loss[~torch.isnan(loss)].mean()\n    return loss\n\nclass MAE(Metric):\n    def __init__(self): \n        self.reset()\n        \n    def reset(self): \n        self.x,self.y = [],[]\n        \n    def accumulate(self, learn):\n        x = learn.pred[learn.y['mask'][:,:learn.pred.shape[1]]]\n        y = learn.y['react'][learn.y['mask']].clip(0,1)\n        self.x.append(x)\n        self.y.append(y)\n\n    @property\n    def value(self):\n        x,y = torch.cat(self.x,0),torch.cat(self.y,0)\n        loss = F.l1_loss(x, y, reduction='none')\n        loss = loss[~torch.isnan(loss)].mean()\n        return loss\n\nclass RNA_Test_Dataset(tud.Dataset):\n    def __init__(self, df, mask_only=False, **kwargs):\n        df['L'] = df.sequence_ext.apply(len)\n        self.Lmax = df['L'].max()\n        self.df = df\n        self.mask_only = mask_only\n\n    def __len__(self):\n        return len(self.df)\n\n    # Enter index data\n    def __getitem__(self, idx):\n        id_min, id_max, seq = self.df.loc[idx, ['id_min','id_max',CFG.fldname_seq]]\n        mask = torch.zeros(self.Lmax, dtype=torch.bool)\n        L = len(seq)\n        mask[:L] = True\n        if self.mask_only: \n            return {'mask':mask},{}\n        ids = np.arange(id_min,id_max+1)        \n        seq = np.array([CFG.seq_map[s] for s in seq])\n        seq = np.pad(seq,(0,self.Lmax-L))\n        ids = np.pad(ids,(0,self.Lmax-L), constant_values=-1)\n        return {'seq':torch.from_numpy(seq), 'mask':mask}, {'ids':ids}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.train:\n    cur_date_time = dt.datetime.now()\n    seed_everything(CFG.SEED)  # Initialize variables\n    os.makedirs(CFG.PATH_OUT, exist_ok=True)\n\n    data_loaders=RNA_Data_Loaders()\n    data_list = [\n        { 'filename': \"base\", 'epoch': HYPER.n_epoch, 'lr_max': HYPER.lr_max, 'wd': HYPER.wd, 'pct_start': HYPER.pct_start, 'data': data_loaders.get_base_loaders() }\n    ]\n    gc.collect()\n\n    model = RNA_Model()\n    model = model.to(CFG.device)\n    if CFG.pre_train_model:\n        model.load_state_dict(torch.load(CFG.pre_train_model, map_location=torch.device(CFG.device)))\n\n    for data_dict in data_list:\n        callbacks = [fva.GradientClip(HYPER.gradient_clip), fva.SaveModelCallback(every_epoch=1), \n                     fva.CSVLogger(fname=\"history_\" + cur_date_time.strftime(\"%Y.%m.%d_%H-%M-%S\") + \".csv\")]\n        if CFG.jupyter:\n            callbacks.append(fva.ShowGraphCallback())\n        learn = fva.Learner(data_dict[\"data\"], model, loss_func=loss, cbs=callbacks, metrics=[MAE()], \n                        path=CFG.PATH_OUT, model_dir=data_dict[\"filename\"]).to_fp16()\n        # fp16 doesn't help at P100 but gives x1.6-1.8 speedup at modern hardware\n        learn.fit_one_cycle(n_epoch=data_dict[\"epoch\"], lr_max=data_dict[\"lr_max\"], wd=data_dict[\"wd\"], pct_start=data_dict[\"pct_start\"])\n        torch.save(learn.model.state_dict(), CFG.PATH_OUT / f'{data_dict[\"filename\"]}.pth')\n        gc.collect()\n\nelse:\n    df_test = pd.read_parquet(CFG.PATH_PREP / CFG.fname_test_parquet)\n    ds = RNA_Test_Dataset(df_test)\n    dl = DeviceDataLoader(tud.DataLoader(ds, batch_size=HYPER.batch_size, shuffle=False, drop_last=False, num_workers=CFG.num_workers), CFG.device)\n    del df_test\n    gc.collect()\n\n    models = []\n    for filename in CFG.MODELS:\n        model = RNA_Model()   \n        model = model.to(CFG.device)\n        model.load_state_dict(torch.load(filename, map_location=torch.device('cpu')))\n        model.eval()\n        models.append(model)\n\n    ids,preds = [],[]\n    #for x,y in tqdm(dl):  # If there is Jupiter in the notebook\n    for x,y in dl:         # If there is IDE VSCode in mode clear Python\n        with torch.no_grad(), tca.autocast():\n            test_list = [torch.nan_to_num(model(x)) for model in models]\n            test_stack = torch.stack(test_list,0).mean(0).clip(0,1)\n        for idx, mask, pi in zip(y['ids'].cpu(), x['mask'].cpu(), test_stack.cpu()):\n            ids.append(idx[mask])\n            preds.append(pi[mask[:pi.shape[0]]])\n\n    ids = torch.concat(ids)\n    preds = torch.concat(preds)\n\n    df=pl.DataFrame({'id':ids.numpy(), 'reactivity_DMS_MaP':preds[:,1].numpy(), 'reactivity_2A3_MaP':preds[:,0].numpy()},\n                    schema={\"id\": pl.Int64, \"reactivity_DMS_MaP\": pl.Float32, \"reactivity_2A3_MaP\": pl.Float32})\n    df.write_csv(CFG.PATH_APP / 'submission.csv',float_precision=4)\n    print(df.head(10))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Comparison table of results\n\n- Signal/Noise 1.00 => 0.15679 LB\n- Signal/Noise 0.75 => 0.15495 LB\n- Signal/Noise 0.50 => 0.15462 LB\n- Signal/Noise 0.25 => 0.15548 LB\n\n`The winner was the notebook with a signal-to-noise ratio of 0.5.`","metadata":{}},{"cell_type":"markdown","source":"## The result of training with a signal-to-noise ratio of 0.50","metadata":{}},{"cell_type":"code","source":"epoch\ttrain_loss\tvalid_loss\tmae\ttime\n0\t0.21952399611473083\t0.2018742561340332\t0.20220555365085602\t02:20\n1\t0.21192215383052826\t0.19368623197078705\t0.1939493864774704\t02:19\n2\t0.20036397874355316\t0.1797645092010498\t0.1801149547100067\t02:18\n3\t0.1915188878774643\t0.17134195566177368\t0.17164038121700287\t02:19\n4\t0.18500596284866333\t0.16295720636844635\t0.16326004266738892\t03:09\n5\t0.18143786489963531\t0.1582898199558258\t0.15869130194187164\t02:43\n6\t0.178629070520401\t0.15582327544689178\t0.15626920759677887\t02:18\n7\t0.174730584025383\t0.1511065810918808\t0.15157507359981537\t02:19\n8\t0.17309235036373138\t0.1478828340768814\t0.14836065471172333\t02:20\n9\t0.17185674607753754\t0.14696258306503296\t0.14737911522388458\t02:19\n10\t0.1706501841545105\t0.14510375261306763\t0.14560483396053314\t02:18\n11\t0.16887266933918\t0.14416782557964325\t0.1446877419948578\t02:18\n12\t0.16734546422958374\t0.1436784714460373\t0.1441936045885086\t02:17\n13\t0.16721786558628082\t0.14256298542022705\t0.14308121800422668\t02:17\n14\t0.16645419597625732\t0.14253811538219452\t0.14310605823993683\t02:17\n15\t0.16608278453350067\t0.14115165174007416\t0.14169082045555115\t02:17\n16\t0.16515161097049713\t0.14239351451396942\t0.14301468431949615\t02:17\n17\t0.16481366753578186\t0.14164477586746216\t0.1422608196735382\t02:18\n18\t0.1643863171339035\t0.14344970881938934\t0.14402815699577332\t02:17\n19\t0.1640401929616928\t0.1387672871351242\t0.13939671218395233\t02:17\n20\t0.16308575868606567\t0.13986961543560028\t0.14051954448223114\t02:17\n21\t0.16226115822792053\t0.14068295061588287\t0.1413256973028183\t02:17\n22\t0.1620968133211136\t0.13763298094272614\t0.13828623294830322\t02:17\n23\t0.1621183305978775\t0.14056837558746338\t0.14123468101024628\t02:17\n24\t0.16142354905605316\t0.137613445520401\t0.13829649984836578\t02:19\n25\t0.1611914038658142\t0.13719676434993744\t0.13787585496902466\t02:19\n26\t0.16104070842266083\t0.13659293949604034\t0.13724173605442047\t02:18\n27\t0.16061939299106598\t0.13646931946277618\t0.13714957237243652\t02:19\n28\t0.15996861457824707\t0.13605475425720215\t0.13671325147151947\t02:19\n29\t0.15956655144691467\t0.13648049533367157\t0.13716191053390503\t02:18\n30\t0.15971782803535461\t0.1365961879491806\t0.13728751242160797\t02:18\n31\t0.1595344990491867\t0.138644739985466\t0.1393289417028427\t02:18\n32\t0.15880629420280457\t0.13504981994628906\t0.1357795149087906\t02:18\n33\t0.15974533557891846\t0.13680821657180786\t0.13752250373363495\t02:18\n34\t0.15851642191410065\t0.13477590680122375\t0.13546277582645416\t02:18\n35\t0.15825678408145905\t0.13457831740379333\t0.13528655469417572\t02:18\n36\t0.1578548550605774\t0.1349400132894516\t0.1356765478849411\t02:18\n37\t0.15742386877536774\t0.1352236568927765\t0.13596338033676147\t02:18\n38\t0.15767015516757965\t0.13486018776893616\t0.13562370836734772\t02:18\n39\t0.15799245238304138\t0.13360854983329773\t0.13436034321784973\t02:18\n40\t0.15758968889713287\t0.1346336305141449\t0.1353917419910431\t02:18\n41\t0.15678773820400238\t0.13345804810523987\t0.13421426713466644\t02:18\n42\t0.15711502730846405\t0.133444145321846\t0.13419871032238007\t02:18\n43\t0.15656504034996033\t0.13359762728214264\t0.13437050580978394\t02:18\n44\t0.15707100927829742\t0.13381080329418182\t0.13456527888774872\t02:18\n45\t0.15635065734386444\t0.13714085519313812\t0.13792692124843597\t02:18\n46\t0.15618592500686646\t0.13395024836063385\t0.13472361862659454\t02:18\n47\t0.156118243932724\t0.13280268013477325\t0.13357524573802948\t02:18\n48\t0.1556410938501358\t0.13327448070049286\t0.13404271006584167\t02:18\n49\t0.15569329261779785\t0.13287247717380524\t0.13363473117351532\t02:18\n50\t0.15633654594421387\t0.1326708048582077\t0.13345898687839508\t02:18\n51\t0.15632487833499908\t0.13242565095424652\t0.1331985741853714\t02:18\n52\t0.1555657833814621\t0.13466070592403412\t0.13547785580158234\t02:18\n53\t0.15550968050956726\t0.13567949831485748\t0.1364736109972\t02:18\n54\t0.15576456487178802\t0.13253574073314667\t0.13333509862422943\t02:18\n55\t0.15495233237743378\t0.13325929641723633\t0.1340455859899521\t02:18\n56\t0.15491580963134766\t0.13222448527812958\t0.13302767276763916\t02:18\n57\t0.15528450906276703\t0.13482484221458435\t0.1356177031993866\t02:18\n58\t0.15448728203773499\t0.1317623406648636\t0.13253554701805115\t02:18\n59\t0.15472327172756195\t0.13287194073200226\t0.13369296491146088\t02:18\n60\t0.1547601968050003\t0.13171690702438354\t0.1325208693742752\t02:18\n61\t0.15411365032196045\t0.13341999053955078\t0.1342271864414215\t02:18\n62\t0.1541948765516281\t0.13229434192180634\t0.13308429718017578\t02:18\n63\t0.15470625460147858\t0.13324399292469025\t0.13405190408229828\t02:18\n64\t0.154360830783844\t0.1321149617433548\t0.13292557001113892\t02:18\n65\t0.15395139157772064\t0.1329200267791748\t0.13373564183712006\t02:18\n66\t0.15362828969955444\t0.1314055323600769\t0.13223496079444885\t02:18\n67\t0.15379555523395538\t0.1313682198524475\t0.13217687606811523\t02:18\n68\t0.1532837599515915\t0.1323830485343933\t0.13320699334144592\t02:18\n69\t0.15320637822151184\t0.13228839635849\t0.1331164389848709\t02:18\n70\t0.15328697860240936\t0.13242113590240479\t0.13324405252933502\t02:18\n71\t0.1532195508480072\t0.13237518072128296\t0.1331942230463028\t02:18\n72\t0.15318992733955383\t0.13224978744983673\t0.1330678015947342\t02:20\n73\t0.15331090986728668\t0.13134077191352844\t0.13217045366764069\t02:20\n74\t0.15280762314796448\t0.13194118440151215\t0.13274206221103668\t02:21\n75\t0.15319357812404633\t0.13217781484127045\t0.133012056350708\t02:21\n76\t0.15232712030410767\t0.1310068517923355\t0.13183343410491943\t02:19\n77\t0.1524437963962555\t0.13189153373241425\t0.13273239135742188\t02:19\n78\t0.15209563076496124\t0.1313638538122177\t0.1321869194507599\t02:21\n79\t0.15249697864055634\t0.1316087543964386\t0.13244591653347015\t02:19\n80\t0.15241333842277527\t0.13125185668468475\t0.13208737969398499\t02:19\n81\t0.1523510068655014\t0.13106989860534668\t0.1319144070148468\t02:19\n82\t0.15181176364421844\t0.13103684782981873\t0.13187876343727112\t02:19\n83\t0.15158343315124512\t0.13152964413166046\t0.1323750764131546\t02:18\n84\t0.15198954939842224\t0.1314055174589157\t0.13221858441829681\t02:19\n85\t0.15163244307041168\t0.13074299693107605\t0.1315961480140686\t02:19\n86\t0.15096482634544373\t0.13134832680225372\t0.13219362497329712\t02:20\n87\t0.1519349366426468\t0.13082745671272278\t0.1316625326871872\t02:22\n88\t0.15228860080242157\t0.13117524981498718\t0.13202407956123352\t02:32\n89\t0.1519334614276886\t0.1313612312078476\t0.13219155371189117\t02:29\n90\t0.15129354596138\t0.13172224164009094\t0.13257625699043274\t02:22\n91\t0.15128560364246368\t0.13121503591537476\t0.13208186626434326\t02:20\n92\t0.1510164439678192\t0.131975457072258\t0.13282237946987152\t02:19\n93\t0.15119951963424683\t0.1309327483177185\t0.13178199529647827\t02:18\n94\t0.15057654678821564\t0.13158123195171356\t0.13244090974330902\t02:19\n95\t0.15110467374324799\t0.1313420981168747\t0.13218167424201965\t02:19\n96\t0.15020997822284698\t0.13082462549209595\t0.13167361915111542\t02:19\n97\t0.15112513303756714\t0.13101385533809662\t0.13187547028064728\t02:19\n98\t0.1506556123495102\t0.13113589584827423\t0.1319972723722458\t02:19\n99\t0.15067362785339355\t0.13147366046905518\t0.1323259323835373\t02:19\n100\t0.15019366145133972\t0.1310030072927475\t0.13185203075408936\t02:19\n101\t0.15026897192001343\t0.13105003535747528\t0.13190977275371552\t02:19\n102\t0.15078204870224\t0.13077785074710846\t0.13163545727729797\t02:19\n103\t0.15043048560619354\t0.13084085285663605\t0.13170750439167023\t02:19\n104\t0.15042634308338165\t0.13076291978359222\t0.13161899149417877\t02:19\n105\t0.15009742975234985\t0.13175036013126373\t0.13260582089424133\t02:19\n106\t0.14973853528499603\t0.13121236860752106\t0.13209295272827148\t02:19\n107\t0.14962410926818848\t0.13103920221328735\t0.1318962275981903\t02:19\n108\t0.1492924839258194\t0.13058343529701233\t0.13144811987876892\t02:19\n109\t0.14931994676589966\t0.1307896375656128\t0.1316414475440979\t02:19\n110\t0.14993879199028015\t0.13102935254573822\t0.13189329206943512\t02:19\n111\t0.14981293678283691\t0.13113431632518768\t0.1320044845342636\t02:19\n112\t0.14997103810310364\t0.13056334853172302\t0.13143543899059296\t02:19\n113\t0.14943496882915497\t0.13095851242542267\t0.13183018565177917\t02:19\n114\t0.14939084649085999\t0.13095605373382568\t0.13182973861694336\t02:19\n115\t0.14946644008159637\t0.13126827776432037\t0.13214685022830963\t02:21\n116\t0.14865019917488098\t0.13071151077747345\t0.1315840482711792\t02:15\n117\t0.14899663627147675\t0.1307237148284912\t0.13159599900245667\t02:15\n118\t0.1491432785987854\t0.13048647344112396\t0.13137555122375488\t02:14\n119\t0.1484801322221756\t0.13066700100898743\t0.13153201341629028\t02:18\n120\t0.14896684885025024\t0.13109220564365387\t0.1319645196199417\t02:19\n121\t0.14883384108543396\t0.1307862251996994\t0.13167162239551544\t02:19\n122\t0.1489192694425583\t0.1301812380552292\t0.13106366991996765\t02:20\n123\t0.14810726046562195\t0.13076812028884888\t0.13164526224136353\t02:20\n124\t0.14846260845661163\t0.1306280791759491\t0.1315220594406128\t02:20\n125\t0.14891956746578217\t0.1305767446756363\t0.13145148754119873\t02:20\n126\t0.14871007204055786\t0.13067862391471863\t0.13155022263526917\t02:20\n127\t0.14796972274780273\t0.13118644058704376\t0.13206778466701508\t02:32\n128\t0.14833298325538635\t0.13055257499217987\t0.13144703209400177\t02:21\n129\t0.1478838324546814\t0.1304476261138916\t0.13133233785629272\t02:21\n130\t0.14780685305595398\t0.1311316192150116\t0.1320149153470993\t02:19\n131\t0.14812161028385162\t0.13058200478553772\t0.13146057724952698\t02:15\n132\t0.14820800721645355\t0.13045136630535126\t0.13132844865322113\t02:15\n133\t0.14744554460048676\t0.13071121275424957\t0.13160640001296997\t02:15\n134\t0.14774712920188904\t0.13086843490600586\t0.13176165521144867\t02:15\n135\t0.14813034236431122\t0.1305559128522873\t0.1314397156238556\t02:15\n136\t0.1477779746055603\t0.13051773607730865\t0.13139908015727997\t02:15\n137\t0.14744915068149567\t0.13068242371082306\t0.1315656453371048\t02:15\n138\t0.14757463335990906\t0.13055294752120972\t0.13145264983177185\t02:19\n139\t0.14670418202877045\t0.13090690970420837\t0.1317967176437378\t02:19\n140\t0.14697863161563873\t0.13060308992862701\t0.13149625062942505\t02:18\n141\t0.14731863141059875\t0.13032707571983337\t0.13121619820594788\t02:17\n142\t0.14703592658042908\t0.13059914112091064\t0.13150431215763092\t02:18\n143\t0.14728759229183197\t0.13039334118366241\t0.13127784430980682\t02:18\n144\t0.14714942872524261\t0.13082250952720642\t0.13171207904815674\t02:22\n145\t0.1471250206232071\t0.13064809143543243\t0.13154305517673492\t02:18\n146\t0.146876722574234\t0.13070571422576904\t0.13160012662410736\t02:17\n147\t0.14655733108520508\t0.1303761601448059\t0.1312759667634964\t02:18\n148\t0.14684580266475677\t0.13063526153564453\t0.13152958452701569\t02:18\n149\t0.14676128327846527\t0.13053001463413239\t0.1314239352941513\t02:17\n150\t0.1465868353843689\t0.13042473793029785\t0.1313333362340927\t02:18\n151\t0.1463003009557724\t0.13087229430675507\t0.13176773488521576\t02:18\n152\t0.14718911051750183\t0.13057444989681244\t0.13147646188735962\t02:18\n153\t0.1464705467224121\t0.13065533339977264\t0.13156215846538544\t02:18\n154\t0.1462501734495163\t0.13073498010635376\t0.1316356658935547\t02:17\n155\t0.14572080969810486\t0.13068610429763794\t0.13158577680587769\t02:19\n156\t0.14676381647586823\t0.13065369427204132\t0.13155387341976166\t02:19\n157\t0.14658929407596588\t0.13073834776878357\t0.13163886964321136\t02:20\n158\t0.14646771550178528\t0.1306859850883484\t0.1315872073173523\t02:19\n159\t0.14576557278633118\t0.13073882460594177\t0.1316511034965515\t02:39\n160\t0.14634349942207336\t0.130767360329628\t0.1316651701927185\t04:15\n161\t0.14597903192043304\t0.13082754611968994\t0.13173893094062805\t04:09\n162\t0.14606069028377533\t0.13065403699874878\t0.13155992329120636\t04:13\n163\t0.1456727385520935\t0.130754292011261\t0.13166341185569763\t04:15\n164\t0.14645162224769592\t0.13066351413726807\t0.13157619535923004\t04:07\n165\t0.1456223428249359\t0.1306784451007843\t0.13158220052719116\t04:15\n166\t0.14598889648914337\t0.13071154057979584\t0.13162347674369812\t04:06\n167\t0.1458662450313568\t0.13067695498466492\t0.13157439231872559\t04:15\n168\t0.1458815336227417\t0.13051456212997437\t0.1314234435558319\t04:06\n169\t0.14532752335071564\t0.13081219792366028\t0.13171891868114471\t04:14\n170\t0.14639125764369965\t0.13058067858219147\t0.13149067759513855\t04:09\n171\t0.14558690786361694\t0.13062407076358795\t0.13153943419456482\t04:11\n172\t0.14576753973960876\t0.13068918883800507\t0.13160963356494904\t04:15\n173\t0.145662322640419\t0.13060811161994934\t0.13151255249977112\t04:06\n174\t0.14571470022201538\t0.13050256669521332\t0.13141214847564697\t04:15\n175\t0.14506155252456665\t0.13076305389404297\t0.1316715031862259\t04:05\n176\t0.14577990770339966\t0.13070279359817505\t0.13161273300647736\t04:15\n177\t0.1457250714302063\t0.13074816763401031\t0.13166159391403198\t04:05\n178\t0.14566074311733246\t0.130692258477211\t0.1315944939851761\t04:15\n179\t0.1454046070575714\t0.1306314468383789\t0.1315383017063141\t04:10\n180\t0.1453235149383545\t0.13059328496456146\t0.13150453567504883\t04:12\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Important Note\n\nI am unable to reproduce the same result every time, although the seed_everything function is called, but every time I run it I get a different result. If anyone can tell me how to solve this problem for this notebook, then write in the comments, I will be very grateful.","metadata":{}}]}