{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":51294,"databundleVersionId":6512296,"sourceType":"competition"}],"dockerImageVersionId":30558,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **<span style=\"color:#F7B2B0;\">Introduction</span>**\n\nRibonucleic acid (RNA) is essential for most biological functions. A better understanding of how to manipulate RNA could help usher in an age of programmable medicine, including first cures for pancreatic cancer and Alzheimer’s disease as well as much-needed antibiotics and new biotechnology approaches for climate change. But first, researchers must better understand each RNA molecule's structure, an ideal problem for data science.\nRecent efforts to predict RNA structure have run into a number of challenges: (1) a paucity of training data, (2) lack of intellectual and computational power, and (3) difficulties in rigorously splitting training and test data. Can a Kaggle competition close these gaps?\n\n# **<span style=\"color:#F7B2B0;\">Goal</span>**\n\nThe goal of this competition is to create a model that predicts the structures of any RNA molecule and the resulting chemical mapping profile, which can be compared to data collected for each position in the RNA.\n\n# **<span style=\"color:#F7B2B0;\">Install the Libraries</span>**\n","metadata":{}},{"cell_type":"code","source":"!pip install -q wandb","metadata":{"execution":{"iopub.status.busy":"2023-09-15T09:15:03.273360Z","iopub.execute_input":"2023-09-15T09:15:03.273755Z","iopub.status.idle":"2023-09-15T09:15:17.164843Z","shell.execute_reply.started":"2023-09-15T09:15:03.273722Z","shell.execute_reply":"2023-09-15T09:15:17.163500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **<span style=\"color:#F7B2B0;\">Import the Packages</span>**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.preprocessing import LabelEncoder, MinMaxScaler\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_absolute_error\n\nfrom keras.models import Sequential\nfrom keras.layers import Dense, LSTM, Embedding, Dropout, Bidirectional\nfrom keras.optimizers import Adam\n\nfrom kaggle_secrets import UserSecretsClient\nimport wandb\nfrom datetime import datetime\n\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"614f8844-d39d-4bf8-add5-49359a727425","_cell_guid":"05589905-1fb1-475c-8567-cd6f64c7a0c3","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-15T09:33:51.206255Z","iopub.execute_input":"2023-09-15T09:33:51.206726Z","iopub.status.idle":"2023-09-15T09:34:02.922403Z","shell.execute_reply.started":"2023-09-15T09:33:51.206685Z","shell.execute_reply":"2023-09-15T09:34:02.921242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<img src=\"https://camo.githubusercontent.com/dd842f7b0be57140e68b2ab9cb007992acd131c48284eaf6b1aca758bfea358b/68747470733a2f2f692e696d6775722e636f6d2f52557469567a482e706e67\">\n\n> I will be integrating W&B for visualizations and logging artifacts!\n> \n> [Stanford Ribonanza RNA Folding Project on W&B Dashboard](https://wandb.ai/usharengaraju/stanford_rna)\n> \n> - To get the API key, create an account in the [website](https://wandb.ai/site) .\n> - Use secrets to use API Keys more securely ","metadata":{}},{"cell_type":"code","source":"# Setup user secrets for login\nuser_secrets = UserSecretsClient()\nwandb_api = user_secrets.get_secret(\"api_key\") \nwandb.login(key=wandb_api)","metadata":{"execution":{"iopub.status.busy":"2023-09-15T09:34:02.924727Z","iopub.execute_input":"2023-09-15T09:34:02.925516Z","iopub.status.idle":"2023-09-15T09:34:06.768771Z","shell.execute_reply.started":"2023-09-15T09:34:02.925476Z","shell.execute_reply":"2023-09-15T09:34:06.767705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **<span style=\"color:#F7B2B0;\">Data Pipeline</span>**","metadata":{}},{"cell_type":"code","source":"data = pd.read_csv('/kaggle/input/stanford-ribonanza-rna-folding/train_data.csv',nrows=100000)","metadata":{"_uuid":"5efdd510-8922-4ddf-9e67-82451f04e63e","_cell_guid":"23132663-0ff2-47f4-a438-4e089a5b892a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-15T09:34:12.146939Z","iopub.execute_input":"2023-09-15T09:34:12.147723Z","iopub.status.idle":"2023-09-15T09:34:17.986194Z","shell.execute_reply.started":"2023-09-15T09:34:12.147687Z","shell.execute_reply":"2023-09-15T09:34:17.984312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.fillna(0)\ndata = data.query('signal_to_noise < 0.7 and signal_to_noise > 0')","metadata":{"execution":{"iopub.status.busy":"2023-09-15T09:34:17.988327Z","iopub.execute_input":"2023-09-15T09:34:17.988662Z","iopub.status.idle":"2023-09-15T09:34:18.323580Z","shell.execute_reply.started":"2023-09-15T09:34:17.988634Z","shell.execute_reply":"2023-09-15T09:34:18.322329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_sequence_length = data['sequence'].apply(len).max()\ndata['padded_sequence'] = data['sequence'].apply(lambda x: x.ljust(max_sequence_length, 'N'))\nencoder = LabelEncoder()\ndata['sequence_encoded'] = data['padded_sequence'].apply(lambda x: encoder.fit_transform(list(x)))\nX = np.array(data['sequence_encoded'].tolist())\ny = data['reactivity_0001'].values\nX = X.reshape(X.shape[0], X.shape[1], 1)\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)","metadata":{"_uuid":"6e73aadc-2e66-447d-a503-4e75ead7c261","_cell_guid":"337f39e9-02b4-4411-a886-9ef3171f88b6","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-15T09:34:18.325177Z","iopub.execute_input":"2023-09-15T09:34:18.325587Z","iopub.status.idle":"2023-09-15T09:34:25.280895Z","shell.execute_reply.started":"2023-09-15T09:34:18.325550Z","shell.execute_reply":"2023-09-15T09:34:25.279789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape","metadata":{"execution":{"iopub.status.busy":"2023-09-15T09:34:35.573311Z","iopub.execute_input":"2023-09-15T09:34:35.574159Z","iopub.status.idle":"2023-09-15T09:34:35.582157Z","shell.execute_reply.started":"2023-09-15T09:34:35.574117Z","shell.execute_reply":"2023-09-15T09:34:35.581031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run = wandb.init(project = 'stanford_rna',\n                 config = {},\n                 save_code = True,\n                 \n)\ntable = wandb.Table(dataframe=data[[\"padded_sequence\",\"reactivity_0001\"]])\n\nwandb.log({\"Table\":table})\nrun.finish()","metadata":{"execution":{"iopub.status.busy":"2023-09-15T09:17:18.693396Z","iopub.execute_input":"2023-09-15T09:17:18.693794Z","iopub.status.idle":"2023-09-15T09:18:25.685418Z","shell.execute_reply.started":"2023-09-15T09:17:18.693764Z","shell.execute_reply":"2023-09-15T09:18:25.684371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from wandb.keras import WandbCallback, WandbMetricsLogger\nrun = wandb.init(project = 'stanford_rna',\n                 save_code = True,\n                 name='grownet'\n                 \n)","metadata":{"execution":{"iopub.status.busy":"2023-09-15T09:38:11.762714Z","iopub.execute_input":"2023-09-15T09:38:11.763161Z","iopub.status.idle":"2023-09-15T09:38:44.027777Z","shell.execute_reply.started":"2023-09-15T09:38:11.763127Z","shell.execute_reply":"2023-09-15T09:38:44.026774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# **<span style=\"color:#F7B2B0;\">GRUNN</span>**","metadata":{}},{"cell_type":"code","source":"from wandb.keras import WandbCallback, WandbMetricsLogger\n\nrun = wandb.init(project = 'stanford_rna',\n                 save_code = True,\n                 name='GRUNN'\n                 \n)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-09-15T09:50:51.913399Z","iopub.execute_input":"2023-09-15T09:50:51.913830Z","iopub.status.idle":"2023-09-15T09:51:24.510744Z","shell.execute_reply.started":"2023-09-15T09:50:51.913799Z","shell.execute_reply":"2023-09-15T09:51:24.509711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport random\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model, clone_model\nfrom tensorflow.keras.layers import *","metadata":{"execution":{"iopub.status.busy":"2023-09-15T09:51:24.515917Z","iopub.execute_input":"2023-09-15T09:51:24.518655Z","iopub.status.idle":"2023-09-15T09:51:25.470267Z","shell.execute_reply.started":"2023-09-15T09:51:24.518614Z","shell.execute_reply":"2023-09-15T09:51:25.469253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential()\nmodel.add(Bidirectional(GRU(128, return_sequences=True), input_shape=(X_train.shape[1], 1)))\nmodel.add(Dropout(0.5))\nmodel.add(Bidirectional(GRU(64)))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(32, activation='relu'))\nmodel.add(Dense(1, activation='relu'))","metadata":{"execution":{"iopub.status.busy":"2023-09-15T09:52:36.020240Z","iopub.execute_input":"2023-09-15T09:52:36.021147Z","iopub.status.idle":"2023-09-15T09:52:38.212923Z","shell.execute_reply.started":"2023-09-15T09:52:36.021108Z","shell.execute_reply":"2023-09-15T09:52:38.211638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(Adam(0.00001), 'mae',metrics=['mae'])\nbs = 128*2\nmodel.fit(X_train, y_train,epochs=10,batch_size=bs,validation_data=(X_val,y_val),callbacks=[WandbCallback()])","metadata":{"execution":{"iopub.status.busy":"2023-09-15T09:53:24.945421Z","iopub.execute_input":"2023-09-15T09:53:24.945838Z","iopub.status.idle":"2023-09-15T09:55:12.241012Z","shell.execute_reply.started":"2023-09-15T09:53:24.945807Z","shell.execute_reply":"2023-09-15T09:55:12.239827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run.finish()","metadata":{"execution":{"iopub.status.busy":"2023-09-15T09:55:12.292739Z","iopub.execute_input":"2023-09-15T09:55:12.293112Z","iopub.status.idle":"2023-09-15T09:55:17.922327Z","shell.execute_reply.started":"2023-09-15T09:55:12.293077Z","shell.execute_reply":"2023-09-15T09:55:17.921346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# **<span style=\"color:#F7B2B0;\">TabNet</span>**\n\nTabNet uses sequential attention to choose which features to reason from at each decision step, enabling interpretability and more efficient learning as the learning capacity is used for the most salient features. TabNet outperforms other neural network and decision tree variants on a wide range of non-performance-saturated tabular datasets and yields interpretable feature attributions plus insights into the global model behavior. \n\nThe main features of TabNet are \n\nThe main contributions are summarized as:\n\n📌 TabNet inputs raw tabular data without any preprocessing\n\n📌 TabNet uses sequential attention to choose which features to reason from at each decision step, enabling interpretability and better learning as the learning capacity\n\n📌 TabNet outperforms or is on par with other tabular learning models on various datasets for classification and regression problems from different domains\n\n📌 TabNet shows significant performance improvements by using unsupervised pre-training to predict masked features \n\n![](https://drive.google.com/uc?id=1snKduiQHakIeulnr7jKwt2uQvmv8rDcl)\n\n[Source](https://arxiv.org/pdf/1908.07442.pdf)\n\n# **<span style=\"color:#F7B2B0;\">TabNet for Timeseries Data</span>**\n\nSome resources using TabNet for timeseries data\n\n[Github](https://github.com/AlbertoCastelo/tabnet-timeseries-spike)\n\nShort Term Load Forecasting using TabNet - MDPI\n\nRainfall Forecast using TabNet - MDPI\n\n\nThe below explanation is taken from medium article [here](https://towardsdatascience.com/tabnet-e1b979907694)\n\n# **<span style=\"color:#F7B2B0;\">Steps:</span>**\n\nEach Step is a block of components. The number of Steps is a hyperparameter option when training the model. Increasing the number of steps will increase the learning capacity of the model, but will also increase training time, memory usage and the chance of overfitting.Each Step gets its own vote in the final classification and these votes are equally weighted. This mimics an ensemble classification.\n\n# **<span style=\"color:#F7B2B0;\">Feature Transformer:</span>**\n\nThe Feature Transformer is a network which has an architecture of its own.It has multiple layers, some of which are shared across every Step while others are unique to each Step. Each layer contains a fully connected layer, batch normalisation and a Gated Linear Unit activiation.\n\nSharing some layers between decision Steps leads to “parameter-efficient and robust learning with high capacity” and that normalization with root 0.5 “helps to stabilize learning by ensuring that the variance throughout does not change dramatically”. The output of the feature transformer uses a ReLU activation function.\n\n![](https://drive.google.com/uc?id=1iuVE-7hkmh2ZMFfY3FdrZ1UbptidK-mI)\n\n# **<span style=\"color:#F7B2B0;\">Feature Selection :</span>**\n\nOnce features have been transformed, they are passed to the Attentive Transformer and the Mask for feature selection.The Attentive Transformer is comprised of a fully connected layer, batch normalisation and Sparsemax normalisation. It also includes prior scales, meaning it knows how much each feature has been used by the previous steps. This is used to derive the Mask using the processed features from the previous Feature Transformer.\n\n![](https://drive.google.com/uc?id=12PNJHZqt7bso16m0H8NZ0wrDq9uLdX0U)\n\nThe Mask ensures the model focuses on the most important features and is also used to derive explainability. It essentially covers up features, meaning the model is only able to use those that have been considered important by the Attentive Transformer.We can also understand feature importance by looking at how much a feature has been masked for all decisions and and an individual prediction.\nTabNet employs soft feature selection with controllable sparsity in end-to-end learning\nThis means one model jointly performs feature selection and output mapping, which leads to better performance.TabNet uses instance-wise feature selection, which means features are selected for each input and each prediction can use different features.\nThis feature selection is essential as it allows decision boundaries to be generalised to a linear combination of features, where coefficients determine the proportion of each feature, which in the end leads to the model’s interpretability","metadata":{}},{"cell_type":"code","source":"!pip install -q tensorflow_addons","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from wandb.keras import WandbCallback, WandbMetricsLogger\nrun = wandb.init(project = 'stanford_rna',\n                 save_code = True,\n                 name='tabnet'\n                 \n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow_addons.activations import sparsemax\nimport tensorflow as tf\nimport random\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model, clone_model\nfrom tensorflow.keras.layers import Input, Dropout, Dense, ReLU, BatchNormalization, Activation, Concatenate\nfrom copy import deepcopy\n\n\ndef GLU(x):\n    return x * tf.sigmoid(x)\n\nclass FCBlock(layers.Layer):\n    def __init__(self, units):\n        super().__init__()\n        self.layer = layers.Dense(units)\n        self.bn = layers.BatchNormalization()\n\n    def call(self, x):\n        return GLU(self.bn(self.layer(x)))\n\nclass SharedBlock(layers.Layer):\n    def __init__(self, units, mult=tf.sqrt(0.5)):\n        super().__init__()\n        self.layer1 = FCBlock(units)\n        self.layer2 = FCBlock(units)\n        self.mult = mult\n\n    def call(self, x):\n        out1 = self.layer1(x)\n        out2 = self.layer2(out1)\n        return out2 + self.mult * out1\n\nclass DecisionBlock(SharedBlock):\n    def __init__(self, units, mult=tf.sqrt(0.5)):\n        super().__init__(units, mult)\n\n    def call(self, x):\n        out1 = x * self.mult + self.layer1(x)\n        out2 = out1 * self.mult + self.layer2(out1)\n        return out2\n\nclass Prior(layers.Layer):\n    def __init__(self, gamma=1.1):\n        super().__init__()\n        self.gamma = gamma\n\n    def reset(self):\n        self.P = 1.0\n\n    def call(self, mask):\n        self.P = self.P * (self.gamma - mask)\n        return self.P\nclass AttentiveTransformer(layers.Layer):\n    def __init__(self, units):\n        super().__init__()\n        self.layer = layers.Dense(units)\n        self.bn = layers.BatchNormalization()\n\n    def call(self, x, prior):\n        return sparsemax(prior * self.bn(self.layer(x)))\nclass TabNet(keras.Model):\n    def __init__(self, input_dim, output_dim, steps, n_d, n_a, gamma=1.3):\n        super().__init__()\n        # hyper-parameters\n        self.n_d, self.n_a, self.steps = n_d, n_a, steps\n        # input-normalisation\n        self.bn = layers.BatchNormalization()\n        # Feature Transformer\n        self.shared = SharedBlock(n_d+n_a)\n        self.first_block = DecisionBlock(n_d+n_a)\n        self.decision_blocks = [DecisionBlock(n_d+n_a)] * steps\n        # Attentive Transformer\n        self.attention = [AttentiveTransformer(input_dim)] * steps\n        self.prior_scale = Prior(gamma)\n        # final layer\n        self.final = layers.Dense(output_dim,activation='relu')\n\n        self.eps = 1e-8\n        self.add_layer = layers.Add()\n\n    @tf.function\n    def call(self, x):\n        self.prior_scale.reset()\n        final_outs = []\n        mask_losses = []\n\n        x = self.bn(x)\n        attention = self.first_block(self.shared(x))[:,:self.n_a]\n        for i in range(self.steps):\n            mask = self.attention[i](attention, self.prior_scale.P)\n            entropy = mask * tf.math.log(mask + self.eps)\n            mask_losses.append(\n                -tf.reduce_sum(entropy, axis=-1) / self.steps\n            )\n\n            prior = self.prior_scale(mask)\n            out = self.decision_blocks[i](self.shared(x * prior))\n            attention, output = out[:,:self.n_a], out[:,self.n_a:]\n            final_outs.append(tf.nn.relu(output))\n\n        final_out = self.add_layer(final_outs)\n        mask_loss = self.add_layer(mask_losses)\n\n        return self.final(final_out)\n\n    def mask_importance(self, x):\n        self.prior_scale.reset()\n        feature_importance = 0\n\n        x = self.bn(x)\n        attention = self.first_block(self.shared(x))[:,:self.n_a]\n        for i in range(self.steps):\n            mask = self.attention[i](attention, self.prior_scale.P)\n\n            prior = self.prior_scale(mask)\n            out = self.decision_blocks[i](self.shared(x * prior))\n            attention, output = out[:,:self.n_a], out[:,self.n_a:]\n            step_importance = tf.reduce_sum(tf.nn.relu(output), axis=1, keepdims=True)\n            feature_importance += mask * step_importance\n\n        return feature_importance\n    ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"basemodel = TabNet(128, 64, 2, 10, 10, 1.3)\n\ninp = Input(X_train.shape[1])\nl = BatchNormalization()(inp)\nl = Dense(128,activation='relu')(l)\nl = basemodel(l)\nl = Activation('relu')(l)\nout = Dense(1,activation='sigmoid')(l)\n\nmodel = Model(inp, out)\nmodel.compile(Adam(0.00001), 'mae',metrics=['mae'])\nmodel.summary()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(X_train, y_train,epochs=10,batch_size = 128*2, validation_data=(X_val,y_val),callbacks=[WandbMetricsLogger()])\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run.finish()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **<span style=\"color:#F7B2B0;\">GrowNet</span>**","metadata":{}},{"cell_type":"code","source":"from wandb.keras import WandbCallback, WandbMetricsLogger\nrun = wandb.init(project = 'stanford_rna',\n                 save_code = True,\n                 name='grownet'\n                 \n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport random\nfrom copy import deepcopy\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model, clone_model\nfrom tensorflow.keras.layers import Input, Dropout, Dense, ReLU, BatchNormalization, Activation, Concatenate\n\nclass DynamicNet(object):\n    def __init__(self, c0 = None, lr = None, concat_input=False, additive_boosting=False, encoder_layers=None):\n        self.models = []\n        self.c0 = tf.Variable(np.float32(c0) if c0 is not None else random.uniform(0.0, 1.0))\n        self.lr = lr\n        self.boost_rate  = tf.Variable(lr if lr is not None else random.uniform(0.0, 1.0))\n        self.concat_input = None\n        self.additive_boosting = False\n        self.encoder_layers = encoder_layers\n    def freeze_all_networks(self):\n        for model in self.models:\n            for l in model.layers: l.trainable = False\n    def unfreeze_all_networks(self):\n        for model in self.models:\n            for l in model.layers: l.trainable = True\n    def add(self, model):\n        last_activation = 'sigmoid'\n        if last_activation in ['sigmoid', 'softmax']: model.layers[-1].activation = None\n        if hasattr(model, 'optimizer') and model.optimizer is not None: self.loss = model.loss\n        if hasattr(model, 'optimizer') and model.optimizer is not None: self.optimizer = model.optimizer\n        if hasattr(model, 'optimizer') and model.optimizer is not None: self.lr = model.optimizer.lr if self.lr is None else self.lr\n        if len(self.models) == 0:\n            self.models = [model]\n            self.full_model = self.models[-1]\n            self.embed_full_model = self.models[-1]\n        else: self.models.append(model)\n        full_inp = Input(shape=177)\n        out_orig = self.embed_full_model(full_inp)\n        out = out_orig\n        if self.concat_input: out = Concatenate()([out, full_inp])\n        if len(self.models) > 1:\n            # print(out)\n            # print(K.int_shape(out))\n            if K.int_shape(out) != K.int_shape(self.models[-2].input): out = Dense(K.int_shape(self.models[-1].input)[-1])(out)\n            new_out = self.models[-1](out)\n        else: new_out = self.models[-1](full_inp)\n        new_full_out = (self.c0) + (self.boost_rate * new_out)\n        self.full_model = Model(full_inp, Activation(last_activation)(new_full_out))\n        if self.encoder_layers is not None:\n            if len(self.models) > 1: self.embed_full_model = Model(full_inp, Model(self.models[-1].input, self.models[-1].layers[self.encoder_layers].output)(out))\n            else: \n              # print(self.models)\n              # print(self.models[-1].input)\n              # print(self.models[-1].layers[self.encoder_layers])\n              self.embed_full_model = Model(full_inp, Model(self.models[-1].input, self.models[-1].layers[self.encoder_layers].output)(full_inp))\n        else: self.embed_full_model = Model(full_inp, new_out)\n\n    def fit(self, x_train, y_train, lr, w_decay=0.0, epochs=10, validation_data = None, **kwargs):\n        if self.optimizer is None: optimizer = Adam(lr, decay=w_decay)\n        else: optimizer =self.optimizer\n        self.full_model.compile(optimizer, self.loss,['mae'])\n        self.full_model.fit(x_train, y_train, epochs=epochs, validation_data = (x_train, y_train), **kwargs)\n    def predict(self, x_train, **kwargs): return self.full_model.predict(x_train, **kwargs)\n\n\nclass GradientBoost(object):\n    def __init__(self, base_model, lr = 1e-3, weight_decay = 1e-5, early_stopping_steps = 5, batch_size = 256, correct_epoch = 1, model_order = \"second\", n_boosting_rounds = 5 , boost_rate = 1.0, hidden_size=512, epochs_per_stage=5, encoder_layers=2):\n        self.lr = lr\n        self.base_model = base_model\n        self.batch_size = batch_size\n        self.boost_rate = boost_rate\n        self.model_order = model_order\n        self.hidden_size = hidden_size\n        self.weight_decay = weight_decay\n        self.num_nets = n_boosting_rounds\n        self.encoder_layers = encoder_layers\n        self.correct_epoch = correct_epoch\n        self.epochs_per_stage = epochs_per_stage\n        self.early_stopping_steps = early_stopping_steps\n        self.base_model.summary()\n\n    def fit(self, x_train, y_train, validation_data = None, n_boosting_rounds=None, correct_epoch=None, epochs_per_stage=None, **kwargs):\n        self.num_nets = n_boosting_rounds if n_boosting_rounds is not None else self.num_nets\n        self.correct_epoch = correct_epoch if correct_epoch is not None else self.correct_epoch\n        self.epochs_per_stage = epochs_per_stage if epochs_per_stage is not None else self.epochs_per_stage\n        x_val , y_val = validation_data if validation_data is not None else (None, None)\n        net_ensemble = DynamicNet(concat_input=True, encoder_layers=self.encoder_layers)\n        lr = self.lr\n        L2 = self.weight_decay\n        for stage in range(self.num_nets):\n            print(\"Stage is {}\".format(stage))\n            params = {}\n            params[\"feat_d\"] = x_train.shape[1]\n            params[\"hidden_size\"] = self.hidden_size\n            new_model = self.base_model\n            new_model.optimizer = self.base_model.optimizer\n            new_model.loss = self.base_model.loss\n            net_ensemble.freeze_all_networks()\n            net_ensemble.add(new_model)\n            net_ensemble.fit(x_train, y_train,self.lr, epochs=self.epochs_per_stage,batch_size=self.batch_size,  validation_data = validation_data,callbacks=[WandbCallback()])\n            lr_scaler = 2\n            if stage != 0:\n                if stage % 3 == 0: lr /= 2\n                net_ensemble.unfreeze_all_networks()\n                net_ensemble.fit(x_train, y_train,lr / lr_scaler, epochs=self.correct_epoch,batch_size=self.batch_size,  w_decay=L2, validation_data = validation_data,callbacks=[WandbCallback()])\n        self.model = net_ensemble\n\n    def predict(self, x_test, **kwargs): return self.model.predict(x_test, **kwargs)\n\n\ninp = Input(X_train.shape[1])\nl = BatchNormalization()(inp)\nl = ReLU()(l)\nl = Dense(64)(l)\nl = BatchNormalization()(l)\nl = ReLU()(l)\nl = Dropout(0.4)(l)\nl = Dense(64)(l)\nl = BatchNormalization()(l)\nl = ReLU()(l)\nout = Dense(1, activation='sigmoid')(l)\nsimple_model = Model(inp, out)\nsimple_model.compile(Adam(0.001), 'mae',metrics=['mae'])\nbs = 128*2\nmodel = GradientBoost(simple_model,batch_size=bs)\nmodel.fit(X_train[:,:,0], y_train,epochs=10,batch_size=bs,validation_data=(X_val[:,:,0],y_val))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run.finish()","metadata":{},"execution_count":null,"outputs":[]}]}