{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[],"dockerImageVersionId":30198,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# Load Libraries","metadata":{}},{"cell_type":"code","source":"import cupy, cudf # GPU LIBRARIES\nimport numpy as np, pandas as pd # CPU LIBRARIES\nimport matplotlib.pyplot as plt, gc, os\n\nPATH_TO_DATA = '../input/amex-data-for-transformers-and-rnns/data/'\n\n# IF YOU WISH TO INFER A MODEL YOU TRAINED OFFLINE\n# THEN SET TO FALSE AND PROVIDE KAGGLE DATASET URL\nTRAIN_MODEL = True\nPATH_TO_MODEL = './model/'\n\nINFER_TEST = True","metadata":{"execution":{"iopub.status.busy":"2022-05-31T19:09:21.771349Z","iopub.execute_input":"2022-05-31T19:09:21.771721Z","iopub.status.idle":"2022-05-31T19:09:21.776995Z","shell.execute_reply.started":"2022-05-31T19:09:21.771691Z","shell.execute_reply":"2022-05-31T19:09:21.776234Z"},"_kg_hide-input":false,"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Build Transformer Model","metadata":{}},{"cell_type":"code","source":"os.environ[\"TF_GPU_ALLOCATOR\"]=\"cuda_malloc_async\"  # TF will not use all memory\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n\nprint('Using TensorFlow version',tf.__version__)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-31T19:09:21.799884Z","iopub.execute_input":"2022-05-31T19:09:21.800429Z","iopub.status.idle":"2022-05-31T19:09:27.726176Z","shell.execute_reply.started":"2022-05-31T19:09:21.800394Z","shell.execute_reply":"2022-05-31T19:09:27.725177Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class TransformerBlock(layers.Layer):\n    def __init__(self, embed_dim, feat_dim, num_heads, ff_dim, rate=0.1):\n        super(TransformerBlock, self).__init__()\n        self.att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n        self.ffn = keras.Sequential(\n            [layers.Dense(ff_dim, activation=\"gelu\"), layers.Dense(feat_dim),]\n        )\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.dropout1 = layers.Dropout(rate)\n        self.dropout2 = layers.Dropout(rate)\n\n    def call(self, inputs, training):\n        attn_output = self.att(inputs, inputs)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(inputs + attn_output)\n        ffn_output = self.ffn(out1)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        return self.layernorm2(out1 + ffn_output)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T19:09:27.728098Z","iopub.execute_input":"2022-05-31T19:09:27.729106Z","iopub.status.idle":"2022-05-31T19:09:27.738651Z","shell.execute_reply.started":"2022-05-31T19:09:27.729063Z","shell.execute_reply":"2022-05-31T19:09:27.737849Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feat_dim = 188\nembed_dim = 64  # Embedding size for attention\nnum_heads = 4  # Number of attention heads\nff_dim = 128  # Hidden layer size in feed forward network inside transformer\ndropout_rate = 0.3\nnum_blocks = 2\n\ndef build_model():\n    \n    # INPUT EMBEDDING LAYER\n    inp = layers.Input(shape=(13,188))\n    embeddings = []\n    for k in range(11):\n        emb = layers.Embedding(10,4)\n        embeddings.append( emb(inp[:,:,k]) )\n    x = layers.Concatenate()([inp[:,:,11:]]+embeddings)\n    x = layers.Dense(feat_dim)(x)\n    \n    # TRANSFORMER BLOCKS\n    for k in range(num_blocks):\n        x_old = x\n        transformer_block = TransformerBlock(embed_dim, feat_dim, num_heads, ff_dim, dropout_rate)\n        x = transformer_block(x)\n        x = 0.9*x + 0.1*x_old # SKIP CONNECTION\n    \n    # CLASSIFICATION HEAD\n    x = layers.Dense(64, activation=\"relu\")(x[:,-1,:])\n    x = layers.Dense(32, activation=\"relu\")(x)\n    outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n    \n    model = keras.Model(inputs=inp, outputs=outputs)\n    opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n    loss = tf.keras.losses.BinaryCrossentropy()\n    model.compile(loss=loss, optimizer = opt)\n        \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-05-31T19:09:27.739898Z","iopub.execute_input":"2022-05-31T19:09:27.740435Z","iopub.status.idle":"2022-05-31T19:09:27.758021Z","shell.execute_reply.started":"2022-05-31T19:09:27.740397Z","shell.execute_reply":"2022-05-31T19:09:27.757322Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Define Learning Schedule","metadata":{}},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-05-31T19:31:44.904122Z","iopub.execute_input":"2022-05-31T19:31:44.90451Z","iopub.status.idle":"2022-05-31T19:31:45.136625Z","shell.execute_reply.started":"2022-05-31T19:31:44.904476Z","shell.execute_reply":"2022-05-31T19:31:45.135801Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Define Competition Metric","metadata":{}},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-05-31T19:09:27.985576Z","iopub.execute_input":"2022-05-31T19:09:27.986241Z","iopub.status.idle":"2022-05-31T19:09:27.996245Z","shell.execute_reply.started":"2022-05-31T19:09:27.986199Z","shell.execute_reply":"2022-05-31T19:09:27.995508Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train Model","metadata":{}},{"cell_type":"code","source":"","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-05-31T19:09:27.999393Z","iopub.execute_input":"2022-05-31T19:09:27.999732Z","iopub.status.idle":"2022-05-31T19:29:31.659812Z","shell.execute_reply.started":"2022-05-31T19:09:27.999696Z","shell.execute_reply":"2022-05-31T19:29:31.657883Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Infer Test Data","metadata":{}},{"cell_type":"code","source":"","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-05-31T19:32:03.638504Z","iopub.execute_input":"2022-05-31T19:32:03.639105Z","iopub.status.idle":"2022-05-31T19:37:17.455802Z","shell.execute_reply.started":"2022-05-31T19:32:03.639057Z","shell.execute_reply":"2022-05-31T19:37:17.454012Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Create Submission","metadata":{}},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-05-31T19:37:44.332771Z","iopub.execute_input":"2022-05-31T19:37:44.334686Z","iopub.status.idle":"2022-05-31T19:37:44.686999Z","shell.execute_reply.started":"2022-05-31T19:37:44.33463Z","shell.execute_reply":"2022-05-31T19:37:44.686147Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-05-31T19:38:35.111504Z","iopub.execute_input":"2022-05-31T19:38:35.112221Z","iopub.status.idle":"2022-05-31T19:38:35.912221Z","shell.execute_reply.started":"2022-05-31T19:38:35.112183Z","shell.execute_reply":"2022-05-31T19:38:35.911371Z"},"trusted":true},"outputs":[],"execution_count":null}]}