{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# TensorFlow Transformer Starter - LB 0.790\nIn this notebook we present starter code for a transformer model. Using a transformer requires 3D data (whereas Kaggle provides 2D data as a CSV). The shape of the data is `(number_of_customers, 13, 188)` which is `(batch size, sequence length, feature length)`. Each customer is a time series with 13 credit card statements. And each statement has 188 features. The data was created and saved to NumPy files in my previous notebook [here][1] with data discussion [here][6] and [here][7]. EDA displaying customer time series is [here][2].\n\nKeras provides tutorials on transformers [here][3] and [here][4]. This simple transformer was used in Kaggle's Ventilator Comp and achieved solo model gold medal [here][5]\n\n# TensorFlow GRU (RNN) Starter - LB 0.790\nIf you want to experiment with RNN, (i.e LSTM or GRU), check out my TensorFlow GRU Starter [here][1] with discussion [here][8]. The data used in this notebook was created in my GRU starter notebook. Both RNNs and Transformers require 3D data.\n\n[1]: https://www.kaggle.com/code/cdeotte/tensorflow-gru-starter-0-790\n[2]: https://www.kaggle.com/cdeotte/time-series-eda\n[3]: https://keras.io/examples/nlp/text_classification_with_transformer/\n[4]: https://www.tensorflow.org/text/tutorials/transformer\n[5]: https://www.kaggle.com/code/cdeotte/tensorflow-transformer-0-112\n[6]: https://www.kaggle.com/competitions/amex-default-prediction/discussion/327828\n[7]: https://www.kaggle.com/competitions/amex-default-prediction/discussion/328054\n[8]: https://www.kaggle.com/competitions/amex-default-prediction/discussion/327761","metadata":{}},{"cell_type":"markdown","source":"# Load Libraries","metadata":{}},{"cell_type":"code","source":"import cupy, cudf # GPU LIBRARIES\nimport numpy as np, pandas as pd # CPU LIBRARIES\nimport matplotlib.pyplot as plt, gc, os\n\nPATH_TO_DATA = '../input/amex-data-for-transformers-and-rnns/data/'\n\n# IF YOU WISH TO INFER A MODEL YOU TRAINED OFFLINE\n# THEN SET TO FALSE AND PROVIDE KAGGLE DATASET URL\nTRAIN_MODEL = True\nPATH_TO_MODEL = './model/'\n\nINFER_TEST = True","metadata":{"execution":{"iopub.status.busy":"2022-05-31T19:09:21.771349Z","iopub.execute_input":"2022-05-31T19:09:21.771721Z","iopub.status.idle":"2022-05-31T19:09:21.776995Z","shell.execute_reply.started":"2022-05-31T19:09:21.771691Z","shell.execute_reply":"2022-05-31T19:09:21.776234Z"},"_kg_hide-input":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build Transformer Model","metadata":{}},{"cell_type":"code","source":"os.environ[\"TF_GPU_ALLOCATOR\"]=\"cuda_malloc_async\"  # TF will not use all memory\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n\nprint('Using TensorFlow version',tf.__version__)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-31T19:09:21.799884Z","iopub.execute_input":"2022-05-31T19:09:21.800429Z","iopub.status.idle":"2022-05-31T19:09:27.726176Z","shell.execute_reply.started":"2022-05-31T19:09:21.800394Z","shell.execute_reply":"2022-05-31T19:09:27.725177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TransformerBlock(layers.Layer):\n    def __init__(self, embed_dim, feat_dim, num_heads, ff_dim, rate=0.1):\n        super(TransformerBlock, self).__init__()\n        self.att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n        self.ffn = keras.Sequential(\n            [layers.Dense(ff_dim, activation=\"gelu\"), layers.Dense(feat_dim),]\n        )\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.dropout1 = layers.Dropout(rate)\n        self.dropout2 = layers.Dropout(rate)\n\n    def call(self, inputs, training):\n        attn_output = self.att(inputs, inputs)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(inputs + attn_output)\n        ffn_output = self.ffn(out1)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        return self.layernorm2(out1 + ffn_output)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T19:09:27.728098Z","iopub.execute_input":"2022-05-31T19:09:27.729106Z","iopub.status.idle":"2022-05-31T19:09:27.738651Z","shell.execute_reply.started":"2022-05-31T19:09:27.729063Z","shell.execute_reply":"2022-05-31T19:09:27.737849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feat_dim = 188\nembed_dim = 64  # Embedding size for attention\nnum_heads = 4  # Number of attention heads\nff_dim = 128  # Hidden layer size in feed forward network inside transformer\ndropout_rate = 0.3\nnum_blocks = 2\n\ndef build_model():\n    \n    # INPUT EMBEDDING LAYER\n    inp = layers.Input(shape=(13,188))\n    embeddings = []\n    for k in range(11):\n        emb = layers.Embedding(10,4)\n        embeddings.append( emb(inp[:,:,k]) )\n    x = layers.Concatenate()([inp[:,:,11:]]+embeddings)\n    x = layers.Dense(feat_dim)(x)\n    \n    # TRANSFORMER BLOCKS\n    for k in range(num_blocks):\n        x_old = x\n        transformer_block = TransformerBlock(embed_dim, feat_dim, num_heads, ff_dim, dropout_rate)\n        x = transformer_block(x)\n        x = 0.9*x + 0.1*x_old # SKIP CONNECTION\n    \n    # CLASSIFICATION HEAD\n    x = layers.Dense(64, activation=\"relu\")(x[:,-1,:])\n    x = layers.Dense(32, activation=\"relu\")(x)\n    outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n    \n    model = keras.Model(inputs=inp, outputs=outputs)\n    opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n    loss = tf.keras.losses.BinaryCrossentropy()\n    model.compile(loss=loss, optimizer = opt)\n        \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-05-31T19:09:27.739898Z","iopub.execute_input":"2022-05-31T19:09:27.740435Z","iopub.status.idle":"2022-05-31T19:09:27.758021Z","shell.execute_reply.started":"2022-05-31T19:09:27.740397Z","shell.execute_reply":"2022-05-31T19:09:27.757322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Learning Schedule","metadata":{}},{"cell_type":"code","source":"import math\nLR_START = 1e-6\nLR_MAX = 1e-3\nLR_MIN = 1e-6\nLR_RAMPUP_EPOCHS = 0\nLR_SUSTAIN_EPOCHS = 0\nEPOCHS = 8\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        decay_total_epochs = EPOCHS - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS - 1\n        decay_epoch_index = epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS\n        phase = math.pi * decay_epoch_index / decay_total_epochs\n        cosine_decay = 0.5 * (1 + math.cos(phase))\n        lr = (LR_MAX - LR_MIN) * cosine_decay + LR_MIN\n    return lr\n\nrng = [i for i in range(EPOCHS)]\nlr_y = [lrfn(x) for x in rng]\nplt.figure(figsize=(10, 4))\nplt.plot(rng, lr_y, '-o')\nplt.xlabel('Epoch'); plt.ylabel('LR')\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\". \\\n      format(lr_y[0], max(lr_y), lr_y[-1]))\nLR = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T19:31:44.904122Z","iopub.execute_input":"2022-05-31T19:31:44.904510Z","iopub.status.idle":"2022-05-31T19:31:45.136625Z","shell.execute_reply.started":"2022-05-31T19:31:44.904476Z","shell.execute_reply":"2022-05-31T19:31:45.135801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Competition Metric","metadata":{}},{"cell_type":"code","source":"# COMPETITION METRIC FROM Konstantin Yakovlev\n# https://www.kaggle.com/kyakovlev\n# https://www.kaggle.com/competitions/amex-default-prediction/discussion/327534\ndef amex_metric_mod(y_true, y_pred):\n\n    labels     = np.transpose(np.array([y_true, y_pred]))\n    labels     = labels[labels[:, 1].argsort()[::-1]]\n    weights    = np.where(labels[:,0]==0, 20, 1)\n    cut_vals   = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four   = np.sum(cut_vals[:,0]) / np.sum(labels[:,0])\n\n    gini = [0,0]\n    for i in [1,0]:\n        labels         = np.transpose(np.array([y_true, y_pred]))\n        labels         = labels[labels[:, i].argsort()[::-1]]\n        weight         = np.where(labels[:,0]==0, 20, 1)\n        weight_random  = np.cumsum(weight / np.sum(weight))\n        total_pos      = np.sum(labels[:, 0] *  weight)\n        cum_pos_found  = np.cumsum(labels[:, 0] * weight)\n        lorentz        = cum_pos_found / total_pos\n        gini[i]        = np.sum((lorentz - weight_random) * weight)\n\n    return 0.5 * (gini[1]/gini[0] + top_four)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T19:09:27.985576Z","iopub.execute_input":"2022-05-31T19:09:27.986241Z","iopub.status.idle":"2022-05-31T19:09:27.996245Z","shell.execute_reply.started":"2022-05-31T19:09:27.986199Z","shell.execute_reply":"2022-05-31T19:09:27.995508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Model","metadata":{}},{"cell_type":"code","source":"if TRAIN_MODEL:\n    # SAVE TRUE AND OOF\n    true = np.array([])\n    oof = np.array([])\n    VERBOSE = 2 # use 1 for interactive \n\n    for fold in range(5):\n\n        # INDICES OF TRAIN AND VALID FOLDS\n        valid_idx = [2*fold+1, 2*fold+2]\n        train_idx = [x for x in [1,2,3,4,5,6,7,8,9,10] if x not in valid_idx]\n\n        print('#'*25)\n        print(f'### Fold {fold+1} with valid files', valid_idx)\n\n        # READ TRAIN DATA FROM DISK\n        X_train = []; y_train = []\n        for k in train_idx:\n            X_train.append( np.load(f'{PATH_TO_DATA}data_{k}.npy'))\n            y_train.append( pd.read_parquet(f'{PATH_TO_DATA}targets_{k}.pqt') )\n        X_train = np.concatenate(X_train,axis=0)\n        y_train = pd.concat(y_train).target.values\n        print('### Training data shapes', X_train.shape, y_train.shape)\n\n        # READ VALID DATA FROM DISK\n        X_valid = []; y_valid = []\n        for k in valid_idx:\n            X_valid.append( np.load(f'{PATH_TO_DATA}data_{k}.npy'))\n            y_valid.append( pd.read_parquet(f'{PATH_TO_DATA}targets_{k}.pqt') )\n        X_valid = np.concatenate(X_valid,axis=0)\n        y_valid = pd.concat(y_valid).target.values\n        print('### Validation data shapes', X_valid.shape, y_valid.shape)\n        print('#'*25)\n\n        # BUILD AND TRAIN MODEL\n        K.clear_session()\n        model = build_model()\n        h = model.fit(X_train,y_train, \n                      validation_data = (X_valid,y_valid),\n                      batch_size=512, epochs=EPOCHS, verbose=VERBOSE,\n                      callbacks = [LR])\n        if not os.path.exists(PATH_TO_MODEL): os.makedirs(PATH_TO_MODEL)\n        model.save_weights(f'{PATH_TO_MODEL}transformer_fold_{fold+1}.h5')\n\n        # INFER VALID DATA\n        print('Inferring validation data...')\n        p = model.predict(X_valid, batch_size=512, verbose=VERBOSE).flatten()\n\n        print()\n        print(f'Fold {fold+1} CV=', amex_metric_mod(y_valid, p) )\n        print()\n        true = np.concatenate([true, y_valid])\n        oof = np.concatenate([oof, p])\n        \n        # CLEAN MEMORY\n        del model, X_train, y_train, X_valid, y_valid, p\n        gc.collect()\n\n    # PRINT OVERALL RESULTS\n    print('#'*25)\n    print(f'Overall CV =', amex_metric_mod(true, oof) )","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-05-31T19:09:27.999393Z","iopub.execute_input":"2022-05-31T19:09:27.999732Z","iopub.status.idle":"2022-05-31T19:29:31.659812Z","shell.execute_reply.started":"2022-05-31T19:09:27.999696Z","shell.execute_reply":"2022-05-31T19:29:31.657883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Infer Test Data","metadata":{}},{"cell_type":"code","source":"if INFER_TEST:\n    # BUILD MODEL\n    K.clear_session()\n    model = build_model()\n    \n    # LOAD SAMPLE SUBMISSION\n    start = 0; end = 0\n    sub = cudf.read_csv('../input/amex-default-prediction/sample_submission.csv')\n    \n    # REARANGE SUB ROWS TO MATCH 20 TEST FILES\n    sub['hash'] = sub['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\n    test_hash_index = cupy.load(f'{PATH_TO_DATA}test_hashes_data.npy')\n    sub = sub.set_index('hash').loc[test_hash_index].reset_index(drop=True)\n    \n    for k in range(20):\n        print(f'Inferring Test_File_{k+1}')\n        X_test = np.load(f'{PATH_TO_DATA}test_data_{k+1}.npy')\n        end = start + X_test.shape[0]\n\n        # INFER 5 FOLD MODELS\n        model.load_weights(f'{PATH_TO_MODEL}transformer_fold_1.h5')\n        p = model.predict(X_test, batch_size=512, verbose=0).flatten() \n        for j in range(1,5):\n            model.load_weights(f'{PATH_TO_MODEL}transformer_fold_{j+1}.h5')\n            p += model.predict(X_test, batch_size=512, verbose=0).flatten()\n        p /= 5.0\n\n        sub.loc[start:end-1,'prediction'] = p\n        start = end","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-05-31T19:32:03.638504Z","iopub.execute_input":"2022-05-31T19:32:03.639105Z","iopub.status.idle":"2022-05-31T19:37:17.455802Z","shell.execute_reply.started":"2022-05-31T19:32:03.639057Z","shell.execute_reply":"2022-05-31T19:37:17.454012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Submission","metadata":{}},{"cell_type":"code","source":"if INFER_TEST:\n    sub.to_csv(f'submission.csv',index=False)\n    print('Submission file shape is', sub.shape )\n    display( sub.head() )","metadata":{"execution":{"iopub.status.busy":"2022-05-31T19:37:44.332771Z","iopub.execute_input":"2022-05-31T19:37:44.334686Z","iopub.status.idle":"2022-05-31T19:37:44.686999Z","shell.execute_reply.started":"2022-05-31T19:37:44.334630Z","shell.execute_reply":"2022-05-31T19:37:44.686147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if INFER_TEST:\n    # DISPLAY SUBMISSION PREDICTIONS\n    plt.hist(sub.to_pandas().prediction, bins=100)\n    plt.title('Test Predictions')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-31T19:38:35.111504Z","iopub.execute_input":"2022-05-31T19:38:35.112221Z","iopub.status.idle":"2022-05-31T19:38:35.912221Z","shell.execute_reply.started":"2022-05-31T19:38:35.112183Z","shell.execute_reply":"2022-05-31T19:38:35.911371Z"},"trusted":true},"execution_count":null,"outputs":[]}]}