{"cells":[{"metadata":{},"cell_type":"markdown","source":"The reference of this notebook is as follows :\n\n1.https://tensorflow.google.cn/tutorials/text/transformer\n\n2.our host's papers (https://arxiv.org/abs/2002.07033, https://arxiv.org/abs/2010.12042)"},{"metadata":{},"cell_type":"markdown","source":"![](http://)"},{"metadata":{},"cell_type":"markdown","source":"# **Main ideas is as follows：**\n\n1.Basic transformer：  Portugese-English translation . I consider contend_id as Portugese，answered_correctly as English, so SAINT is contend_id -answered_correctly translation.\n\n\n2.modify create_masks function\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nfrom sklearn.metrics import roc_auc_score\nimport time\nimport matplotlib.pyplot as plt\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Data**"},{"metadata":{"trusted":true},"cell_type":"code","source":"target = 'answered_correctly'\ncontent_count=13523\ntarget_count=2\n\nMAX_LENGTH=50\n\nBUFFER_SIZE = 2000\nBATCH_SIZE = 128\n\n#train parameter\n#The values used in the base model of transformer were:\n#num_layers=6, d_model = 512, dff = 2048. See the paper for all the other versions of the transformer.\n\n#SAINT:The window size, dropout rate, and batch size are set to 100, 0.1, and 128 respectively.\n#The best performing model of SAINT has 4 layers and a latent space dimension of 512.\nnum_layers = 4\nd_model = 128\ndff = 512\nnum_heads = 8\n\n# input_vocab_size = content_count + 1\n# target_vocab_size = target_count + 1\n\n#content_id have 0，so add 1\ninput_vocab_size = content_count+1\ntarget_vocab_size = target_count\ndropout_rate = 0.1\n\n#\nEPOCHS = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndtype = {'timestamp':'int64', \n         'user_id':'int32' ,\n         'content_id':'int16',\n         'content_type_id':'int8',\n         'answered_correctly':'int8'}\n\ntrain_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', \n                       usecols=[1, 2, 3, 4, 7], \n                       dtype=dtype,\n                       nrows=3300*10000\n                      )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##content_id have 0，so add 1\ntrain_df['content_id'] += 1\n# train_df['answered_correctly'] += 1\n\n\ntrain_df = train_df[train_df.content_type_id == False]\n\nvalid_df=train_df[3000*10000:]\ntrain_df=train_df[0:100*10000]\n\ntrain_dataset = train_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id').apply(lambda r:[\n            r['content_id'].values,\n            r['answered_correctly'].values])\n\nvalid_dataset = valid_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id').apply(lambda r:[\n            r['content_id'].values,\n            r['answered_correctly'].values])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_df\ndel valid_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('len(train_dataset):',len(train_dataset))\nprint('len(valid_dataset):',len(valid_dataset))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset=pd.DataFrame(train_dataset)\ntrain_dataset=train_dataset.reset_index()\ntrain_dataset.columns = ['user_id','q_qa']\n#\nvalid_dataset=pd.DataFrame(valid_dataset)\nvalid_dataset=valid_dataset.reset_index()\nvalid_dataset.columns = ['user_id','q_qa']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def getlimitdata(x):    \n    seq_len = len(x)\n    r = np.zeros(MAX_LENGTH, dtype=int)\n\n    if seq_len >= MAX_LENGTH:          \n        r[:] = x[-MAX_LENGTH:]\n       \n    else:        \n        r[-seq_len:] = x\n            \n    return r\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset['q']=train_dataset['q_qa'].apply(lambda r:getlimitdata(r[0]))\ntrain_dataset['qa']=train_dataset['q_qa'].apply(lambda r:getlimitdata(r[1]))\ntrain_dataset.drop(columns=['q_qa'], inplace=True)\n\nvalid_dataset['q']=valid_dataset['q_qa'].apply(lambda r:getlimitdata(r[0]))\nvalid_dataset['qa']=valid_dataset['q_qa'].apply(lambda r:getlimitdata(r[1]))\nvalid_dataset.drop(columns=['q_qa'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"qs=[]\nfor q in train_dataset['q'].values:\n    qs.append(q.tolist())\nqas=[]\nfor qa in train_dataset['qa'].values:\n    qas.append(qa.tolist())\n    \nqs_valid=[]\nfor q in valid_dataset['q'].values:\n    qs_valid.append(q.tolist())\nqas_valid=[]\nfor qa in valid_dataset['qa'].values:\n    qas_valid.append(qa.tolist())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = tf.data.Dataset.from_tensor_slices((qs,qas))\nvalid_dataset = tf.data.Dataset.from_tensor_slices((qs_valid,qas_valid))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del qs\ndel qas\ndel qs_valid\ndel qas_valid","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = train_dataset.cache()\ntrain_dataset = train_dataset.shuffle(BUFFER_SIZE).padded_batch(BATCH_SIZE)\ntrain_dataset = train_dataset.prefetch(tf.data.experimental.AUTOTUNE)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_dataset = valid_dataset.padded_batch(2000)#BATCH_SIZE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"q_batch, qa_batch =next(iter(train_dataset))\nq_batch, qa_batch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"qa_batch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"qa_batch[:, :-1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#tar_real = \nqa_batch[:, 1:]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Model**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_angles(pos, i, d_model):\n    angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))\n    return pos * angle_rates\n\ndef positional_encoding(position, d_model):\n    angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n                          np.arange(d_model)[np.newaxis, :],\n                          d_model)\n\n    # apply sin to even indices in the array; 2i\n    angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n\n    # apply cos to odd indices in the array; 2i+1\n    angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n\n    pos_encoding = angle_rads[np.newaxis, ...]\n\n    return tf.cast(pos_encoding, dtype=tf.float32)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Mask**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_padding_mask(seq):\n    seq = tf.cast(tf.math.equal(seq, 0), tf.float32)#\n\n    # add extra dimensions to add the padding\n    # to the attention logits.\n    return seq[:, tf.newaxis, tf.newaxis, :]  # (batch_size, 1, 1, seq_len)\n\ndef create_look_ahead_mask(size):\n    mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n    return mask  # (seq_len, seq_len)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Scaled dot product attention**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def scaled_dot_product_attention(q, k, v, mask):\n#     \"\"\"Calculate the attention weights.\n#       q, k, v must have matching leading dimensions.\n#       k, v must have matching penultimate dimension, i.e.: seq_len_k = seq_len_v.\n#       The mask has different shapes depending on its type(padding or look ahead) \n#       but it must be broadcastable for addition.\n\n#       Args:\n#         q: query shape == (..., seq_len_q, depth)\n#         k: key shape == (..., seq_len_k, depth)\n#         v: value shape == (..., seq_len_v, depth_v)\n#         mask: Float tensor with shape broadcastable \n#               to (..., seq_len_q, seq_len_k). Defaults to None.\n\n#       Returns:\n#         output, attention_weights\n#     \"\"\"\n\n    matmul_qk = tf.matmul(q, k, transpose_b=True)  # (..., seq_len_q, seq_len_k)\n\n    # scale matmul_qk\n    dk = tf.cast(tf.shape(k)[-1], tf.float32)\n    scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n\n    # add the mask to the scaled tensor.\n    if mask is not None:\n        scaled_attention_logits += (mask * -1e9)  \n\n    # softmax is normalized on the last axis (seq_len_k) so that the scores\n    # add up to 1.\n    attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)  # (..., seq_len_q, seq_len_k)\n\n    output = tf.matmul(attention_weights, v)  # (..., seq_len_q, depth_v)\n\n    return output, attention_weights","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Multi-head attention**"},{"metadata":{"trusted":true},"cell_type":"code","source":"class MultiHeadAttention(tf.keras.layers.Layer):\n    def __init__(self, d_model, num_heads):\n        super(MultiHeadAttention, self).__init__()\n        self.num_heads = num_heads\n        self.d_model = d_model\n\n        assert d_model % self.num_heads == 0\n\n        self.depth = d_model // self.num_heads\n\n        self.wq = tf.keras.layers.Dense(d_model)\n        self.wk = tf.keras.layers.Dense(d_model)\n        self.wv = tf.keras.layers.Dense(d_model)\n\n        self.dense = tf.keras.layers.Dense(d_model)\n\n    def split_heads(self, x, batch_size):\n       \n        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n        return tf.transpose(x, perm=[0, 2, 1, 3])\n\n    def call(self, v, k, q, mask):\n        batch_size = tf.shape(q)[0]\n\n        q = self.wq(q)  # (batch_size, seq_len, d_model)\n        k = self.wk(k)  # (batch_size, seq_len, d_model)\n        v = self.wv(v)  # (batch_size, seq_len, d_model)\n\n        q = self.split_heads(q, batch_size)  # (batch_size, num_heads, seq_len_q, depth)\n        k = self.split_heads(k, batch_size)  # (batch_size, num_heads, seq_len_k, depth)\n        v = self.split_heads(v, batch_size)  # (batch_size, num_heads, seq_len_v, depth)\n\n        # scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)\n        # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)\n        scaled_attention, attention_weights = scaled_dot_product_attention(\n            q, k, v, mask)\n\n        scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])  # (batch_size, seq_len_q, num_heads, depth)\n\n        concat_attention = tf.reshape(scaled_attention, \n                                      (batch_size, -1, self.d_model))  # (batch_size, seq_len_q, d_model)\n\n        output = self.dense(concat_attention)  # (batch_size, seq_len_q, d_model)\n\n        return output, attention_weights","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Point wise feed forward network**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def point_wise_feed_forward_network(d_model, dff):\n    return tf.keras.Sequential([\n      tf.keras.layers.Dense(dff, activation='relu'),  # (batch_size, seq_len, dff)\n      tf.keras.layers.Dense(d_model)  # (batch_size, seq_len, d_model)\n    ])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Encoder and decoder**"},{"metadata":{"trusted":true},"cell_type":"code","source":"class EncoderLayer(tf.keras.layers.Layer):\n    def __init__(self, d_model, num_heads, dff, rate=0.1):\n        super(EncoderLayer, self).__init__()\n\n        self.mha = MultiHeadAttention(d_model, num_heads)\n        self.ffn = point_wise_feed_forward_network(d_model, dff)\n\n        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n\n        self.dropout1 = tf.keras.layers.Dropout(rate)\n        self.dropout2 = tf.keras.layers.Dropout(rate)\n\n    def call(self, x, training, mask):\n\n        attn_output, _ = self.mha(x, x, x, mask)  # (batch_size, input_seq_len, d_model)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(x + attn_output)  # (batch_size, input_seq_len, d_model)\n\n        ffn_output = self.ffn(out1)  # (batch_size, input_seq_len, d_model)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        out2 = self.layernorm2(out1 + ffn_output)  # (batch_size, input_seq_len, d_model)\n\n        return out2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Encoder(tf.keras.layers.Layer):\n    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size,\n               maximum_position_encoding, rate=0.1):\n        super(Encoder, self).__init__()\n\n        self.d_model = d_model\n        self.num_layers = num_layers\n\n        self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)\n        self.pos_encoding = positional_encoding(maximum_position_encoding, \n                                                self.d_model)\n\n\n        self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate) \n                           for _ in range(num_layers)]\n\n        self.dropout = tf.keras.layers.Dropout(rate)\n\n    def call(self, x, training, mask):\n\n        seq_len = tf.shape(x)[1]\n\n        # \n        x = self.embedding(x)  # (batch_size, input_seq_len, d_model)\n        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n        x += self.pos_encoding[:, :seq_len, :]\n\n        x = self.dropout(x, training=training)\n\n        for i in range(self.num_layers):\n            x = self.enc_layers[i](x, training, mask)\n\n        return x  # (batch_size, input_seq_len, d_model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DecoderLayer(tf.keras.layers.Layer):\n    def __init__(self, d_model, num_heads, dff, rate=0.1):\n        super(DecoderLayer, self).__init__()\n\n        self.mha1 = MultiHeadAttention(d_model, num_heads)\n        self.mha2 = MultiHeadAttention(d_model, num_heads)\n\n        self.ffn = point_wise_feed_forward_network(d_model, dff)\n\n        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n\n        self.dropout1 = tf.keras.layers.Dropout(rate)\n        self.dropout2 = tf.keras.layers.Dropout(rate)\n        self.dropout3 = tf.keras.layers.Dropout(rate)\n\n\n    def call(self, x, enc_output, training, \n           look_ahead_mask, padding_mask):\n    # enc_output.shape == (batch_size, input_seq_len, d_model)\n\n        attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask)  # (batch_size, target_seq_len, d_model)\n        attn1 = self.dropout1(attn1, training=training)\n        out1 = self.layernorm1(attn1 + x)\n\n        attn2, attn_weights_block2 = self.mha2(\n            enc_output, enc_output, out1, padding_mask)  # (batch_size, target_seq_len, d_model)\n        attn2 = self.dropout2(attn2, training=training)\n        out2 = self.layernorm2(attn2 + out1)  # (batch_size, target_seq_len, d_model)\n\n        ffn_output = self.ffn(out2)  # (batch_size, target_seq_len, d_model)\n        ffn_output = self.dropout3(ffn_output, training=training)\n        out3 = self.layernorm3(ffn_output + out2)  # (batch_size, target_seq_len, d_model)\n\n        return out3, attn_weights_block1, attn_weights_block2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Decoder(tf.keras.layers.Layer):\n    def __init__(self, num_layers, d_model, num_heads, dff, target_vocab_size,\n               maximum_position_encoding, rate=0.1):\n        super(Decoder, self).__init__()\n\n        self.d_model = d_model\n        self.num_layers = num_layers\n\n        self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)\n        self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)\n\n        self.dec_layers = [DecoderLayer(d_model, num_heads, dff, rate) \n                           for _ in range(num_layers)]\n        self.dropout = tf.keras.layers.Dropout(rate)\n\n    def call(self, x, enc_output, training, \n           look_ahead_mask, padding_mask):\n\n        seq_len = tf.shape(x)[1]\n        attention_weights = {}\n\n        x = self.embedding(x)  # (batch_size, target_seq_len, d_model)\n        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n        x += self.pos_encoding[:, :seq_len, :]\n\n        x = self.dropout(x, training=training)\n\n        for i in range(self.num_layers):\n            x, block1, block2 = self.dec_layers[i](x, enc_output, training,\n                                                 look_ahead_mask, padding_mask)\n\n            attention_weights['decoder_layer{}_block1'.format(i+1)] = block1\n            attention_weights['decoder_layer{}_block2'.format(i+1)] = block2\n\n        # x.shape == (batch_size, target_seq_len, d_model)\n        return x, attention_weights","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# ** Transformer**"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"class Transformer(tf.keras.Model):\n    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, \n               target_vocab_size, pe_input, pe_target, rate=0.1):\n        super(Transformer, self).__init__()\n\n        self.encoder = Encoder(num_layers, d_model, num_heads, dff, \n                               input_vocab_size, pe_input, rate)\n\n        self.decoder = Decoder(num_layers, d_model, num_heads, dff, \n                               target_vocab_size, pe_target, rate)\n\n        self.final_layer = tf.keras.layers.Dense(target_vocab_size,activation='softmax')\n\n    def call(self, inp, tar, training, enc_padding_mask, \n           look_ahead_mask, dec_padding_mask):\n\n        enc_output = self.encoder(inp, training, enc_padding_mask)  # (batch_size, inp_seq_len, d_model)\n\n        # dec_output.shape == (batch_size, tar_seq_len, d_model)\n        dec_output, attention_weights = self.decoder(\n            tar, enc_output, training, look_ahead_mask, dec_padding_mask)\n\n        final_output = self.final_layer(dec_output)  # (batch_size, tar_seq_len, target_vocab_size)\n        \n        #增加的        \n        #final_output = final_output[: ,-1:, :]  # (batch_size, 1, vocab_size)\n#         final_output = tf.keras.layers.GlobalAveragePooling1D()(dec_output)\n#         final_output = tf.keras.layers.Dropout(0.1)(final_output)         \n#         final_output = tf.keras.layers.Dense(1, activation='sigmoid', name='output')(final_output)\n\n        return final_output, attention_weights","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Optimizer**"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n    def __init__(self, d_model, warmup_steps=4000):\n        super(CustomSchedule, self).__init__()\n\n        self.d_model = d_model\n        self.d_model = tf.cast(self.d_model, tf.float32)\n\n        self.warmup_steps = warmup_steps\n\n    def __call__(self, step):\n        arg1 = tf.math.rsqrt(step)\n        arg2 = step * (self.warmup_steps ** -1.5)\n\n        return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learning_rate = CustomSchedule(d_model)\n#SAINT:We use the Adam optimizer  with lr =0:001;b1 = 0:9;b2 = 0:999 and epsilon = 1e􀀀8.\noptimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98, \n                                     epsilon=1e-9)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Loss and metrics**"},{"metadata":{"trusted":true},"cell_type":"code","source":"loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n    from_logits=True, reduction='none')\n\ndef loss_function(mask,real, pred):\n    #mask = tf.math.logical_not(tf.math.equal(real, 0))\n    loss_ = loss_object(real, pred)\n\n    mask = tf.cast(mask, dtype=loss_.dtype)\n    loss_ *= mask\n\n    return tf.reduce_sum(loss_)/tf.reduce_sum(mask)\n\n\ndef accuracy_function(mask,real, pred):\n    real = tf.cast(real, dtype=tf.int64)\n    accuracies = tf.equal(real, tf.argmax(pred, axis=2))\n\n    #mask = tf.math.logical_not(tf.math.equal(real, 0))\n    accuracies = tf.math.logical_and(mask, accuracies)\n\n    accuracies = tf.cast(accuracies, dtype=tf.float32)\n    mask = tf.cast(mask, dtype=tf.float32)\n    return tf.reduce_sum(accuracies)/tf.reduce_sum(mask)\n\ntrain_loss = tf.keras.metrics.Mean(name='train_loss')\ntrain_accuracy = tf.keras.metrics.Mean(name='train_accuracy')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Training and checkpointing**"},{"metadata":{"trusted":true},"cell_type":"code","source":"transformer = Transformer(num_layers, d_model, num_heads, dff,\n                          input_vocab_size, target_vocab_size, \n                          pe_input=50, \n                          pe_target=50,\n                          rate=dropout_rate)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_masks(inp, tar):\n    # \n    enc_padding_mask = create_look_ahead_mask(tf.shape(inp)[1]) \n    dec_padding_mask = create_look_ahead_mask(tf.shape(inp)[1])\n    dec_padding_mask=dec_padding_mask[1:,]\n    combined_mask = create_look_ahead_mask(tf.shape(tar)[1])\n\n    return enc_padding_mask, combined_mask, dec_padding_mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def create_masks(inp, tar):\n#     # \n#     enc_padding_mask = create_padding_mask(inp)\n#     look_ahead_mask_in = create_look_ahead_mask(tf.shape(inp)[1])\n#     enc_padding_mask = tf.maximum(enc_padding_mask, look_ahead_mask_in)\n    \n#     # Used in the 2nd attention block in the decoder.\n#     # This padding mask is used to mask the encoder outputs.\n#     dec_padding_mask = create_padding_mask(inp)\n\n#     # Used in the 1st attention block in the decoder.\n#     # It is used to pad and mask future tokens in the input received by \n#     # the decoder.\n#     look_ahead_mask = create_look_ahead_mask(tf.shape(tar)[1])\n    \n#     #tar_inp = tar[:, :-1] input is same with out,but input have no 0,so use input create mask\n#     #dec_target_padding_mask = create_padding_mask(tar)\n#     dec_target_padding_mask = create_padding_mask(inp[:, :-1])\n#     combined_mask = tf.maximum(dec_target_padding_mask, look_ahead_mask)\n\n#     return enc_padding_mask, combined_mask, dec_padding_mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint_path = \"../input/riiid-saint-tf-transformer/checkpoints/train\"\n\nckpt = tf.train.Checkpoint(transformer=transformer,\n                           optimizer=optimizer)\n\nckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)\n\n# \nif ckpt_manager.latest_checkpoint:\n    ckpt.restore(ckpt_manager.latest_checkpoint)\n    print ('Latest checkpoint restored!!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# The @tf.function trace-compiles train_step into a TF graph for faster\n# execution. The function specializes to the precise shape of the argument\n# tensors. To avoid re-tracing due to the variable sequence lengths or variable\n# batch sizes (the last batch is smaller), use input_signature to specify\n# more generic shapes.\n\n# train_step_signature = [\n#     tf.TensorSpec(shape=(None, None), dtype=tf.int64),\n#     tf.TensorSpec(shape=(None, None), dtype=tf.int64),\n# ]\n\n# @tf.function(input_signature=train_step_signature)\ndef train_step(inp, tar):\n    tar_inp = tar[:, :-1]\n    tar_real = tar[:, 1:]\n    #sigmoid,just last tar\n    #tar_real=tar[: ,-1]\n    \n    mask = tf.math.logical_not(tf.math.equal(inp[:, 1:], 0))\n\n    enc_padding_mask, combined_mask, dec_padding_mask = create_masks(inp, tar_inp)\n\n    with tf.GradientTape() as tape:\n        predictions, _ = transformer(inp, tar_inp, \n                                     True, \n                                     enc_padding_mask, \n                                     combined_mask, \n                                     dec_padding_mask)\n        loss = loss_function(mask,tar_real, predictions)\n\n    gradients = tape.gradient(loss, transformer.trainable_variables)    \n    optimizer.apply_gradients(zip(gradients, transformer.trainable_variables))\n\n    train_loss(loss)\n    train_accuracy(accuracy_function(mask,tar_real, predictions))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('start train....')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for epoch in range(EPOCHS):\n#     start = time.time()\n\n#     train_loss.reset_states()\n#     train_accuracy.reset_states()\n\n#     # inp -> portuguese, tar -> english\n#     for (batch, (inp, tar)) in enumerate(train_dataset):\n#         train_step(inp, tar)\n\n#         if batch % 10 == 0:\n#             print ('Epoch {} Batch {} Loss {:.4f} accuracy {:.4f}'.format(\n#               epoch + 1, batch, train_loss.result(), train_accuracy.result()))\n\n#     if (epoch + 1) % 5 == 0:\n#         ckpt_save_path = ckpt_manager.save()\n#         print ('Saving checkpoint for epoch {} at {}'.format(epoch+1,\n#                                                              ckpt_save_path))\n\n#     print ('Epoch {} Loss {:.4f} accuracy {:.4f}'.format(epoch + 1, \n#                                                 train_loss.result(), \n#                                                 train_accuracy.result()))\n\n#     print ('Time taken for 1 epoch: {} secs\\n'.format(time.time() - start))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import datetime","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ni=0\nfor (batch, (encoder_input, qa_batch)) in enumerate(valid_dataset):\n    start = time.time()\n    #label=qa_batch[:, 1:]\n    label=qa_batch[: ,-1]\n    tar_inp = qa_batch[:, :-1]\n    print(encoder_input.shape)\n\n    enc_padding_mask, combined_mask, dec_padding_mask = create_masks(\n            encoder_input, tar_inp)\n    \n    predictions, attention_weights = transformer(encoder_input, \n                                                     tar_inp,\n                                                     False,\n                                                     enc_padding_mask,\n                                                     combined_mask,\n                                                     dec_padding_mask)\n    \n    predictions = predictions[: ,-1:, :]  # (batch_size, 1, vocab_size)\n    predictions=tf.nn.softmax(predictions)[:,:,-1]\n    end = time.time()\n    print (end - start)\n    print('valid-auc:', roc_auc_score(label, predictions))\n    i=i+1\n    if i==3:\n        break\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(encoder_input.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dtype = {'timestamp':'int64', \n         'user_id':'int32' ,\n         'content_id':'int16',\n         'content_type_id':'int8',\n         'answered_correctly':'int8'}\n\ntrain_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', \n                       usecols=[1, 2, 3, 4, 7], \n                       dtype=dtype,\n                       #nrows=120*10000\n                      )\ntrain_df = train_df[train_df.content_type_id == False]\n#when train ,content_id add 1\ntrain_df['content_id'] += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain_dataset = train_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id').apply(lambda r:[\n            r['content_id'].values,\n            r['answered_correctly'].values])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset[115]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import defaultdict\ntrain_dataset=train_dataset.to_dict(defaultdict(int))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset[115]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nif 10000 in train_dataset.keys():\n    print(100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain_dataset[110]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#train_dataset[115][0]=np.concatenate((train_dataset[115][0],[100+1]))\n#train_dataset2[115]\ntrain_dataset[110]=[np.array([100+1]),np.array([1])]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def getpredata(user_id,content_id):\n    #\n    if user_id in train_dataset.keys():\n        #produce encoder_input, tar_inp\n        q_qa=train_dataset[user_id]\n        q=q_qa[0]\n        qa=q_qa[1]\n        #q.append(content_id)\n        #concatenate new content_id\n        q=np.concatenate((q,[content_id]))\n        #\n        q=getlimitdata(q)\n        qa=getlimitdata(qa)\n    else:\n        q = np.zeros(MAX_LENGTH-1, dtype=int)\n        q=np.concatenate((q,[content_id]))\n        qa=np.zeros(MAX_LENGTH, dtype=int)\n    \n    return q, qa\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def getpredatas(user_ids,content_ids):\n    encoder_inputs=[]\n    tar_inps=[]\n    for user_id, content_id in zip(user_ids,content_ids):\n        encoder_input, tar_inp=getpredata(user_id,content_id)\n        tar_inp=tar_inp[1:]#\n        \n        encoder_inputs.append(encoder_input.tolist())\n        tar_inps.append(tar_inp.tolist())\n    \n    encoder_inputs=tf.convert_to_tensor(encoder_inputs)\n    tar_inps=tf.convert_to_tensor(tar_inps)\n    return encoder_inputs,tar_inps\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', \n                       usecols=[1, 2, 3, 4, 7], \n                       dtype=dtype,\n                       nrows=1*10000\n                      )\ntest_df=test_df[0:108]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nstart = time.time()\ntest_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\nprint(len(test_df))\nuser_ids = test_df['user_id'].values\ncontent_ids = test_df['content_id'].values\n\n\nencoder_inputs, tar_inps=getpredatas(user_ids,content_ids)\n\nenc_padding_mask, combined_mask, dec_padding_mask = create_masks(\n    encoder_inputs, tar_inps)\nprint(time.time()-start)\npredictions, attention_weights = transformer(encoder_inputs, \n                                             tar_inps,\n                                             False,\n                                             enc_padding_mask,\n                                             combined_mask,\n                                             dec_padding_mask)\nprint(time.time()-start)\npredictions = predictions[: ,-1:, :]  # (batch_size, 1, vocab_size)\npredictions=tf.nn.softmax(predictions)[:,:,-1]\n\ntest_df[target]=np.squeeze(predictions.numpy())\nprint(time.time()-start)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\nenv = riiideducation.make_env()\niter_test = env.iter_test()\nprior_test_df = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nfor (test_df, sample_prediction_df) in iter_test: \n    if prior_test_df is not None:\n        prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])\n        prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)\n        user_ids = prior_test_df['user_id'].values\n        content_ids = prior_test_df['content_id'].values\n        targets = prior_test_df[target].values \n        for user_id, answered_correctly,content_id in zip(user_ids,targets,content_ids):\n            #when train ,content_id add 1\n            if user_id in train_dataset.keys():\n                train_dataset[user_id][0]=np.concatenate((train_dataset[user_id][0],[content_id+1]))\n                train_dataset[user_id][1]=np.concatenate((train_dataset[user_id][1],[answered_correctly]))\n            else:\n                train_dataset[user_id]=[np.array([content_id+1]),np.array([answered_correctly])]\n             \n            \n        \n    prior_test_df = test_df.copy()\n    \n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n    \n    user_ids = test_df['user_id'].values\n    content_ids = test_df['content_id'].values\n    \n   \n \n    encoder_inputs, tar_inps=getpredatas(user_ids,content_ids)\n    enc_padding_mask, combined_mask, dec_padding_mask = create_masks(\n        encoder_inputs, tar_inps)\n    predictions, attention_weights = transformer(encoder_inputs, \n                                                 tar_inps,\n                                                 False,\n                                                 enc_padding_mask,\n                                                 combined_mask,\n                                                 dec_padding_mask)\n    predictions = predictions[: ,-1:, :]  # (batch_size, 1, vocab_size)\n    predictions=tf.nn.softmax(predictions)[:,:,-1]\n    \n    test_df[target]=np.squeeze(predictions.numpy())\n       \n    #    \n    env.predict(test_df[['row_id', target]])\n    ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}