{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport time\nimport numpy as np\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def get_angles(pos, i, d_model):\n    angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))\n    return pos * angle_rates\n\ndef positional_encoding(position, d_model):\n    angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n                          np.arange(d_model)[np.newaxis, :],\n                          d_model)\n\n    # apply sin to even indices in the array; 2i\n    angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n\n    # apply cos to odd indices in the array; 2i+1\n    angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n\n    pos_encoding = angle_rads[np.newaxis, ...]\n\n    return tf.cast(pos_encoding, dtype=tf.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pos_encoding = positional_encoding(50, 512)\nprint (pos_encoding.shape)\n\nplt.pcolormesh(pos_encoding[0], cmap='RdBu')\nplt.xlabel('Depth')\nplt.xlim((0, 512))\nplt.ylabel('Position')\nplt.colorbar()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_padding_mask(seq):\n    seq = tf.cast(tf.math.equal(seq, 0), tf.float32)\n\n    # add extra dimensions to add the padding\n    # to the attention logits.\n    return seq[:, tf.newaxis, tf.newaxis, :]  # (batch_size, 1, 1, seq_len)\n\ndef create_look_ahead_mask(size):\n    mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n    return mask  # (seq_len, seq_len)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def scaled_dot_product_attention(q, k, v, mask):\n    \"\"\"Calculate the attention weights.\n    q, k, v must have matching leading dimensions.\n    k, v must have matching penultimate dimension, i.e.: seq_len_k = seq_len_v.\n    The mask has different shapes depending on its type(padding or look ahead) \n    but it must be broadcastable for addition.\n\n    Args:\n    q: query shape == (..., seq_len_q, depth)\n    k: key shape == (..., seq_len_k, depth)\n    v: value shape == (..., seq_len_v, depth_v)\n    mask: Float tensor with shape broadcastable \n          to (..., seq_len_q, seq_len_k). Defaults to None.\n\n    Returns:\n    output, attention_weights\n    \"\"\"\n\n    matmul_qk = tf.matmul(q, k, transpose_b=True)  # (..., seq_len_q, seq_len_k)\n\n    # scale matmul_qk\n    dk = tf.cast(tf.shape(k)[-1], tf.float32)\n    scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n\n    # add the mask to the scaled tensor.\n    if mask is not None:\n        scaled_attention_logits += (mask * -1e9)  \n\n    # softmax is normalized on the last axis (seq_len_k) so that the scores\n    # add up to 1.\n    attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)  # (..., seq_len_q, seq_len_k)\n\n    output = tf.matmul(attention_weights, v)  # (..., seq_len_q, depth_v)\n\n    return output, attention_weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MultiHeadAttention(tf.keras.layers.Layer):\n    def __init__(self, d_model, num_heads):\n        super(MultiHeadAttention, self).__init__()\n        self.num_heads = num_heads\n        self.d_model = d_model\n\n        assert d_model % self.num_heads == 0\n\n        self.depth = d_model // self.num_heads\n\n        self.wq = tf.keras.layers.Dense(d_model)\n        self.wk = tf.keras.layers.Dense(d_model)\n        self.wv = tf.keras.layers.Dense(d_model)\n\n        self.dense = tf.keras.layers.Dense(d_model)\n\n    def split_heads(self, x, batch_size):\n        \"\"\"Split the last dimension into (num_heads, depth).\n        Transpose the result such that the shape is (batch_size, num_heads, seq_len, depth)\n        \"\"\"\n        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n        return tf.transpose(x, perm=[0, 2, 1, 3])\n\n    def call(self, v, k, q, mask):\n        batch_size = tf.shape(q)[0]\n\n        q = self.wq(q)  # (batch_size, seq_len, d_model)\n        k = self.wk(k)  # (batch_size, seq_len, d_model)\n        v = self.wv(v)  # (batch_size, seq_len, d_model)\n\n        q = self.split_heads(q, batch_size)  # (batch_size, num_heads, seq_len_q, depth)\n        k = self.split_heads(k, batch_size)  # (batch_size, num_heads, seq_len_k, depth)\n        v = self.split_heads(v, batch_size)  # (batch_size, num_heads, seq_len_v, depth)\n\n        # scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)\n        # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)\n        scaled_attention, attention_weights = scaled_dot_product_attention(\n            q, k, v, mask)\n\n        scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])  # (batch_size, seq_len_q, num_heads, depth)\n\n        concat_attention = tf.reshape(scaled_attention, \n                                      (batch_size, -1, self.d_model))  # (batch_size, seq_len_q, d_model)\n\n        output = self.dense(concat_attention)  # (batch_size, seq_len_q, d_model)\n\n        return output, attention_weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp_mha = MultiHeadAttention(d_model=512, num_heads=8)\ny = tf.random.uniform((1, 60, 512))  # (batch_size, encoder_sequence, d_model)\nout, attn = temp_mha(y, k=y, q=y, mask=None)\nout.shape, attn.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def point_wise_feed_forward_network(d_model, dff):\n    return tf.keras.Sequential([\n      tf.keras.layers.Dense(dff, activation='relu'),  # (batch_size, seq_len, dff)\n      tf.keras.layers.Dense(d_model)  # (batch_size, seq_len, d_model)\n  ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class EncoderLayer(tf.keras.layers.Layer):\n    def __init__(self, d_model, num_heads, dff, rate=0.1):\n        super(EncoderLayer, self).__init__()\n\n        self.mha = MultiHeadAttention(d_model, num_heads)\n        self.ffn = point_wise_feed_forward_network(d_model, dff)\n\n        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n\n        self.dropout1 = tf.keras.layers.Dropout(rate)\n        self.dropout2 = tf.keras.layers.Dropout(rate)\n\n    def call(self, x, training, mask):\n\n        attn_output, _ = self.mha(x, x, x, mask)  # (batch_size, input_seq_len, d_model)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(x + attn_output)  # (batch_size, input_seq_len, d_model)\n\n        ffn_output = self.ffn(out1)  # (batch_size, input_seq_len, d_model)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        out2 = self.layernorm2(out1 + ffn_output)  # (batch_size, input_seq_len, d_model)\n\n        return out2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DecoderLayer(tf.keras.layers.Layer):\n    def __init__(self, d_model, num_heads, dff, rate=0.1):\n        super(DecoderLayer, self).__init__()\n\n        self.mha1 = MultiHeadAttention(d_model, num_heads)\n        self.mha2 = MultiHeadAttention(d_model, num_heads)\n\n        self.ffn = point_wise_feed_forward_network(d_model, dff)\n\n        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n\n        self.dropout1 = tf.keras.layers.Dropout(rate)\n        self.dropout2 = tf.keras.layers.Dropout(rate)\n        self.dropout3 = tf.keras.layers.Dropout(rate)\n\n\n    def call(self, x, enc_output, training, \n           look_ahead_mask, padding_mask):\n        # enc_output.shape == (batch_size, input_seq_len, d_model)\n\n        attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask)  # (batch_size, target_seq_len, d_model)\n        attn1 = self.dropout1(attn1, training=training)\n        out1 = self.layernorm1(attn1 + x)\n\n        attn2, attn_weights_block2 = self.mha2(\n            enc_output, enc_output, out1, padding_mask)  # (batch_size, target_seq_len, d_model)\n        attn2 = self.dropout2(attn2, training=training)\n        out2 = self.layernorm2(attn2 + out1)  # (batch_size, target_seq_len, d_model)\n\n        ffn_output = self.ffn(out2)  # (batch_size, target_seq_len, d_model)\n        ffn_output = self.dropout3(ffn_output, training=training)\n        out3 = self.layernorm3(ffn_output + out2)  # (batch_size, target_seq_len, d_model)\n\n        return out3, attn_weights_block1, attn_weights_block2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Encoder(tf.keras.layers.Layer):\n    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size,\n               maximum_position_encoding,emb_matrix, inp2_vocab_size,inp3_vocab_size,dec_inp2_size,dec_inp3_size,rate=0.1):\n        super(Encoder, self).__init__()\n\n        self.d_model = d_model\n        self.num_layers = num_layers\n        \n        \n#         self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model,weights=[emb_matrix],trainable=True)\n        self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)\n        \n        self.embedding2 = tf.keras.layers.Embedding(inp2_vocab_size, d_model)\n        \n        self.embedding3 = tf.keras.layers.Embedding(inp3_vocab_size, d_model)\n        \n        self.pos_encoding = positional_encoding(maximum_position_encoding, \n                                                self.d_model)\n\n\n        self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate) \n                           for _ in range(num_layers)]\n\n        self.dropout = tf.keras.layers.Dropout(rate)\n\n    def call(self, x,x2,x3, training, mask):\n\n        seq_len = tf.shape(x)[1]\n\n        # adding embedding and position encoding.\n        x = self.embedding(x)  # (batch_size, input_seq_len, d_model)\n#         x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n        \n        x2 = self.embedding2(x2)  # (batch_size, input_seq_len, d_model)\n#         x2 *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n        \n        x3 = self.embedding3(x3)  # (batch_size, input_seq_len, d_model)\n#         x3 *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n        \n        x=x+x2+x3\n        \n        x += self.pos_encoding[:, :seq_len, :]\n\n        x = self.dropout(x, training=training)\n\n        for i in range(self.num_layers):\n            x = self.enc_layers[i](x, training, mask)\n\n        return x  # (batch_size, input_seq_len, d_model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Decoder(tf.keras.layers.Layer):\n    def __init__(self, num_layers, d_model, num_heads, dff, target_vocab_size,\n               maximum_position_encoding,emb_matrix, inp2_vocab_size,dec_inp2_size,dec_inp3_size,rate=0.1):\n        super(Decoder, self).__init__()\n\n        self.d_model = d_model\n        self.num_layers = num_layers\n\n        self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)\n        self.embedding_2=tf.keras.layers.Embedding(dec_inp2_size, d_model)\n        self.embedding_3=tf.keras.layers.Embedding(dec_inp3_size, d_model)\n        \n#         self.embedding_4=tf.keras.layers.Dense(d_model)\n        \n        self.embedding_41 = tf.keras.layers.Dense(d_model)\n        \n        self.embedding_4 = tf.keras.layers.TimeDistributed(self.embedding_41)\n        \n        self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)\n\n        self.dec_layers = [DecoderLayer(d_model, num_heads, dff, rate) \n                           for _ in range(num_layers)]\n        self.dropout = tf.keras.layers.Dropout(rate)\n\n    def call(self, x,x2,x3,x4, enc_output, training, \n           look_ahead_mask, padding_mask):\n\n        seq_len = tf.shape(x)[1]\n        attention_weights = {}\n\n        x = self.embedding(x)  # (batch_size, target_seq_len, d_model)\n        \n#         x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n        \n        x2 = self.embedding_2(x2)  # (batch_size, target_seq_len, d_model)\n        \n#         x2 *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n        \n        x3 = self.embedding_3(x3)  # (batch_size, target_seq_len, d_model)\n        \n#         x3 *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n        \n        x4 = self.embedding_4(x4)  # (batch_size, target_seq_len, d_model)\n#         print(x4.shape)\n        \n#         x4 *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n        \n        x=x+x2+x3+x4\n        \n        x += self.pos_encoding[:, :seq_len, :]\n\n        x = self.dropout(x, training=training)\n\n        for i in range(self.num_layers):\n            x, block1, block2 = self.dec_layers[i](x, enc_output, training,\n                                                 look_ahead_mask, padding_mask)\n\n            attention_weights['decoder_layer{}_block1'.format(i+1)] = block1\n            attention_weights['decoder_layer{}_block2'.format(i+1)] = block2\n\n        # x.shape == (batch_size, target_seq_len, d_model)\n        return x, attention_weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Transformer(tf.keras.Model):\n    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, \n               target_vocab_size, pe_input, pe_target, emb_matrix, inp2_vocab_size,inp3_vocab_size,dec_inp2_size,dec_inp3_size,rate=0.1):\n        super(Transformer, self).__init__()\n\n        self.encoder = Encoder(num_layers, d_model, num_heads, dff, \n                               input_vocab_size, pe_input,emb_matrix, inp2_vocab_size,inp3_vocab_size,dec_inp2_size,dec_inp3_size,rate)\n\n        self.decoder = Decoder(num_layers, d_model, num_heads, dff, \n                               target_vocab_size, pe_target,emb_matrix, inp2_vocab_size,dec_inp2_size,dec_inp3_size,rate)\n   \n        self.final_layer = tf.keras.layers.Dense(3,activation='softmax')\n        \n        self.final_layer_1 = tf.keras.layers.TimeDistributed(self.final_layer)\n\n    def call(self, inp1,inp2,inp3,tar1,tar2,tar3,tar4,training, enc_padding_mask, \n           look_ahead_mask, dec_padding_mask):\n\n        enc_output = self.encoder(inp1,inp2,inp3, training, enc_padding_mask)  # (batch_size, inp_seq_len, d_model)\n\n        # dec_output.shape == (batch_size, tar_seq_len, d_model)\n        dec_output, attention_weights = self.decoder(\n            tar1,tar2,tar3,tar4,enc_output, training, look_ahead_mask, dec_padding_mask)\n\n        final_output = self.final_layer_1(dec_output)  # (batch_size, tar_seq_len, target_vocab_size)\n#         print(final_output.shape)\n        return final_output, attention_weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import MinMaxScaler\nimport tensorflow as tf\n\nfrom keras.models import Sequential\nfrom keras.layers import Activation\nfrom keras.models import Model\nfrom keras.layers import Input, LSTM, Dense\nfrom keras.models import Sequential\nfrom keras.layers import LSTM, Dense, Dropout, Masking, Embedding, RNN\nfrom keras import optimizers\nfrom keras.layers import TimeDistributed\nfrom keras.layers import Bidirectional\nfrom keras.layers import RepeatVector\n# from keras_contrib.layers import CRF\n\nfrom keras.layers import Conv2D,concatenate,MaxPool2D,MaxPooling2D,Flatten\nfrom keras.layers import Lambda\nfrom keras.layers.core import Reshape\nfrom keras.utils import to_categorical\nimport datatable as dt\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\nimport os\n\nimport riiideducation\n\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.preprocessing import LabelEncoder, OneHotEncoder, StandardScaler, MultiLabelBinarizer\nimport gc\nfrom keras.layers import Reshape\nimport gensim\nimport pickle\nfrom keras import layers\nfrom sklearn.model_selection import train_test_split\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv = dt.fread(\"../input/riiid-test-answer-prediction/train.csv\",columns=set(['timestamp','user_id','content_id','task_container_id','answered_correctly','prior_question_elapsed_time','prior_question_had_explanation'])).to_pandas()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"w2v_model = gensim.models.Word2Vec.load(\"../input/w2v-model/w2v_model.model\")\nwith open(r\"../input/w2v-model/tokenizer.pkl\", \"rb\") as input_file:\n    tokenizer = pickle.load(input_file)\n\nwith open(r\"../input/w2v-model/embedding_matrix.pkl\", \"rb\") as input_file:\n    embedding_matrix = pickle.load(input_file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_csv = pd.read_csv(\"../input/riiid-test-answer-prediction/questions.csv\")\nexample_test_csv = pd.read_csv(\"../input/riiid-test-answer-prediction/example_test.csv\")\nlectures_csv = pd.read_csv(\"../input/riiid-test-answer-prediction/lectures.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_csv['tags'].fillna('200',inplace=True)\n\nquestions_csv['tags1']=questions_csv['tags'].apply(lambda x : int(x.split(' ')[0]))\n\n# from sklearn.preprocessing import MultiLabelBinarizer\n\n# mlb = MultiLabelBinarizer()\n# mlb.fit(questions_csv['tags1'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv['lag'] = train_csv.groupby('user_id')['answered_correctly'].shift()\ncum = train_csv.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])\ntrain_csv['user_correctness'] = cum['cumsum'] / cum['cumcount']\ntrain_csv.drop(columns=['lag'], inplace=True)\nuser_agg = train_csv.groupby('user_id')['answered_correctly'].agg(['sum', 'count'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv = train_csv.groupby('user_id').tail(105).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv = train_csv[train_csv['answered_correctly'] != -1].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv['answered_correctly_1'] = train_csv.groupby('user_id')['answered_correctly'].shift()\ntrain_csv['answered_correctly_1'] = train_csv.groupby(['user_id','task_container_id'],sort=False)['answered_correctly_1'].transform('first')\ngc.collect()\ntrain_csv.answered_correctly_1.fillna(0,inplace=True)\ntrain_csv.answered_correctly_1=train_csv.answered_correctly_1.astype('int')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv=train_csv.merge(questions_csv[['question_id','part','tags1']],how='left',left_on='content_id',right_on='question_id')\ntrain_csv['prior_question_had_explanation'].fillna(False, inplace=True)\ntrain_csv['prior_question_elapsed_time'].fillna(0,inplace=True)\n\ntrain_csv.head()\n\ntrain_csv['prior_question_elapsed_time']=train_csv['prior_question_elapsed_time']/100\ntrain_csv['prior_question_elapsed_time']=train_csv['prior_question_elapsed_time'].astype(int)\ntrain_csv['prior_question_elapsed_time'].values[train_csv['prior_question_elapsed_time'] > 300] = 300\n\ntrain_csv['prior_question_elapsed_time'].values[train_csv['prior_question_elapsed_time'] > 300] = 300\n\ntrain_csv.head()\n\ntrain_csv['answered_correctly'].replace(to_replace=0, value=2, inplace=True)\ntrain_csv['answered_correctly_1'].replace(to_replace=0, value=2, inplace=True)\n\n\ntrain_csv['prior_question_had_explanation'].replace(to_replace={False:2,True:1}, inplace=True)\n\n# label_enc_dict={}\n# for i in ['prior_question_had_explanation']:\n#     le = LabelEncoder()\n#     train_csv[i] = le.fit_transform(train_csv[i])\n#     label_enc_dict[i]=le.classes_\n    \n# scaler = StandardScaler()\n# train_csv['prior_question_elapsed_time']=scaler.fit_transform(train_csv['prior_question_elapsed_time'].values.reshape(-1,1))  \n\ntrain_csv.content_id=train_csv.content_id.astype('str')\ntrain_csv['user_correctness'].fillna(0,inplace=True)\n# current=train_csv.groupby('user_id')\n# train_csv['mask']=1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"max_length=70\na=train_csv[['user_id','content_id','answered_correctly','prior_question_elapsed_time','prior_question_had_explanation','part','user_correctness','tags1','answered_correctly_1']].groupby('user_id').agg({'content_id':lambda x: np.array(x[-max_length::]).tolist(),'answered_correctly_1':lambda x: x[-max_length::].tolist(),'prior_question_elapsed_time':lambda x: x[-max_length::].tolist(),'prior_question_had_explanation':lambda x: x[-max_length::].tolist(),'part':lambda x: x[-max_length::].tolist(),'user_correctness':lambda x: x[-max_length::].tolist(),'tags1':lambda x: x[-max_length::].tolist(),'answered_correctly':lambda x: x[-max_length::].tolist()})\na.columns=['history_content_id','history_answered_correctly','history_prior_question_elapsed_time','history_prior_question_had_explanation','history_part','history_user_correctness','history_tags','answered_correctly']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# a_c=pd.concat([a,c],axis=1)\n# del(a)\n# del(c)\ndel(train_csv)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a_b_both_train,a_b_both_test=train_test_split(a,test_size=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.sequence import pad_sequences\nfrom keras.preprocessing import sequence\nfrom keras.preprocessing.text import one_hot\nfrom keras.preprocessing.text import Tokenizer\n# from tensorflow.keras.preprocessing.sequence import pad_sequences","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features_content=['history_content_id']\nenc_cat_features=['history_part','history_tags']\ndec_cat_feature_explain=['history_prior_question_had_explanation']\ndec_cat_feature_el_time=['history_prior_question_elapsed_time']\ndec_cat_feature_hist_correct=['history_answered_correctly']\ncont_features=['history_user_correctness']\n# mask_feature=['history_mask']\ntarget=['answered_correctly']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.utils import to_categorical","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_training_test_sets(a,tokenizer,features_content,cont_features,enc_cat_features,dec_cat_feature_explain,dec_cat_feature_el_time,dec_cat_feature_hist_correct,target,lenght=max_length): \n    X=list()\n    \n    \n    for j in features_content:\n        X=[sequence.pad_sequences(tokenizer.texts_to_sequences(a[j].to_list()),maxlen=lenght,padding='pre')]\n    \n    for i in enc_cat_features:\n        p=[pad_sequences(a[i],maxlen=lenght,padding='pre')]\n        X.extend(p)\n        \n    for i in dec_cat_feature_explain:\n        p=[pad_sequences(a[i],maxlen=lenght,padding='pre')]\n        p[0][:,0]=3\n        \n        X.extend(p) \n        \n    for i in dec_cat_feature_el_time:\n        p=[pad_sequences(a[i],maxlen=lenght,padding='pre')]\n        p[0][:,0]=301\n        \n        X.extend(p) \n\n    for i in dec_cat_feature_hist_correct:\n        p=[pad_sequences(a[i],maxlen=lenght,padding='pre')]\n        p[0][:,0]=3\n        \n        X.extend(p) \n    \n        \n    for i in cont_features:\n        p=[pad_sequences(a[i],maxlen=lenght,padding='pre',dtype='float32')]\n        X.extend(p)\n        \n        \n#     for i in mask_feature:\n#         p=[pad_sequences(a[i],maxlen=lenght,padding='pre')]\n#         X.extend(p)    \n        \n#     for j in curr_cat_features_1:\n#         p=[sequence.pad_sequences(tokenizer.texts_to_sequences(a[j].to_list()))]\n#         X.extend(p)\n        \n        \n    \n        \n#     X_cat = [a.loc[:, curr_cat_features].values[:, k] for k in range(a.loc[:, curr_cat_features].values.shape[1])]\n# #     print(np.array(data.loc[:,items]).shape)\n#     X_con = [a.loc[:, curr_cont_features].values[:, k] for k in range(a.loc[:, curr_cont_features].values.shape[1])]\n    \n#     X.extend(X_cat)\n#     X.extend(X_con)\n    \n    for i in target:\n        Y=pad_sequences(a[i],maxlen=lenght,padding='pre')\n        Y_t=to_categorical(Y)\n            \n        \n    \n    return X, Y_t\n\n# X_train, Y_train = create_training_test_sets(a_c,tokenizer,features_content,cont_features,enc_cat_features,dec_cat_feature_explain,dec_cat_feature_el_time,dec_cat_feature_hist_correct,target,lenght=70)\nX_train, Y_train = create_training_test_sets(a_b_both_train,tokenizer,features_content,cont_features,enc_cat_features,dec_cat_feature_explain,dec_cat_feature_el_time,dec_cat_feature_hist_correct,target,lenght=max_length)\nX_test, Y_test = create_training_test_sets(a_b_both_test,tokenizer,features_content,cont_features,enc_cat_features,dec_cat_feature_explain,dec_cat_feature_el_time,dec_cat_feature_hist_correct,target,lenght=max_length)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(a,embedding_matrix,max_length):    \n    inputs = []\n    outputs = []\n    outputs_dense = []\n    categorical_output = []\n    \n    \n    inp_1 = layers.Input(shape=(max_length,))\n#     out_content = embedding_layer(inp_1)\n    inputs.append(inp_1)\n    \n#     print(out_content.shape)\n\n    inp_2 = layers.Input(shape=(max_length,))\n#     out_2=Reshape((max_length, 1), input_shape=(max_length,))(inp_2)\n    inputs.append(inp_2)\n\n    \n    #y = Model(inputs=inputB, outputs=y)\n    \n    inp_8 = layers.Input(shape=(max_length,))\n#     out_2=Reshape((max_length, 1), input_shape=(max_length,))(inp_2)\n    inputs.append(inp_8)\n    \n    inp_3 = layers.Input(shape=(max_length,))\n#     out_3=Reshape((max_length, 1), input_shape=(max_length,))(inp_3)\n    inputs.append(inp_3)\n    \n    inp_4= layers.Input(shape=(max_length,))\n#     out_had_explaination=Embedding(3,3,input_length=max_length,mask_zero=True)(inp_4)\n    inputs.append(inp_4)\n    \n    \n    inp_5= layers.Input(shape=(max_length,))\n#     out_part=Embedding(8,20,input_length=max_length,mask_zero=True)(inp_5)\n    inputs.append(inp_5)\n    \n    inp_6= layers.Input(shape=(max_length,))\n#     out_answered_correctly=Embedding(3,4,input_length=max_length)(inp_6)\n    out_6=Reshape((max_length, 1), input_shape=(max_length,))(inp_6) \n    inputs.append(inp_6)\n    \n#     inp_7= layers.Input(shape=(max_length,))\n#     out_answered_correctly=Embedding(3,4,input_length=max_length)(inp_6)\n#     inputs.append(inp_7)\n    \n    sample_transformer = Transformer(\n    num_layers=2, d_model=512, num_heads=4, dff=2048, \n    input_vocab_size=13524, target_vocab_size=5, \n    pe_input=10000, pe_target=6000,emb_matrix=embedding_matrix,inp2_vocab_size=9,inp3_vocab_size=202,dec_inp2_size=302,dec_inp3_size=5,rate=0.2)\n\n    \n# embedding_matrix, inp2_vocab_size,dec_inp2_size,dec_inp3_size,rate=0.1    \n#     temp_input = tf.random.uniform((64, 38), dtype=tf.int64, minval=0, maxval=200)\n#     temp_target = tf.random.uniform((64, 36), dtype=tf.int64, minval=0, maxval=200)\n\n# inp1,inp2, tar1,tar2,tar3\n    enc_mask = create_padding_mask(inp_1)\n    dec_mask = create_padding_mask(inp_1)\n    \n    look_mask=create_look_ahead_mask(max_length)\n\n    fn_out, _ = sample_transformer(inp_1,inp_2,inp_8,inp_5,inp_4,inp_3,out_6,training=True, \n                               enc_padding_mask=look_mask, \n                               look_ahead_mask=look_mask,\n                               dec_padding_mask=dec_mask)\n    \n    print(fn_out.shape)\n    \n    \n    \n    \n    model = Model(inputs=inputs, outputs=fn_out)\n    return model\n\nmodel = create_model(a_b_both_train,embedding_matrix,max_length=max_length)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n    def __init__(self, d_model, warmup_steps=3000):\n        super(CustomSchedule, self).__init__()\n\n        self.d_model = d_model\n        self.d_model = tf.cast(self.d_model, tf.float32)\n\n        self.warmup_steps = warmup_steps\n\n    def __call__(self, step):\n        arg1 = tf.math.rsqrt(step)\n        arg2 = step * (self.warmup_steps ** -1.5)\n\n        return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learning_rate = CustomSchedule(512)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(loss='CategoricalCrossentropy', optimizer=optimizers.Adam(learning_rate=learning_rate, beta_1=0.9, beta_2=0.98, epsilon=1e-9),metrics=['accuracy']) \nmodel.fit(X_train,Y_train,epochs=12,batch_size=256,validation_split=.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_predict=model.predict(X_test)\nY_correct_test=Y_test[:,:,1]\nY_correct_predict=y_predict[:,:,1]\n\nscores=Y_correct_predict[:,-1]\nactual=Y_correct_test[:,-1]\nfrom sklearn.metrics import roc_auc_score,accuracy_score\nroc_auc_score(actual,scores)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# lm=Y_correct_predict[a_b_both_test['history_content_id'].apply(lambda x: len(x))<5,:]\n# ls=Y_correct_test[a_b_both_test['history_content_id'].apply(lambda x: len(x))<5,:]\n# scores=lm[:,-1]\n# actual=ls[:,-1]\n# roc_auc_score(actual,scores)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# example_test = pd.read_csv(\"../input/riiid-test-answer-prediction/example_test.csv\")\n# example_test.head()\n# example_test_1=example_test[example_test['group_num']==0]\n# example_test_2=example_test[example_test['group_num']==1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x2=a.loc[115,'history_answered_correctly']\nprior_answer=a.loc[115,'answered_correctly'][-1]\nx2.append(prior_answer)\nx2=x2[-max_length::]\na.loc[115,'history_answered_correctly']=x2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a.loc[115,'history_answered_correctly']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a.loc[115,'answered_correctly'][-1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()\niter_test = env.iter_test()\nprior_test_df = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    if prior_test_df is not None:\n        prior_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])\n        prior_test_df=prior_test_df[prior_test_df['content_type_id'] == 0].reset_index(drop = True)\n        \n        \n        user_ids = prior_test_df['user_id'].values\n        content_ids = prior_test_df['content_id'].values\n        targets = prior_test_df[target].values\n        \n        for user_id, content_id, answered_correctly in zip(user_ids, content_ids, targets):\n            if user_id in user_agg.index:\n                user_agg.loc[user_id, 'sum'] += answered_correctly\n                user_agg.loc[user_id, 'count'] += 1\n            else:\n                user_agg.loc[user_id] = [answered_correctly[0], 1]\n            \n                            \n        prior_test_df['user_correctness'] = prior_test_df['user_id'].map(user_agg['sum'] / user_agg['count']).fillna(0.5)\n        \n        \n#         print(roc_auc_score(prior_test_df['answered_correctly'].values,scores))\n        prior_test_df['answered_correctly'].replace(to_replace=0, value=2, inplace=True)\n        \n        \n        \n        \n        \n        prior_group=prior_test_df.groupby('user_id',sort=False)['row_id'].count().reset_index()\n        \n#         d=0\n#         for user_id, group_num in zip(prior_group.user_id, prior_group.row_id):\n#             for j in range(group_num):\n#                 correct_answer=(prior_test_df.answered_correctly[d])\n#                 a.loc[user_id,'answered_correctly'][-(group_num-j)]=correct_answer\n# #                 print(d)\n# #                 print(group_num)\n# #                 print(group_num-j)\n#                 d=d+1\n\n        for i in range(prior_test_df.shape[0]):\n        \n            user_id=prior_test_df.user_id[i]\n#             if user_id not in(a.index):\n#                 a=a.append(pd.DataFrame({'history_content_id':[['0']],'history_answered_correctly':[[0]],'history_prior_question_elapsed_time':[[0]],'history_prior_question_had_explanation':[[0]],'history_part':[[0]],'history_history_user_correctness':[[0]]},index=[user_id]))\n\n            \n#             x7.append(prior_test_df.answered_correctly[i])\n#         x7.append(prior_answer)\n#             x7=x7[-max_length::]\n#             a.loc[user_id,'answered_correctly']=x7\n            \n            x7=a.loc[user_id,'answered_correctly']\n            \n#             prior_answer=a.loc[user_id,'answered_correctly'][-1]\n            correct_answer=(prior_test_df.answered_correctly[i])\n            x7.append(correct_answer)\n            \n            x7=x7[-70::]\n            a.loc[user_id,'answered_correctly']=x7\n\n            \n\n            \n#         a_df=a_df.append(pd.DataFrame({'content_id':[x1],'answered_correctly':[x5],'prior_question_elapsed_time':[x2],'prior_question_had_explanation':[x3],'part':[x4]}))\n \n    \n        \n    prior_test_df = test_df.copy()\n#     print(test_df)\n    \n    \n    \n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop = True)\n    \n    test_df = pd.merge(test_df, questions_csv, left_on = 'content_id', right_on = 'question_id', how = 'left')\n    \n    test_df['prior_question_had_explanation'].fillna(False, inplace=True)\n    test_df['prior_question_elapsed_time'].fillna(0,inplace=True)\n\n    test_df['prior_question_elapsed_time']=test_df['prior_question_elapsed_time']/100\n    test_df['prior_question_elapsed_time']=test_df['prior_question_elapsed_time'].astype(int)\n    test_df['prior_question_elapsed_time'].values[test_df['prior_question_elapsed_time'] > 300] = 300\n#     test_df['answered_correctly'].replace(to_replace=0, value=2, inplace=True)\n    test_df['prior_question_had_explanation'].replace(to_replace={False:2,True:1}, inplace=True)\n    \n    \n#     for i in ['prior_question_had_explanation']:\n        \n#         encoder=LabelEncoder()\n#         encoder.classes_=label_enc_dict[i]\n#         test_df[i]=encoder.transform(test_df[i]) \n#     test_df['prior_question_elapsed_time']=scaler.transform(test_df['prior_question_elapsed_time'].values.reshape(-1,1))\n\n    \n    test_df['user_correctness'] = test_df['user_id'].map(user_agg['sum'] / user_agg['count']).fillna(0.6)\n        \n#     test_df['content_id_correctness'] = test_df['content_id'].map(content_agg['sum'] / content_agg['count']).fillna(0.7)\n    \n    test_df.content_id=test_df.content_id.astype('str')\n    \n    test_df['answered_correctly']=0\n    \n    a_df=pd.DataFrame()    \n    for i in range(test_df.shape[0]):\n        \n        user_id=test_df.user_id[i]\n        if user_id not in(a.index):\n            a=a.append(pd.DataFrame({'history_content_id':[['0']],'history_answered_correctly':[[0]],'history_prior_question_elapsed_time':[[0]],'history_prior_question_had_explanation':[[0]],'history_part':[[0]],'history_user_correctness':[[0]],'answered_correctly':[[0]],'history_tags':[[0]]},index=[user_id]))\n        \n        \n        \n        \n        \n        x1=a.loc[user_id,'history_content_id']\n        x1.append(test_df.content_id[i])\n        x1=x1[-max_length::]\n        a.loc[user_id,'history_content_id']=x1\n\n        x2=a.loc[user_id,'history_answered_correctly']\n        prior_answer=a.loc[user_id,'answered_correctly'][-1]\n        x2.append(prior_answer)\n        x2=x2[-max_length::]\n        a.loc[user_id,'history_answered_correctly']=x2\n\n\n\n        x3=a.loc[user_id,'history_prior_question_elapsed_time']\n        x3.append(test_df.prior_question_elapsed_time[i])\n        x3=x3[-max_length::]\n        a.loc[user_id,'history_prior_question_elapsed_time']=x3\n\n        x4=a.loc[user_id,'history_prior_question_had_explanation']\n        x4.append(test_df.prior_question_had_explanation[i])\n        x4=x4[-max_length::]\n        a.loc[user_id,'history_prior_question_had_explanation']=x4\n\n        x5=a.loc[user_id,'history_part']\n        x5.append(test_df.part[i])\n        x5[-max_length::]\n        a.loc[user_id,'history_part']=x5\n\n        x6=a.loc[user_id,'history_user_correctness']\n        x6.append(test_df.user_correctness[i])\n        x6=x6[-max_length::]\n        a.loc[user_id,'history_user_correctness']=x6\n        \n        \n        x7=a.loc[user_id,'answered_correctly']\n#         prior_answer=a.loc[user_id,'answered_correctly'][-1]\n#         x7.append(test_df.answered_correctly[i])\n#         x7.append(prior_answer)\n#         x7=x7[-max_length::]\n#         a.loc[user_id,'answered_correctly']=x7\n\n        x8=a.loc[user_id,'history_tags']\n        x8.append(test_df.tags1[i])\n        x8=x8[-max_length::]\n        a.loc[user_id,'history_tags']=x8\n        \n        \n        \n        \n        \n        \n        \n\n        \n        \n        \n        a_df=a_df.append(pd.DataFrame({'history_content_id':[x1],'history_answered_correctly':[x2],'history_prior_question_elapsed_time':[x3],'history_prior_question_had_explanation':[x4],'history_part':[x5],'history_user_correctness':[x6],'answered_correctly':[x7],'history_tags':[x8]},index=[i]))\n\n        \n        \n        \n                       \n#     a_df=a[a.index.isin(test_df.user_id.values)]\n    \n    \n    #print(a_df.shape)\n#     a_final=pd.concat([a_df,test_df],axis=1)\n    \n    X_train, Y_train = create_training_test_sets(a_df,tokenizer,features_content,cont_features,enc_cat_features,dec_cat_feature_explain,dec_cat_feature_el_time,dec_cat_feature_hist_correct,target,lenght=max_length)\n    \n#     test_df.drop('answered_correctly',axis=1,inplace=True)\n\n    Y=model.predict(X_train)\n    \n    Y_correct_predict=Y[:,:,1]\n    scores=Y_correct_predict[:,-1]\n    \n    \n#     answer_df=pd.DataFrame({'user_id':a_df.index,'answered_correctly':Y[:,39,0]})\n    \n#     test_df=test_df.merge(answer_df,on='user_id',how='left')\n\n    test_df['answered_correctly']=scores\n\n    #print(Y)\n       \n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}