{"cells":[{"metadata":{},"cell_type":"markdown","source":"## References\n1. Loop Feature Engineering: https://www.kaggle.com/its7171/lgbm-with-loop-feature-engineering\n2. Cross Validation: https://www.kaggle.com/its7171/cv-strategy\n3. Deep Knowledge Tracing (DKT): https://stanford.edu/~cpiech/bio/papers/deepKnowledgeTracing.pdf\n4. Self-Attentive model for Knowledge Tracing (SAKT): https://arxiv.org/pdf/1907.06837.pdf\n5. SAINT: https://arxiv.org/pdf/2010.12042.pdf"},{"metadata":{"execution":{"iopub.execute_input":"2020-10-12T18:21:01.107461Z","iopub.status.busy":"2020-10-12T18:21:01.1067Z","iopub.status.idle":"2020-10-12T18:21:02.19541Z","shell.execute_reply":"2020-10-12T18:21:02.194464Z"},"lines_to_next_cell":2,"papermill":{"duration":1.131405,"end_time":"2020-10-12T18:21:02.195556","exception":false,"start_time":"2020-10-12T18:21:01.064151","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import pickle\nimport pandas as pd\nimport numpy as np\nimport gc\nfrom sklearn.metrics import roc_auc_score\nfrom collections import defaultdict, deque\nfrom tqdm.notebook import tqdm\nimport lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_pickle = '../input/riiid-cross-validation-files/cv1_train.pickle'\nvalid_pickle = '../input/riiid-cross-validation-files/cv1_valid.pickle'\nquestion_file = '../input/riiid-test-answer-prediction/questions.csv'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Feature Engineering"},{"metadata":{},"cell_type":"markdown","source":"For LGBM, I use 4 additional features: \n1. the number of questions that a student solved\n2. the number of questions that a student answered correctly\n3. the correction rate of a student = (2)/(1)\n4. time difference between two interactive sessions: timestamp(t) - timestamp(t-1). When t = 0, I set the difference equals zero"},{"metadata":{"trusted":true},"cell_type":"code","source":"last_time_u_dict = defaultdict(int)\nanswered_correctly_sum_u_dict = defaultdict(int)\ncount_u_dict = defaultdict(int)\n\npast_question = defaultdict(list)\npast_time_diff = defaultdict(list)\npast_prior_elaps = defaultdict(list)\npast_answer = defaultdict(list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# funcs for user stats with loop\ndef add_user_feats(df,last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict):\n    acsu = np.zeros(len(df), dtype=np.int32)\n    cu = np.zeros(len(df), dtype=np.int32)\n    td = np.zeros(len(df), dtype=np.int64)\n\n    for cnt,row in enumerate(tqdm(df[['user_id','answered_correctly','timestamp']].values)):\n        acsu[cnt] = answered_correctly_sum_u_dict[row[0]]\n        cu[cnt] = count_u_dict[row[0]]\n        td[cnt] = row[2] - last_time_u_dict[row[0]] #time difference can be 0 when there are more than 1 questions in an interactive session\n        \n        if row[2] == last_time_u_dict[row[0]]: #This fixes the problem. \n            td[cnt] = td[cnt-1]\n            \n        answered_correctly_sum_u_dict[row[0]] += row[1]\n        count_u_dict[row[0]] += 1\n        last_time_u_dict[row[0]] = row[2]\n                  \n    user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu, 'time_diff':td})\n    user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']\n    df = pd.concat([df, user_feats_df], axis=1)\n    return df\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_user_feats2(df, past_question, past_answer, past_prior_elaps, past_time_diff):\n    \n    for _, row in enumerate(tqdm(df[['user_id','answered_correctly','content_id','prior_question_elapsed_time','time_diff']].values)):\n        \n        if row[0] not in past_question:\n            new_list = deque([0]*30, maxlen = 30)\n            new_list.append(row[2])\n            past_question[row[0]] = new_list\n        else:\n            past_question[row[0]].append(row[2])\n            \n\n        if row[0] not in past_answer:\n            new_list = deque([0]*30, maxlen = 30)\n            new_list.append(row[1])\n            past_answer[row[0]] = new_list\n        else:\n            past_answer[row[0]].append(row[1])     \n\n            \n        if row[0] not in past_prior_elaps:\n            new_list = deque([0]*30, maxlen = 30)\n            new_list.append(row[3])\n            past_prior_elaps[row[0]] = new_list\n        else:\n            past_prior_elaps[row[0]].append(row[3])   \n            \n            \n        if row[0] not in past_time_diff:\n            new_list = deque([0]*30, maxlen = 30)\n            new_list.append(row[4])\n            past_time_diff[row[0]] = new_list\n        else:\n            past_time_diff[row[0]].append(row[4])   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_user_feats_without_update(df ,last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict):\n\n    acsu = np.zeros(len(df), dtype=np.int32)\n    cu = np.zeros(len(df), dtype=np.int32)\n    td = np.zeros(len(df), dtype=np.int64)\n    \n    for cnt, row in enumerate(df[['user_id','part','timestamp']].values):\n        td[cnt] = row[2] - last_time_u_dict[row[0]]\n        acsu[cnt] = answered_correctly_sum_u_dict[row[0]]\n        cu[cnt] = count_u_dict[row[0]]\n        \n                \n    user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu, 'time_diff':td})\n    user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']\n    df = pd.concat([df, user_feats_df], axis=1)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_user_feats_for_nn_without_update(df, past_question, past_answer, past_prior_elaps, past_time_diff):\n    current_question = []\n    past_question_answer = []\n    past_answer_correctly = []\n    past_time = []\n    past_prior = []\n    #past_other_feats = []\n    \n    \n    for cnt,row in enumerate(tqdm(df[['user_id','content_id','prior_question_elapsed_time','time_diff']].values)):\n        \n        if row[0] not in past_answer:\n            temp_answer = [0]*30\n            temp_past_answer = [0]*30\n        else:\n            temp_answer = past_answer[row[0]].copy()\n            temp_past_answer = past_answer[row[0]].copy()\n        \n        \n        if row[0] not in past_question:\n            temp_question = [0]*30\n        else:\n            temp_question = past_question[row[0]].copy()\n        \n        \n        temp_past_answer= [x+1 if y > 0 else 0 for x , y in zip(temp_past_answer, temp_question)]\n        past_answer_correctly.append(temp_past_answer)\n        \n        temp_past_question_answer = [x+y*13523 for x,y in zip(temp_question, temp_answer)]\n        past_question_answer.append(temp_past_question_answer)\n        \n        temp_question.append(row[1]+1)\n        current_question.append([temp_question[i] for i in range(30)])\n        \n        \n        if row[0] not in past_prior_elaps:\n            temp_elaps = [0]*29 + [row[2]/3e5]\n        else:\n            temp_elaps = past_prior_elaps[row[0]].copy()\n            temp_elaps.append(row[2]/3e5)\n        past_prior.append(temp_elaps)\n        \n        \n        if row[0] not in past_time_diff:\n            temp_time_diff = [0]*29 + [row[3]/1e6]\n        else:\n            temp_time_diff = past_time_diff[row[0]].copy()\n            temp_time_diff.append(row[3]/1e6)\n            \n        past_time.append(temp_time_diff)\n    \n    current_question = np.array(current_question)\n    past_question_answer = np.array(past_question_answer)    \n    past_other_feats = np.dstack((past_prior,past_time))\n    past_answer_correctly = np.array(past_answer_correctly)\n    \n    return current_question, past_question_answer, past_other_feats, past_answer_correctly","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def update_user_feats(df, last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict):\n    for row in df[['user_id', 'answered_correctly', 'content_type_id', 'timestamp', 'content_id','prior_question_elapsed_time','time_diff']].values:\n        if row[2] == 0:    \n            \n            if row[0] not in past_question:\n                new_list = deque([0]*30, maxlen = 30)\n                new_list.append(row[4]+1)\n                past_question[row[0]] = new_list\n            else:\n                past_question[row[0]].append(row[4]+1)\n                \n            \n            if row[0] not in past_answer:\n                new_list = deque([0]*30, maxlen = 30)\n                new_list.append(row[1])\n                past_answer[row[0]] = new_list\n            else:\n                past_answer[row[0]].append(row[1])     \n\n            \n            if row[0] not in past_prior_elaps:\n                new_list = deque([0]*30, maxlen = 30)\n                new_list.append(row[5]/3e5)\n                past_prior_elaps[row[0]] = new_list\n            else:\n                past_prior_elaps[row[0]].append(row[5]/3e5) \n                \n                \n            if row[0] not in past_time_diff:\n                new_list = deque([0]*30, maxlen = 30)\n                if row[6] >= 1e6:\n                    new_list.append(0)\n                else:\n                    new_list.append(row[6]/1e6)\n                past_time_diff[row[0]] = new_list\n            else:\n                if row[6] >= 1e6:\n                    past_time_diff[row[0]].append(0)   \n                else:\n                    past_time_diff[row[0]].append(row[6]/1e6)               \n            \n                \n            last_time_u_dict[row[0]] = row[3]\n            answered_correctly_sum_u_dict[row[0]] += row[1]\n            count_u_dict[row[0]] += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# read data\nfeld_needed = ['user_id','content_id','answered_correctly','timestamp','prior_question_elapsed_time','prior_question_had_explanation']\ntrain = pd.read_pickle(train_pickle)[feld_needed]\nvalid = pd.read_pickle(valid_pickle)[feld_needed]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.loc[train.answered_correctly != -1].reset_index(drop=True)\nvalid = valid.loc[valid.answered_correctly != -1].reset_index(drop=True)\n_=gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prior_question_elapsed_time_mean = train.prior_question_elapsed_time.dropna().values.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"content_df = train[['content_id','answered_correctly']].groupby(['content_id']).agg(['mean','std']).reset_index()\ncontent_df.columns = ['content_id', 'answered_correctly_avg_c','answered_correctly_std_c']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train1 = train.groupby('user_id').tail(31).reset_index(drop=True)\nvalid1 = valid.groupby('user_id').tail(31).reset_index(drop=True)\n\ntrain = train[-5000000:].reset_index(drop=True) #this notebook is just an illustration, thus, I will use 5M trainig.\n\n_=gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# extract features for training purpose\n\ntrain = add_user_feats(train, last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict)\nvalid = add_user_feats(valid, last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df = pd.read_csv(question_file)\nquestions_df.tags.fillna('-1-1', inplace = True)\nquestions_df['tag'] = pd.factorize(questions_df.tags)[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# I repeat the step so that I extract features from all students for prediction purpose.\n\nlast_time_u_dict = defaultdict(int)\nanswered_correctly_sum_u_dict = defaultdict(int)\ncount_u_dict = defaultdict(int)\n\n\ntrain1 = add_user_feats(train1, last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict)\nvalid1 = add_user_feats(valid1, last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict)\n\n\ntrain1 = pd.merge(train1, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')\nvalid1 = pd.merge(valid1, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n\ntrain1.time_diff[train1.time_diff >= 1e6] = 1e6\nvalid1.time_diff[valid1.time_diff >= 1e6] = 1e6\n\n\ntrain1.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean, inplace = True)\nvalid1.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean, inplace = True)\n\n\nadd_user_feats2(train1, past_question, past_answer, past_prior_elaps, past_time_diff)\nadd_user_feats2(valid1, past_question, past_answer, past_prior_elaps, past_time_diff)\n\ndel(train1)\ndel(valid1)\n_=gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.merge(train, content_df, on=['content_id'], how=\"left\")\nvalid = pd.merge(valid, content_df, on=['content_id'], how=\"left\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.merge(train, questions_df[['question_id', 'part','tag']], left_on = 'content_id', right_on = 'question_id', how = 'left')\nvalid = pd.merge(valid, questions_df[['question_id', 'part','tag']], left_on = 'content_id', right_on = 'question_id', how = 'left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['prior_question_elapsed_time'] = train.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)\nvalid['prior_question_elapsed_time'] = valid.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)\n\n# changing dtype to avoid lightgbm error\ntrain['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False).astype('int8')\nvalid['prior_question_had_explanation'] = valid.prior_question_had_explanation.fillna(False).astype('int8')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## LGBM"},{"metadata":{"trusted":true},"cell_type":"code","source":"TARGET = 'answered_correctly'\nFEATS = ['answered_correctly_avg_u', 'answered_correctly_avg_c','answered_correctly_std_c','answered_correctly_sum_u','count_u', 'part','tag', 'prior_question_elapsed_time','time_diff', 'prior_question_had_explanation']\ndro_cols = list(set(train.columns) - set(FEATS))\ny_tr = train[TARGET]\ny_va = valid[TARGET]\n#train.drop(dro_cols, axis=1, inplace=True)\n#valid.drop(dro_cols, axis=1, inplace=True)\n_=gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_train = lgb.Dataset(train[FEATS], y_tr, categorical_feature = ['part', 'prior_question_had_explanation', 'tag'],free_raw_data=False)\nlgb_valid = lgb.Dataset(valid[FEATS], y_va, categorical_feature = ['part', 'prior_question_had_explanation', 'tag'], reference=lgb_train,free_raw_data=False)\n#del train, y_tr\n_=gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Hyperparameter Tuning"},{"metadata":{"trusted":true},"cell_type":"code","source":"import optuna","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def objective(trial):    \n    params = {\n            'num_leaves': trial.suggest_int('num_leaves', 32, 512),\n            'max_bin': trial.suggest_int('max_bin', 700, 900),\n            'boosting' : 'gbdt',\n            'objective': 'binary',\n            'metric': 'auc',\n            'max_depth': trial.suggest_int('max_depth', 4, 16),\n            'min_child_weight': trial.suggest_int('min_child_weight', 1, 16),\n            'feature_fraction': trial.suggest_uniform('feature_fraction', 0.4, 1.0),\n            'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),\n            'bagging_freq': trial.suggest_int('bagging_freq', 1, 8),\n            'min_child_samples': trial.suggest_int('min_child_samples', 4, 80),\n            'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 1.0),\n            'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 1.0),\n            'early_stopping_rounds': 5\n            }\n    model = lgb.train(params, lgb_train, valid_sets=[lgb_train, lgb_valid], verbose_eval=1000, num_boost_round=10000)\n    val_pred = model.predict(valid[FEATS])\n    score = roc_auc_score(y_va, val_pred)\n    print(f\"AUC = {score}\")\n    return score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Bayesian optimization\nstudy = optuna.create_study(direction='maximize')\nstudy.optimize(objective, n_trials=1) #you can increase trial numbers to get better parameters","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Number of finished trials: {}'.format(len(study.trials)))\n\nprint('Best trial:')\ntrial = study.best_trial\n\nprint('  Value: {}'.format(trial.value))\n\nprint('  Params: ')\nfor key, value in trial.params.items():\n    print('    {}: {}'.format(key, value))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"parameters = trial.params\nparameters['objective'] = 'binary'\nparameters['metric'] = 'auc'\nparameters['early_stopping_rounds'] = 5\nparameters['boosting'] = 'gbdt'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgbm_model = lgb.train(\n                    parameters, \n                    lgb_train,\n                    valid_sets=[lgb_train, lgb_valid],\n                    verbose_eval=100,\n                    num_boost_round=10000,\n                )\n\nprint('auc:', roc_auc_score(y_va, lgbm_model.predict(valid[FEATS])))\n_ = lgb.plot_importance(lgbm_model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nlgb.plot_importance(lgbm_model, importance_type = 'gain')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# DEEP LEARNING MODELs"},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.layers import Dense, Dropout, LSTM, Masking, Embedding, Concatenate, Input\nfrom tensorflow.keras.layers import concatenate\nfrom tensorflow.keras.metrics import AUC\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.layers import Lambda\nfrom tensorflow.keras.layers import multiply, Reshape\nfrom tensorflow.keras.utils import Sequence","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"###### Feature Engineering for Deep learning Models:\n\nI need to modify some features so that nueral network performs as I intend. So I rescale some variables:\n1. If time difference is larger than 10^6 = around 16 minutes, then I set time difference equals 10^6.\n2. I rescale time difference and prior question elapsed time varialbes by 1e6 and 3e5, respectively so that they take value between 0 and 1. (NN performs funny when the values are too large)"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Feature Engineering for Deep learning Models:\ntrain.time_diff[train.time_diff >= 1e6] = 1e6\nvalid.time_diff[valid.time_diff >= 1e6] = 1e6\n\n\ntrain.prior_question_elapsed_time = train.prior_question_elapsed_time/3e5\ntrain.time_diff = train.time_diff/1e6\n\n\nvalid.prior_question_elapsed_time = valid.prior_question_elapsed_time/3e5 \nvalid.time_diff = valid.time_diff/1e6","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now, I am going to reshape the dataframe into 3d matrix: sample_size,window_size,features\n1. I will throw away new students (who solved less than 30 questions) for the fast computation purpose only.\n2. I choose window size = 30. In SAINT paper, they choose 100. I believe this is one of hyperparameter that you need to tune yourself.\n3. I pad on the left.\n3. I add +1 on tag, question_id, lagged_answered_correctly (decoder input for SAINT) as I will use zeros to mask\n\nThere are 5 features to work on for Deep learning architectures:\n1. answered_correctly (output)\n2. lagged_answered_correctly (decoder input for SAINT)\n3. current_question_id (encoder input for SAINT, query for SAKT, input for DKT)\n4. prior_time_elapsed (decoder input for SAINT, SAKT, and DKT)\n5. time difference (decoder input for SAINT, SAKT, and DKT)"},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_user_count = train.user_id.value_counts()\n#train_del_user = train_user_count[train_user_count<30]\n#train = train[~train.user_id.isin(train_del_user.index)]\n\n\n#valid_user_count = valid.user_id.value_counts()\n#valid_del_user = valid_user_count[valid_user_count<30]\n#valid = valid[~valid.user_id.isin(valid_del_user.index)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.preprocessing.sequence import pad_sequences\nmax_len = 31","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_group = train.groupby('user_id')\ntrain_y = [frame['answered_correctly'].to_numpy()[:, None].tolist()[i:i+max_len] for _, frame in train_group\n           for i in range(0, len(frame['answered_correctly'].to_numpy()[:, None]), max_len)]\ntrain_y = np.reshape(pad_sequences(train_y, padding=\"pre\"),(-1,max_len))\n\n\ntrain_past_answer = [(frame['answered_correctly']+1).to_numpy()[:, None].tolist()[i:i+max_len] for _, frame in train_group\n           for i in range(0, len(frame['answered_correctly'].to_numpy()[:, None]), max_len) ]\ntrain_past_answer = np.reshape(pad_sequences(train_past_answer, padding=\"pre\"),(-1,max_len))\n\n\ntrain_current_question = [(frame['content_id']+1).to_numpy()[:, None].tolist()[i:i+max_len] for _, frame in train_group\n                         for i in range(0, len(frame['content_id'].to_numpy()[:,None]), max_len )]\ntrain_current_question = np.reshape(pad_sequences(train_current_question, padding=\"pre\"),(-1,max_len))\n\n\ntrain_prior_elaps_time = [frame['prior_question_elapsed_time'].to_numpy()[:, None].tolist()[i:i+max_len] for _, frame in train_group\n                         for i in range(0, len(frame['prior_question_elapsed_time'].to_numpy()[:,None]), max_len )]\ntrain_prior_elaps_time = np.reshape(pad_sequences(train_prior_elaps_time, padding=\"pre\", dtype = 'float32'),(-1,max_len))\n\n\ntrain_time_diff = [frame['time_diff'].to_numpy()[:, None].tolist()[i:i+max_len] for _, frame in train_group\n                         for i in range(0, len(frame['time_diff'].to_numpy()[:,None]), max_len )]\ntrain_time_diff = np.reshape(pad_sequences(train_time_diff, padding=\"pre\", dtype = 'float32'),(-1,max_len))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_answered_correcly = train_y[:,1:]\ntrain_past_answered_correctly = train_past_answer[:,:-1]\n\n\n# Following line will create tuple (question_id, answered_correctly)\ntrain_past_question_answer = train_current_question[:,:-1] + train_y[:,:-1]*(train.content_id.max()+1)\n\n\ntrain_current_question = train_current_question[:,1:]\n\n\ntrain_prior_elaps_time = train_prior_elaps_time[:,1:]\ntrain_time_diff = train_time_diff[:,1:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_group = valid.groupby('user_id')\nvalid_y = [frame['answered_correctly'].to_numpy()[:, None].tolist()[i:i+max_len] for _, frame in valid_group\n           for i in range(0, len(frame['answered_correctly'].to_numpy()[:, None]), max_len) ]\nvalid_y = np.reshape(pad_sequences(valid_y, padding=\"pre\"),(-1,max_len))\n\n\nvalid_past_answer = [(frame['answered_correctly']+1).to_numpy()[:, None].tolist()[i:i+max_len] for _, frame in valid_group\n           for i in range(0, len(frame['answered_correctly'].to_numpy()[:, None]), max_len) ]\nvalid_past_answer = np.reshape(pad_sequences(valid_past_answer, padding=\"pre\"),(-1,max_len))\n\n\nvalid_current_question = [(frame['content_id']+1).to_numpy()[:, None].tolist()[i:i+max_len] for _, frame in valid_group\n                         for i in range(0, len(frame['content_id'].to_numpy()[:,None]), max_len )]\nvalid_current_question = np.reshape(pad_sequences(valid_current_question, padding=\"pre\"),(-1,max_len))\n\n\nvalid_prior_elaps_time = [frame['prior_question_elapsed_time'].to_numpy()[:, None].tolist()[i:i+max_len] for _, frame in valid_group\n                         for i in range(0, len(frame['prior_question_elapsed_time'].to_numpy()[:,None]), max_len )]\nvalid_prior_elaps_time = np.reshape(pad_sequences(valid_prior_elaps_time, padding=\"pre\", dtype = 'float32'),(-1,max_len,1))\n\n\nvalid_time_diff = [frame['time_diff'].to_numpy()[:, None].tolist()[i:i+max_len] for _, frame in valid_group\n                         for i in range(0, len(frame['time_diff'].to_numpy()[:,None]), max_len )]\nvalid_time_diff = np.reshape(pad_sequences(valid_time_diff, padding=\"pre\", dtype = 'float32'),(-1,max_len,1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_answered_correcly = valid_y[:,1:]\nvalid_past_answered_correctly = valid_past_answer[:,:-1]\nvalid_past_question_answer = valid_current_question[:,:-1] + valid_y[:,:-1]*(valid.content_id.max()+1)\n\n\nvalid_current_question = valid_current_question[:,1:]\n\n\nvalid_prior_elaps_time = valid_prior_elaps_time[:,1:]\nvalid_time_diff = valid_time_diff[:,1:]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# SAKT"},{"metadata":{},"cell_type":"markdown","source":"I motify the code from https://www.tensorflow.org/tutorials/text/transformer. \nYou will have to look closely at encoder and encoder layer which I modified."},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_angles(pos, i, d_model):\n    angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))\n    return pos * angle_rates\n\n\ndef positional_encoding(position, d_model):\n    angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n                          np.arange(d_model)[np.newaxis, :],\n                          d_model)\n\n  # apply sin to even indices in the array; 2i\n    angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n\n  # apply cos to odd indices in the array; 2i+1\n    angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n\n    pos_encoding = angle_rads[np.newaxis, ...]\n\n    return tf.cast(pos_encoding, dtype=tf.float32)\n\n\ndef create_padding_output_mask(seq):\n    seq = 1 - tf.cast(tf.math.equal(seq, 0), tf.float32)\n\n  # add extra dimensions to add the padding\n  # to the attention logits.\n    return seq[:, :, tf.newaxis]  # (batch_size, 1, 1, seq_len)\n\n\n\ndef create_padding_mask(seq):\n    seq = tf.cast(tf.math.equal(seq, 0), tf.float32)\n\n  # add extra dimensions to add the padding\n  # to the attention logits.\n    return seq[:, tf.newaxis, tf.newaxis, :]  # (batch_size, 1, 1, seq_len)\n\n\n\n\ndef create_look_ahead_mask(size):\n    mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n    return mask  # (seq_len, seq_len)\n\n\n\n\ndef scaled_dot_product_attention(q, k, v, mask):\n    \"\"\"Calculate the attention weights.\n    q, k, v must have matching leading dimensions.\n    k, v must have matching penultimate dimension, i.e.: seq_len_k = seq_len_v.\n    The mask has different shapes depending on its type(padding or look ahead) \n    but it must be broadcastable for addition.\n\n    Args:\n      q: query shape == (..., seq_len_q, depth)\n      k: key shape == (..., seq_len_k, depth)\n      v: value shape == (..., seq_len_v, depth_v)\n      mask: Float tensor with shape broadcastable \n            to (..., seq_len_q, seq_len_k). Defaults to None.\n\n    Returns:\n      output, attention_weights\n    \"\"\"\n\n    matmul_qk = tf.matmul(q, k, transpose_b=True)  # (..., seq_len_q, seq_len_k)\n\n    # scale matmul_qk\n    dk = tf.cast(tf.shape(k)[-1], tf.float32)\n    scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n\n    # add the mask to the scaled tensor.\n    if mask is not None:\n        scaled_attention_logits += (mask * -1e9)  \n\n    # softmax is normalized on the last axis (seq_len_k) so that the scores\n    # add up to 1.\n    attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)  # (..., seq_len_q, seq_len_k)\n\n    output = tf.matmul(attention_weights, v)  # (..., seq_len_q, depth_v)\n\n    return output, attention_weights\n\n\n\n\nclass MultiHeadAttention(tf.keras.layers.Layer):\n    def __init__(self, d_model, num_heads):\n        super(MultiHeadAttention, self).__init__()\n        self.num_heads = num_heads\n        self.d_model = d_model\n\n        assert d_model % self.num_heads == 0\n\n        self.depth = d_model // self.num_heads\n\n        self.wq = tf.keras.layers.Dense(d_model)\n        self.wk = tf.keras.layers.Dense(d_model)\n        self.wv = tf.keras.layers.Dense(d_model)\n\n        self.dense = tf.keras.layers.Dense(d_model)\n\n    def split_heads(self, x, batch_size):\n        \"\"\"Split the last dimension into (num_heads, depth).\n        Transpose the result such that the shape is (batch_size, num_heads, seq_len, depth)\n        \"\"\"\n        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n        return tf.transpose(x, perm=[0, 2, 1, 3])\n\n    def call(self, v, k, q, mask):\n        batch_size = tf.shape(q)[0]\n\n        q = self.wq(q)  # (batch_size, seq_len, d_model)\n        k = self.wk(k)  # (batch_size, seq_len, d_model)\n        v = self.wv(v)  # (batch_size, seq_len, d_model)\n\n        q = self.split_heads(q, batch_size)  # (batch_size, num_heads, seq_len_q, depth)\n        k = self.split_heads(k, batch_size)  # (batch_size, num_heads, seq_len_k, depth)\n        v = self.split_heads(v, batch_size)  # (batch_size, num_heads, seq_len_v, depth)\n\n        # scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)\n        # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)\n        scaled_attention, attention_weights = scaled_dot_product_attention(\n            q, k, v, mask)\n\n        scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])  # (batch_size, seq_len_q, num_heads, depth)\n\n        concat_attention = tf.reshape(scaled_attention, \n                                       (batch_size, -1, self.d_model))  # (batch_size, seq_len_q, d_model)\n\n        output = self.dense(concat_attention)  # (batch_size, seq_len_q, d_model)\n\n        return output, attention_weights\n\n\n\n\n\ndef point_wise_feed_forward_network(d_model, dff):\n    return tf.keras.Sequential([\n        tf.keras.layers.Dense(dff, activation='relu'),  # (batch_size, seq_len, dff)\n        tf.keras.layers.Dense(d_model)  # (batch_size, seq_len, d_model)\n    ])\n\n\n    \n    \n    \nclass EncoderLayer(tf.keras.layers.Layer):\n    def __init__(self, d_model, num_heads, dff, rate=0.2):\n        super(EncoderLayer, self).__init__()\n\n        self.mha = MultiHeadAttention(d_model, num_heads)\n        self.ffn = point_wise_feed_forward_network(d_model, dff)\n\n        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n\n        self.dropout1 = tf.keras.layers.Dropout(rate)\n        self.dropout2 = tf.keras.layers.Dropout(rate)\n\n    def call(self, x, e, training, mask):\n\n        attn_output, _ = self.mha(x, x, e, mask)  # (batch_size, input_seq_len, d_model)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(e + attn_output)  # (batch_size, input_seq_len, d_model)\n\n        ffn_output = self.ffn(out1)  # (batch_size, input_seq_len, d_model)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        out2 = self.layernorm2(out1 + ffn_output)  # (batch_size, input_seq_len, d_model)\n\n        return out2\n\n\n\nclass Encoder(tf.keras.layers.Layer):\n    def __init__(self, num_layers, d_model, num_heads, dff,\n                   maximum_position_encoding, rate=0.2):\n        super(Encoder, self).__init__()\n\n        self.d_model = d_model\n        self.num_layers = num_layers\n\n        #self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)\n        self.pos_encoding = positional_encoding(maximum_position_encoding, \n                                                self.d_model)\n\n\n        self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate) \n                           for _ in range(num_layers)]\n\n        self.dropout = tf.keras.layers.Dropout(rate)\n\n    def call(self, x, e, training, mask):\n\n        seq_len = tf.shape(x)[1]\n\n        # adding embedding and position encoding.\n        #x = self.embedding(x)  # (batch_size, input_seq_len, d_model)\n        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n        x += self.pos_encoding[:, :seq_len, :]\n\n        x = self.dropout(x, training=training)\n    \n        for i in range(self.num_layers):\n            x = self.enc_layers[i](x, e, training, mask)\n\n        return x  # (batch_size, input_seq_len, d_model)\n\n\nclass Transformer(tf.keras.Model):\n    def __init__(self, num_layers, en_d_model, en_num_heads, dff, pe_input, rate=0.2):\n        super(Transformer, self).__init__()\n\n        self.encoder = Encoder(num_layers, en_d_model, en_num_heads, dff, \n                           pe_input, rate)\n\n\n        self.second_final_layer = tf.keras.layers.Dense(dff)\n        self.final_layer = Dense(1,activation = 'sigmoid')\n    \n    def call(self, inp1, inp2, training, mask):\n\n        enc_output = self.encoder(inp1, inp2, training, mask)  # (batch_size, inp_seq_len, d_model)\n            \n        second_final_output = self.second_final_layer(enc_output)  # (batch_size, tar_seq_len, question_answer_pair_size)\n        final_output = self.final_layer(second_final_output)\n        return final_output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_layers = 2\nd_model = 64\nnum_heads = 2\ndff = 256 \nquestion_answer_pair_size = train_past_question_answer.max()+1\n\nn_question = train_current_question.max()+1\n\n\npe_input = 30\n\n\ndef build(num_layers, d_model, num_heads, dff, question_answer_pair_size, n_question, pe_input):\n    \n    masking_func = lambda inputs, previous_mask: previous_mask\n    en_input1 = Input(batch_shape = (None, None), name = 'question_answer_pair')\n    en_input1_embed = Embedding(question_answer_pair_size, d_model)(en_input1)\n    en_input2 = Input(batch_shape = (None, None, 1), name = 'other_feature1')\n    en_input2_embed = Dense(d_model, input_shape = (None, None, 1))(en_input2)\n    en_input3 = Input(batch_shape = (None, None, 1), name = 'other_feature2')\n    en_input3_embed = Dense(d_model, input_shape = (None, None, 1))(en_input3)\n    \n    en_input_embed_sum = tf.math.add_n([en_input1_embed, en_input2_embed, en_input3_embed])\n    \n    \n    en_input4 = Input(batch_shape = (None, None), name = 'current_question')\n    en_input4_embed = Embedding(n_question, d_model)(en_input4)\n    \n    \n    look_ahead_mask = create_look_ahead_mask(tf.shape(en_input_embed_sum)[1])\n    padding_mask = create_padding_mask(en_input1)\n    combined_mask = tf.maximum(look_ahead_mask, padding_mask)\n    \n    \n    transformer = Transformer(num_layers, d_model, num_heads, dff, pe_input)\n    \n    final_output = transformer(en_input_embed_sum, en_input4_embed, True, combined_mask)\n    output_mask = create_padding_output_mask(en_input1)\n    output = multiply([final_output, output_mask])\n    \n    model = Model(inputs=[en_input1, en_input2, en_input3, en_input4], outputs=output)\n    model.compile( optimizer = 'adam',\n                    loss = 'binary_crossentropy',\n                    metrics=['accuracy',AUC()])\n    \n    return model\n\nSAKT_model = build(num_layers, d_model, num_heads, dff, question_answer_pair_size, n_question, pe_input)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SAKT_model.fit([train_past_question_answer, train_prior_elaps_time, train_time_diff, train_current_question],train_answered_correcly, \n             validation_data=([valid_past_question_answer, valid_prior_elaps_time, valid_time_diff, valid_current_question], valid_answered_correcly), batch_size = 200,\n             epochs = 2, verbose = 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# SAINT"},{"metadata":{"trusted":true},"cell_type":"code","source":"class EncoderLayer2(tf.keras.layers.Layer):\n    def __init__(self, d_model, num_heads, dff, rate=0.2):\n        super(EncoderLayer2, self).__init__()\n\n        self.mha = MultiHeadAttention(d_model, num_heads)\n        self.ffn = point_wise_feed_forward_network(d_model, dff)\n\n        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n\n        self.dropout1 = tf.keras.layers.Dropout(rate)\n        self.dropout2 = tf.keras.layers.Dropout(rate)\n\n    def call(self, x, training, mask):\n\n        attn_output, _ = self.mha(x, x, x, mask)  # (batch_size, input_seq_len, d_model)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(x + attn_output)  # (batch_size, input_seq_len, d_model)\n\n        ffn_output = self.ffn(out1)  # (batch_size, input_seq_len, d_model)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        out2 = self.layernorm2(out1 + ffn_output)  # (batch_size, input_seq_len, d_model)\n\n        return out2\n\n\n\nclass Encoder2(tf.keras.layers.Layer):\n    def __init__(self, num_layers, d_model, num_heads, dff,\n                   maximum_position_encoding, rate=0.2):\n        super(Encoder2, self).__init__()\n\n        self.d_model = d_model\n        self.num_layers = num_layers\n\n        #self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)\n        self.pos_encoding = positional_encoding(maximum_position_encoding, \n                                                self.d_model)\n\n\n        self.enc_layers = [EncoderLayer2(d_model, num_heads, dff, rate) \n                           for _ in range(num_layers)]\n\n        self.dropout = tf.keras.layers.Dropout(rate)\n\n    def call(self, x, training, mask):\n\n        seq_len = tf.shape(x)[1]\n\n        # adding embedding and position encoding.\n        #x = self.embedding(x)  # (batch_size, input_seq_len, d_model)\n        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n        x += self.pos_encoding[:, :seq_len, :]\n\n        x = self.dropout(x, training=training)\n    \n        for i in range(self.num_layers):\n            x = self.enc_layers[i](x, training, mask)\n\n        return x  # (batch_size, input_seq_len, d_model)\n\n    \nclass DecoderLayer2(tf.keras.layers.Layer):\n    def __init__(self, d_model, num_heads, dff, rate=0.2):\n        super(DecoderLayer2, self).__init__()\n\n        self.mha1 = MultiHeadAttention(d_model, num_heads)\n        self.mha2 = MultiHeadAttention(d_model, num_heads)\n\n        self.ffn = point_wise_feed_forward_network(d_model, dff)\n\n        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n\n        self.dropout1 = tf.keras.layers.Dropout(rate)\n        self.dropout2 = tf.keras.layers.Dropout(rate)\n        self.dropout3 = tf.keras.layers.Dropout(rate)\n\n\n    def call(self, x, enc_output, training, \n                look_ahead_mask, padding_mask):\n    # enc_output.shape == (batch_size, input_seq_len, d_model)\n\n        attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask)  # (batch_size, target_seq_len, d_model)\n        attn1 = self.dropout1(attn1, training=training)\n        out1 = self.layernorm1(attn1 + x)\n\n        attn2, attn_weights_block2 = self.mha2(\n                enc_output, enc_output, out1, padding_mask)  # (batch_size, target_seq_len, d_model)\n        attn2 = self.dropout2(attn2, training=training)\n        out2 = self.layernorm2(attn2 + out1)  # (batch_size, target_seq_len, d_model)\n\n        ffn_output = self.ffn(out2)  # (batch_size, target_seq_len, d_model)\n        ffn_output = self.dropout3(ffn_output, training=training)\n        out3 = self.layernorm3(ffn_output + out2)  # (batch_size, target_seq_len, d_model)\n\n        return out3, attn_weights_block1, attn_weights_block2\n\n    \n    \nclass Decoder2(tf.keras.layers.Layer):\n    def __init__(self, num_layers, d_model, num_heads, dff,\n                    maximum_position_encoding, rate=0.2):\n        super(Decoder2, self).__init__()\n\n        self.d_model = d_model\n        self.num_layers = num_layers\n\n        #self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)\n        self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)\n\n        self.dec_layers = [DecoderLayer2(d_model, num_heads, dff, rate) \n                       for _ in range(num_layers)]\n        self.dropout = tf.keras.layers.Dropout(rate)\n\n    def call(self, x, enc_output, training, \n               look_ahead_mask, padding_mask):\n\n        seq_len = tf.shape(x)[1]\n        attention_weights = {}\n\n        #x = self.embedding(x)  # (batch_size, target_seq_len, d_model)\n        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n        x += self.pos_encoding[:, :seq_len, :]\n\n        x = self.dropout(x, training=training)\n\n        for i in range(self.num_layers):\n              x, block1, block2 = self.dec_layers[i](x, enc_output, training, look_ahead_mask, padding_mask)\n\n        attention_weights['decoder_layer{}_block1'.format(i+1)] = block1\n        attention_weights['decoder_layer{}_block2'.format(i+1)] = block2\n\n    # x.shape == (batch_size, target_seq_len, d_model)\n        return x, attention_weights    \n\nclass Transformer2(tf.keras.Model):\n    def __init__(self, num_layers, d_model, num_heads, dff, padding_length, rate=0.2):\n        super(Transformer2, self).__init__()\n\n        self.encoder = Encoder2(num_layers, d_model, num_heads, dff, padding_length)\n        \n        self.decoder = Decoder2(num_layers, d_model, num_heads, dff, padding_length)\n\n        self.second_final_layer = tf.keras.layers.Dense(dff)\n        self.final_layer = Dense(1,activation = 'sigmoid')\n    \n    def call(self, inp1, inp2, training, en_combined_mask, de_look_ahead_mask, de_padding_mask):\n\n        enc_output = self.encoder(inp1, training, en_combined_mask)  # (batch_size, inp_seq_len, d_model)\n        dec_output, attention_weights = self.decoder(\n                inp2, enc_output, training, de_look_ahead_mask, de_padding_mask)\n            \n        second_final_output = self.second_final_layer(dec_output)  # (batch_size, tar_seq_len, question_answer_pair_size)\n        final_output = self.final_layer(second_final_output)\n        return final_output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_layers = 1\nd_model = 64\nnum_heads = 2\ndff = 256\n\n\nn_question = train_current_question.max()+1\nn_answer = 3\n\npe_input = 30\n\ndef build(num_layers, d_model, num_heads, dff, n_question, n_answer, pe_input):\n\n    en_input1 = Input(batch_shape = (None, None), name = 'current_question')\n    en_input1_embed = Embedding(n_question, d_model)(en_input1)\n\n    \n    en_look_ahead_mask = create_look_ahead_mask(tf.shape(en_input1)[1])\n    en_padding_mask = create_padding_mask(en_input1)\n    en_combined_mask = tf.maximum(en_look_ahead_mask, en_padding_mask)\n    \n    \n    \n    #en_input1_embed = K.sum(en_input1_embed, axis = -2)\n    de_input2 = Input(batch_shape = (None, None), name = 'past_answer')\n    de_input2_embed = Embedding(n_answer, d_model)(de_input2)\n    de_input3 = Input(batch_shape = (None, None, 1), name = 'other_feature1')\n    de_input3_embed = Dense(d_model, input_shape = (None, None, 1))(de_input3)\n    de_input4 = Input(batch_shape = (None, None, 1), name = 'other_feature2')\n    de_input4_embed = Dense(d_model, input_shape = (None, None, 1))(de_input4)   \n    de_input = tf.math.add_n([de_input2_embed, de_input3_embed, de_input4_embed])\n    \n    #de_look_ahead_mask = create_look_ahead_mask(tf.shape(de_input4)[1])\n    #de_padding_mask = create_padding_mask(de_input4)\n    #de_combined_mask = tf.maximum(de_look_ahead_mask, de_padding_mask)\n    \n    \n    transformer = Transformer2(num_layers, d_model, num_heads, dff, pe_input)\n    \n    final_output = transformer(en_input1_embed, de_input, True, en_combined_mask, en_combined_mask, en_combined_mask)\n    output_mask = create_padding_output_mask(en_input1)\n    \n    model = Model(inputs=[en_input1, de_input2, de_input3, de_input4], outputs=final_output)\n    model.compile( optimizer = 'adam',\n                    loss = 'binary_crossentropy',\n                    metrics=['accuracy',AUC()])\n    \n    return model\n\nSAINT_model = build(num_layers, d_model, num_heads, dff, n_question, n_answer, pe_input)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SAINT_model.fit([train_current_question, train_past_answered_correctly, train_prior_elaps_time, train_time_diff],train_answered_correcly, \n             validation_data=([valid_current_question, valid_past_answered_correctly, valid_prior_elaps_time, valid_time_diff], valid_answered_correcly), \n             batch_size = 200,\n             epochs = 2, verbose = 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# DKT"},{"metadata":{},"cell_type":"markdown","source":"For DKT, I subtract 1 from current_question, (tag, answered_correctly) tuple, (part, answered_correctly) tuple. This is because I will use tensorflow.one_hot to create one hot encoding. tensorflow.one_hot works in a following way: suppose I have a vector \n\n> y = [0, 2, -1, 1]\n\nand I have 4 cateogries and each are indexed by 0-3. Then,\n\n> tensorflow.one_hot(y,4) =\n\n> [[1, 0, 0, 0],  # one_hot(0)\n\n>  [0, 0, 1, 0],  # one_hot(2)\n\n>  [0, 0, 0, 0],  # one_hot(-1)\n\n>  [0, 1, 0, 0]]  # one_hot(1)\n\nThus, I need to change padding with 0 with -1. Then, once I create one hot encoding, I can use zeros to mask."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_answered_correcly = np.reshape(train_y[:,1:],(-1,30,1))\ntrain_past_question_answer -= 1\ntrain_current_question -= 1\ntrain_other_feats = np.dstack((train_prior_elaps_time,train_time_diff))\n\nvalid_answered_correcly = np.reshape(valid_y[:,1:],(-1,30,1))\nvalid_past_question_answer -= 1\nvalid_current_question -= 1\nvalid_other_feats = np.dstack((valid_prior_elaps_time,valid_time_diff))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Parameter setting\nother_input_dim = 2\nhidden_layer_size = 50\ninput_dim_order = train_current_question.max() + 1\nprev_q_perform_dim = train_past_question_answer.max() + 1\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def dkt_build(hidden_layer_size, input_dim_order, prev_q_perform_dim, other_input_dim):    \n    # Inputs of DKT: tuples of question and answers. As we have 13k+ questions, the total number of tuples will be 23k+.\n    # One hot encoding for 23k+ categorical variables will consume too much memory.\n    # Thus, I replace questions with tags and parts.\n    # Plus, I added two additional features: prior_elaps_time and time_diff as used in SAINT\n    \n    masking_func = lambda inputs, previous_mask: previous_mask #masking_func is needed as lambda function I used below will ignore a masking layer\n    \n    prev_question_ans = Input(batch_shape = (None, None), dtype = 'int32', name = 'prev_qn_ans') # input1 (questions, answer correctly) tuple\n    one_hot_prev_question_ans = tf.one_hot(prev_question_ans, prev_q_perform_dim, axis = -1) # one hot encoding input1\n    \n    other_input = Input(batch_shape = (None, None, other_input_dim), name= 'other_input') # other features\n    \n    one_hot = Concatenate()([one_hot_prev_question_ans, other_input]) # concatenate all input features\n    \n    masked_one_hot = (Masking(mask_value= 0, input_shape = (None, None, prev_q_perform_dim + other_input_dim)))(one_hot) # mask\n    \n    \n    lstm_out = LSTM(hidden_layer_size, input_shape = (None, None, prev_q_perform_dim + other_input_dim),\n                    dropout=0.2, recurrent_dropout =0.2, return_sequences = True)(masked_one_hot)\n    \n    \n    dense_out = Dense(input_dim_order, input_shape = (None, None, hidden_layer_size), activation='sigmoid')(lstm_out)\n    \n    order = Input(batch_shape = (None, None), dtype = 'int32', name = 'order') # input3 (questions)\n    one_hot_order = tf.one_hot(order, input_dim_order, axis = -1) # one hot encoding questions\n    \n    # vector multiplication\n    merged = multiply([dense_out, one_hot_order])\n    \n    def reduce_dim(x):\n        x = K.max(x, axis = 2, keepdims = True)\n        return x\n\n    def reduce_dim_shape(input_shape):\n        shape = list(input_shape)\n        shape[-1] = 1\n        print (\"reduced_shape\", shape)\n        return tuple(shape)\n    \n    reduced = Lambda(reduce_dim, output_shape = reduce_dim_shape, mask = masking_func)(merged)\n    \n    \n    model = Model(inputs=[prev_question_ans, other_input, order], outputs=reduced)\n    model.compile( optimizer = 'adam',\n                    loss = 'binary_crossentropy',\n                    metrics=['accuracy',AUC()])\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dkt_model = dkt_build(hidden_layer_size, input_dim_order, prev_q_perform_dim, other_input_dim)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dkt_model.fit([train_past_question_answer, train_other_feats, train_current_question], train_answered_correcly,\n                    validation_data=([valid_past_question_answer, valid_other_feats, valid_current_question], valid_answered_correcly),\n                          epochs = 2, verbose = 1, batch_size = 200)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\nenv = riiideducation.make_env()\niter_test = env.iter_test()\nset_predict = env.predict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"previous_test_df = None\nfor (test_df, sample_prediction_df) in iter_test:\n    if previous_test_df is not None:\n        previous_test_df[TARGET] = np.array(eval(test_df[\"prior_group_answers_correct\"].iloc[0]))[mask]\n        update_user_feats(previous_test_df, last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict)\n    \n    test_df = pd.merge(test_df, questions_df[['question_id', 'part','tag']], left_on='content_id', right_on = 'question_id', how='left')\n    mask = (test_df['content_type_id'] == 0).values.tolist()\n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n    test_df = add_user_feats_without_update(test_df , last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict)\n    test_df = pd.merge(test_df, content_df, on='content_id',  how=\"left\")\n    test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False).astype('int8')\n    test_df['prior_question_elapsed_time'] = test_df.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)\n    previous_test_df = test_df.copy()\n    lgbm_predict =  lgbm_model.predict(test_df[FEATS])\n    \n    current_question, past_question_answer, past_other_feats, past_answer_correctly = get_user_feats_for_nn_without_update(test_df, past_question, past_answer, past_prior_elaps, past_time_diff)\n    prior_elaps_time = np.reshape(past_other_feats[:,:,0],(-1,30,1))\n    time_diff = np.reshape(past_other_feats[:,:,1],(-1,30,1))    \n    \n    sakt_predict = SAKT_model.predict([past_question_answer, prior_elaps_time, time_diff, current_question], batch_size = 500)    \n    \n    saint_predict = SAINT_model.predict([current_question, past_answer_correctly, prior_elaps_time, time_diff], batch_size = 500)\n    \n    past_question_answer = past_question_answer - 1\n    current_question = current_question - 1\n    \n    dkt_predict = dkt_model.predict([past_question_answer, past_other_feats, current_question], batch_size = 200)    \n    \n    test_df[TARGET] = dkt_predict[:,-1,0]*(0.1)  + sakt_predict[:,-1,0]*(0.15) + lgbm_predict*0.6 + saint_predict[:,-1,0]*(0.15)\n    \n    set_predict(test_df[['row_id', TARGET]])\n    #---\n    #print(sample_prediction_df)\n    #print(test_df[['row_id', TARGET]])\n    #print(test_df.shape, sample_prediction_df.shape, test_df[TARGET].shape)\n    #---","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}