{"cells":[{"metadata":{},"cell_type":"markdown","source":"I need take the rest.\nI will upload the training part and add comment later to explain the detail.\n\nThanks for very great notebook. The notebook got many idea from the following.\n1. https://www.kaggle.com/manikanthr5/riiid-sakt-model-training-public\n2. https://www.kaggle.com/wangsg/a-self-attentive-model-for-knowledge-tracing\n3. https://www.kaggle.com/leadbest/sakt-with-randomization-state-updates\n\n"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import psutil\nimport joblib\n\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nversion 6 is add three lag feature and add zero padding .\n\nfeature list:\n\n1.content_id\n2.answered_correctly\n3.part\n4.prior_question_elapsed_time\n5.time_lag1\n6.time_lag2\n7.time_lag3\n8.prior_question_had_explanation\n\n\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## cv 0.7993","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## epoch - 0 train_loss - 0.4103 train_auc - 0.7990 val_loss - 0.5121 val_auc - 0.7993 time=641.96s","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_SEQ = 100\nn_part = 7\nD_MODEL = 256\nN_LAYER = 2\nDROPOUT = 0.1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def feature_time_lag(df, time_dict):\n\n    tt = np.zeros(len(df), dtype=np.int64)\n\n    for ind, row in enumerate(df[['user_id','timestamp','task_container_id']].values):\n\n        if row[0] in time_dict.keys():\n            if row[2]-time_dict[row[0]][1] == 0:\n\n                tt[ind] = time_dict[row[0]][2]\n\n            else:\n                t_last = time_dict[row[0]][0]\n                task_ind_last = time_dict[row[0]][1]\n                tt[ind] = row[1]-t_last\n                time_dict[row[0]] = (row[1], row[2], tt[ind])\n        else:\n            # time_dict : timestamp, task_container_id, lag_time\n            time_dict[row[0]] = (row[1], row[2], -1)\n            tt[ind] =  0\n\n    df[\"time_lag\"] = tt\n    return df\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FFN(nn.Module):\n    def __init__(self, state_size=200):\n        super(FFN, self).__init__()\n        self.state_size = state_size\n\n        self.lr1 = nn.Linear(state_size, state_size)\n        self.relu = nn.ReLU()\n        self.lr2 = nn.Linear(state_size, state_size)\n        self.dropout = nn.Dropout(DROPOUT)\n    \n    def forward(self, x):\n        x = self.lr1(x)\n        x = self.relu(x)\n        x = self.lr2(x)\n        return self.dropout(x)\n\ndef future_mask(seq_length):\n    future_mask = np.triu(np.ones((seq_length, seq_length)), k=1).astype('bool')\n    return torch.from_numpy(future_mask)\n\n\nclass SAINTModel(nn.Module):\n    def __init__(self, n_skill, n_part, max_seq=MAX_SEQ, embed_dim= 128, elapsed_time_cat_flag = True):\n        super(SAINTModel, self).__init__()\n\n        self.n_skill = n_skill\n        self.embed_dim = embed_dim\n        self.n_cat = n_part\n        self.elapsed_time_cat_flag = elapsed_time_cat_flag\n\n        self.e_embedding = nn.Embedding(self.n_skill+1, embed_dim) ## exercise\n        self.c_embedding = nn.Embedding(self.n_cat+1, embed_dim) ## category\n        self.pos_embedding = nn.Embedding(max_seq-1, embed_dim) ## position\n        self.res_embedding = nn.Embedding(2+1, embed_dim) ## response\n\n\n        if self.elapsed_time_cat_flag == True:\n            self.elapsed_time_embedding = nn.Embedding(300+1, embed_dim) ## elapsed time (the maximum elasped time is 300)\n            self.lag_embedding1 = nn.Embedding(300+1, embed_dim) ## lag time1 for 300 seconds\n            self.lag_embedding2 = nn.Embedding(1440+1, embed_dim) ## lag time2 for 1440 minutes\n            self.lag_embedding3 = nn.Embedding(365+1, embed_dim) ## lag time3 for 365 days\n\n        else:\n            self.elapsed_time_embedding = nn.Linear(1, embed_dim, bias=False) ## elapsed time\n            self.lag_embedding = nn.Linear(1, embed_dim, bias=False) ## lag time\n\n\n        self.exp_embedding = nn.Embedding(2+1, embed_dim) ## user had explain\n\n        self.transformer = nn.Transformer(nhead=8, d_model = embed_dim, num_encoder_layers= N_LAYER, num_decoder_layers= N_LAYER, dropout = DROPOUT)\n\n        self.dropout = nn.Dropout(DROPOUT)\n        self.layer_normal = nn.LayerNorm(embed_dim) \n        self.ffn = FFN(embed_dim)\n        self.pred = nn.Linear(embed_dim, 1)\n    \n    def forward(self, question, part, response, elapsed_time, lag_time, exp):\n\n        device = question.device  \n\n        ## embedding layer\n        question = self.e_embedding(question)\n        part = self.c_embedding(part)\n        pos_id = torch.arange(question.size(1)).unsqueeze(0).to(device)\n        pos_id = self.pos_embedding(pos_id)\n        res = self.res_embedding(response)\n        exp = self.exp_embedding(exp)\n\n        if self.elapsed_time_cat_flag == True:\n\n            ## feature engineering\n            ## elasped time\n            elapsed_time = torch.true_divide(elapsed_time, 1000)\n            elapsed_time = torch.round(elapsed_time)\n            elapsed_time = torch.where(elapsed_time.float() <= 300, elapsed_time, torch.tensor(300.0).to(device)).long()\n            elapsed_time = self.elapsed_time_embedding(elapsed_time)\n\n            ## lag_time1\n            lag_time = torch.true_divide(lag_time, 1000)\n            lag_time = torch.round(lag_time)\n            lag_time1 = torch.where(lag_time.float() <= 300, lag_time, torch.tensor(300.0).to(device)).long()\n\n            ## lag_time2\n            lag_time = torch.true_divide(lag_time, 60)\n            lag_time = torch.round(lag_time)\n            lag_time2 = torch.where(lag_time.float() <= 1440, lag_time, torch.tensor(1440.0).to(device)).long()\n\n            ## lag_time3\n            lag_time = torch.true_divide(lag_time, 1440)\n            lag_time = torch.round(lag_time)\n            lag_time3 = torch.where(lag_time.float() <= 365, lag_time, torch.tensor(365.0).to(device)).long()\n\n            ## lag time\n            lag_time1 = self.lag_embedding1(lag_time1) \n            lag_time2 = self.lag_embedding2(lag_time2) \n            lag_time3 = self.lag_embedding3(lag_time3)\n\n        else:\n\n            elapsed_time = elapsed_time.view(-1,1)\n            elapsed_time = self.elapsed_time_embedding(elapsed_time)\n            elapsed_time = elapsed_time.view(-1, MAX_SEQ-1, self.embed_dim)\n\n            lag_time = lag_time.view(-1,1)\n            lag_time = self.lag_embedding(lag_time)\n            lag_time = lag_time.view(-1, MAX_SEQ-1, self.embed_dim)\n\n            # elapsed_time = elapsed_time.view(-1, MAX_SEQ-1, 1)  ## [batch, s_len] => [batch, s_len, 1]\n            # elapsed_time = self.elapsed_time_embedding(elapsed_time)\n\n\n        enc = question + part + pos_id + exp\n        dec = pos_id + res + elapsed_time + lag_time1 + lag_time2 + lag_time3\n\n        enc = enc.permute(1, 0, 2) # x: [bs, s_len, embed] => [s_len, bs, embed]\n        dec = dec.permute(1, 0, 2)\n        mask = future_mask(enc.size(0)).to(device)\n\n        att_output = self.transformer(enc, dec, src_mask=mask, tgt_mask=mask, memory_mask = mask)\n        att_output = self.layer_normal(att_output)\n        att_output = att_output.permute(1, 0, 2) # att_output: [s_len, bs, embed] => [bs, s_len, embed]\n\n        x = self.ffn(att_output)\n        x = self.layer_normal(x + att_output)\n        x = self.pred(x)\n\n        return x.squeeze(-1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Pretrained Models"},{"metadata":{"trusted":true},"cell_type":"code","source":"n_skill = 13523\ngroup = joblib.load(\"../input/saint-plus-data-new/group_20210102.pkl.zip\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"time_dict = joblib.load(\"../input/saint-plus-data-new/time_dict.pkl.zip\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nmodel = SAINTModel(n_skill, n_part, embed_dim= D_MODEL)\ntry:\n    model.load_state_dict(torch.load(\"../input/saint-plus-model/saint_plus_model_20210108_v1.pt\"))\nexcept:\n    model.load_state_dict(torch.load(\"../input/saint-plus-model/saint_plus_model_20210108_v1.pt\", map_location='cpu'))\nmodel.to(device)\nmodel.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, samples, test_df, n_skills, max_seq=MAX_SEQ): \n        super(TestDataset, self).__init__()\n        self.samples = samples\n        self.user_ids = [x for x in test_df[\"user_id\"].unique()]\n        self.test_df = test_df\n        self.n_skill = n_skills\n        self.max_seq = max_seq\n\n    def __len__(self):\n        return self.test_df.shape[0]\n\n    def __getitem__(self, index):\n        test_info = self.test_df.iloc[index]\n\n        user_id = test_info[\"user_id\"]\n        target_id = test_info[\"content_id\"]\n        part = test_info[\"part\"]\n        pri_quest_elap = test_info[\"prior_question_elapsed_time\"]\n        time_lag = test_info[\"time_lag\"]\n        pri_quest_exp = test_info[\"prior_question_had_explanation\"]\n        \n        q = np.zeros(self.max_seq, dtype=int)\n        qa = np.zeros(self.max_seq, dtype=int)\n        res = np.zeros(self.max_seq, dtype=int)\n        p = np.zeros(self.max_seq, dtype=int)\n        pri_elap = np.zeros(self.max_seq, dtype=int)\n        lag = np.zeros(self.max_seq, dtype=int)\n        pri_exp = np.zeros(self.max_seq, dtype=int)\n\n        if user_id in self.samples.index:\n            q_, qa_, p_, pri_elap_, lag_, pri_exp_ = self.samples[user_id]\n            \n            seq_len = len(q_)\n            \n            ## for zero padding\n            q_ = q_+1\n            pri_exp_ = pri_exp_ + 1\n            res_ = qa_ + 1\n            \n\n            if seq_len >= self.max_seq:\n                q = q_[-self.max_seq:]\n                qa = qa_[-self.max_seq:]\n                res = res_[-self.max_seq:]\n                p = p_[-self.max_seq:]\n                pri_elap = pri_elap_[-self.max_seq:]\n                lag = lag_[-self.max_seq:]\n                pri_exp = pri_exp_[-self.max_seq:]\n                \n            else:\n                q[-seq_len:] = q_\n                qa[-seq_len:] = qa_\n                res[-seq_len:] = res_\n                p[-seq_len:] = p_\n                pri_elap[-seq_len:] = pri_elap_\n                lag[-seq_len:] = lag_\n                pri_exp[-seq_len:] = pri_exp_\n                \n        \n        exercise = np.append(q[2:], [target_id+1])\n        part = np.append(p[2:], [part])\n        elap = np.append(pri_elap[2:], [pri_quest_elap])\n        lag = np.append(lag[2:], [time_lag])\n        pri_exp = np.append(pri_exp[2:], [pri_quest_exp+1])\n\n        response = res[1:]\n\n        return  exercise, part, response, elap, lag, pri_exp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prev_test_df = None\n\nfor (test_df, sample_prediction_df) in iter_test:\n  \n    if (prev_test_df is not None) & (psutil.virtual_memory().percent < 90):\n        prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])\n        prev_test_df = prev_test_df[prev_test_df.content_type_id == False]\n        \n        ## lag time\n        prev_test_df = feature_time_lag(prev_test_df, time_dict)\n\n\n        prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly', 'part', 'prior_question_elapsed_time', 'time_lag', 'prior_question_had_explanation']].groupby('user_id').apply(lambda r: (\n            r['content_id'].values,\n            r['answered_correctly'].values,\n            r['part'].values,\n            r['prior_question_elapsed_time'].values,\n            r['time_lag'].values,\n            r['prior_question_had_explanation'].values))\n        \n        for prev_user_id in prev_group.index:\n            if prev_user_id in group.index:\n                group[prev_user_id] = (\n                    np.append(group[prev_user_id][0], prev_group[prev_user_id][0])[-MAX_SEQ:], \n                    np.append(group[prev_user_id][1], prev_group[prev_user_id][1])[-MAX_SEQ:],\n                    np.append(group[prev_user_id][2], prev_group[prev_user_id][2])[-MAX_SEQ:],\n                    np.append(group[prev_user_id][3], prev_group[prev_user_id][3])[-MAX_SEQ:],\n                    np.append(group[prev_user_id][4], prev_group[prev_user_id][4])[-MAX_SEQ:],\n                    np.append(group[prev_user_id][5], prev_group[prev_user_id][5])[-MAX_SEQ:]\n                )\n \n            else:\n                group[prev_user_id] = (\n                    prev_group[prev_user_id][0], \n                    prev_group[prev_user_id][1],\n                    prev_group[prev_user_id][2],\n                    prev_group[prev_user_id][3],\n                    prev_group[prev_user_id][4],\n                    prev_group[prev_user_id][5]\n                )\n\n            \n    ## elapsed time\n    test_df.prior_question_elapsed_time = test_df.prior_question_elapsed_time.fillna(0)\n    \n    ## prior_question_had_explanation\n    test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(value = False).astype(int)\n    \n    test_df = test_df.merge(questions_df[[\"question_id\",\"part\"]], how = \"left\",left_on = 'content_id', right_on = 'question_id')  \n              \n    prev_test_df = test_df.copy()\n            \n    ## drop lecture\n    test_df = test_df[test_df.content_type_id == False]\n    \n    \n    ## lag time\n    test_df = feature_time_lag(test_df, time_dict)\n    \n    \n    test_dataset = TestDataset(group, test_df, n_skill)\n    test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)\n    \n    outs = []\n\n    for item in test_dataloader:\n        exercise = item[0].to(device).long()\n        part = item[1].to(device).long()\n        response = item[2].to(device).long()\n        elapsed_time = item[3].to(device).long()\n        lag_time = item[4].to(device).long()\n        pri_exp = item[5].to(device).long()\n        \n        with torch.no_grad():\n            output = model(exercise, part, response, elapsed_time, lag_time, pri_exp)\n        outs.extend(torch.sigmoid(output)[:, -1].view(-1).data.cpu().numpy())\n        \n    test_df['answered_correctly'] = outs\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}