{"cells":[{"metadata":{},"cell_type":"markdown","source":"Thanks for very great notebook. The notebook got many idea from the following.\n1. https://www.kaggle.com/manikanthr5/riiid-sakt-model-training-public\n2. https://www.kaggle.com/wangsg/a-self-attentive-model-for-knowledge-tracing\n3. https://www.kaggle.com/leadbest/sakt-with-randomization-state-updates\n\nThe notebook will raise out of memory error. I trained the model in local machine.\nIt needs around 20~30GB ram."},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:53:11.259976Z","start_time":"2021-01-07T01:53:10.519848Z"},"id":"zWFPwwQiJHkK","trusted":true},"cell_type":"code","source":"import psutil\nimport joblib\nimport random\nimport logging\nfrom tqdm import tqdm\n\nimport numpy as np\nimport gc\nimport pandas as pd\nimport time\n\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.preprocessing import QuantileTransformer\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:53:13.777967Z","start_time":"2021-01-07T01:53:13.774985Z"},"trusted":true},"cell_type":"code","source":"# logging.basicConfig(level=logging.DEBUG, filename=\"logfile20.txt\", filemode=\"a+\",\n#                         format=\"%(asctime)-15s %(levelname)-8s %(message)s\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* feature_time_lag function is to calculate the time delta between current time and previous time. The previous time is not allowed with same **task_container_id**."},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:53:14.424076Z","start_time":"2021-01-07T01:53:14.417275Z"},"id":"XCN8rj_IXkk5","trusted":true},"cell_type":"code","source":"def feature_time_lag(df, time_dict):\n\n    tt = np.zeros(len(df), dtype=np.int64)\n\n    for ind, row in enumerate(df[['user_id','timestamp','task_container_id']].values):\n\n        if row[0] in time_dict.keys():\n            if row[2]-time_dict[row[0]][1] == 0:\n\n                tt[ind] = time_dict[row[0]][2]\n\n            else:\n                t_last = time_dict[row[0]][0]\n                task_ind_last = time_dict[row[0]][1]\n                tt[ind] = row[1]-t_last\n                time_dict[row[0]] = (row[1], row[2], tt[ind])\n        else:\n            # time_dict : timestamp, task_container_id, lag_time\n            time_dict[row[0]] = (row[1], row[2], -1)\n            tt[ind] =  0\n\n    df[\"time_lag\"] = tt\n    return df\n","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:53:15.239119Z","start_time":"2021-01-07T01:53:15.093196Z"},"executionInfo":{"elapsed":1913,"status":"ok","timestamp":1609255337160,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"},"user_tz":-480},"id":"VAPKPK6VdjpJ","outputId":"02780bbd-7fac-4b71-c49a-e80258cbee5a","trusted":true},"cell_type":"code","source":"!nvidia-smi","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Parameters"},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:53:19.928876Z","start_time":"2021-01-07T01:53:19.925791Z"},"id":"2pX27onNJijU","trusted":true},"cell_type":"code","source":"MAX_SEQ = 100\nD_MODEL = 256 \nN_LAYER = 2\nBATCH_SIZE = 256\nDROPOUT = 0.1\nTIME_CAT_FLAG = True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data preprocessing"},{"metadata":{},"cell_type":"markdown","source":"* train_df.pkl is made from tito's [CV Strategy](https://www.kaggle.com/its7171/cv-strategy) and sorted by **viretual_time_stamp**."},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nimport pickle\nwith open(\"../input/saint-plus-data-new/train_df.pkl\",\"rb\") as f:\n    train_df = pickle.load(f)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:53:20.671794Z","start_time":"2021-01-07T01:53:20.657949Z"},"id":"UPByaJr85LHD","trusted":true},"cell_type":"code","source":"question = pd.read_csv(\"../input/riiid-test-answer-prediction/questions.csv\")","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:53:21.364464Z","start_time":"2021-01-07T01:53:21.362392Z"},"id":"qXx52EDcXu0j","trusted":true},"cell_type":"code","source":"%%time\ntime_dict = dict()\ntrain_df = feature_time_lag(train_df, time_dict)\n# del time_dict","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:53:24.389958Z","start_time":"2021-01-07T01:53:24.364425Z"},"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Save the time_dict.pkl for inference."},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:53:24.392761Z","start_time":"2021-01-07T01:53:24.390726Z"},"id":"7i49bXOrCt16","trusted":true},"cell_type":"code","source":"# joblib.dump(time_dict,\"/content/drive/MyDrive/riiid/SAINT_data/time_dict.pkl.zip\")\n# joblib.dump(time_dict, os.path.join(path, \"time_dict.pkl.zip\"))","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:53:25.914702Z","start_time":"2021-01-07T01:53:24.393603Z"},"id":"O6_5CnyCK25o","trusted":true},"cell_type":"code","source":"train_df = train_df[[\"timestamp\",\"user_id\",\"content_id\",\"content_type_id\",\"answered_correctly\",\"prior_question_elapsed_time\",\"prior_question_had_explanation\",\"time_lag\"]]","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:53:28.376386Z","start_time":"2021-01-07T01:53:25.915567Z"},"id":"-p0yLPjjeuW3","trusted":true},"cell_type":"code","source":"train_df = train_df[train_df.content_type_id == 0]\n# train_df = train_df.sort_values(['timestamp'], ascending=True).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:53:28.89015Z","start_time":"2021-01-07T01:53:28.37722Z"},"id":"BzsoCI73pnVx","trusted":true},"cell_type":"code","source":"train_df.prior_question_elapsed_time = train_df.prior_question_elapsed_time.fillna(0)\ntrain_df['prior_question_had_explanation'] = train_df['prior_question_had_explanation'].fillna(value = False).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:53:37.917896Z","start_time":"2021-01-07T01:53:28.891533Z"},"id":"AcLu2a135PNu","trusted":true},"cell_type":"code","source":"train_df = train_df.merge(question[[\"question_id\",\"part\"]], how = \"left\", left_on = 'content_id', right_on = 'question_id')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* **TIME_CAT_FLAG** is flag for continuous or categorical embedding. If we choose continuous embedding, it use **rank gauss** to normalize **lag time** and **prior question elapsed time**."},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:53:38.780099Z","start_time":"2021-01-07T01:53:38.773701Z"},"trusted":true},"cell_type":"code","source":"if TIME_CAT_FLAG == False:\n    scaler_elapsed_time = QuantileTransformer(n_quantiles=100, random_state=0, output_distribution=\"normal\")\n    scaler_lag_time = QuantileTransformer(n_quantiles=100, random_state=0, output_distribution=\"normal\")\n    \n    raw_vec_elapsed_time = train_df.prior_question_elapsed_time.values.reshape(-1, 1)\n    raw_vec_lag_time = train_df.time_lag.values.reshape(-1, 1)\n    \n    scaler_elapsed_time.fit(raw_vec_elapsed_time)\n    scaler_lag_time.fit(raw_vec_lag_time)\n    \n    train_df.prior_question_elapsed_time = scaler_elapsed_time.transform(raw_vec_elapsed_time).reshape(1, -1)[0]\n    train_df.time_lag = scaler_lag_time.transform(raw_vec_lag_time).reshape(1, -1)[0]\n    ","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:53:38.78916Z","start_time":"2021-01-07T01:53:38.781091Z"},"id":"e1MZPihxrKje","trusted":true},"cell_type":"code","source":"train = train_df.iloc[:int(95/100 * len(train_df))]\nval = train_df.iloc[int(95/100 * len(train_df)):]","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:53:38.795111Z","start_time":"2021-01-07T01:53:38.790151Z"},"executionInfo":{"elapsed":98636,"status":"ok","timestamp":1609255447802,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"},"user_tz":-480},"id":"LDbKltCOfhlT","outputId":"81345a77-d32c-41ef-8eae-095348b8e466","trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:53:38.802832Z","start_time":"2021-01-07T01:53:38.796161Z"},"executionInfo":{"elapsed":98477,"status":"ok","timestamp":1609255447803,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"},"user_tz":-480},"id":"VKER_kmDrWud","outputId":"3dfac15d-fe73-4dbc-d651-5cd9e4189f19","trusted":true},"cell_type":"code","source":"val.shape","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:53:38.80755Z","start_time":"2021-01-07T01:53:38.803723Z"},"id":"SdLO_x64YkP0","trusted":true},"cell_type":"code","source":"## drop outlier\n# temp_df = train_df.groupby(\"user_id\").count().reset_index()\n# outlier = temp_df[(temp_df.timestamp>9000)].user_id.tolist()\n# train_df = train_df[~train_df.user_id.isin(outlier)]","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:53:39.178658Z","start_time":"2021-01-07T01:53:38.808547Z"},"executionInfo":{"elapsed":98478,"status":"ok","timestamp":1609255448160,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"},"user_tz":-480},"id":"SCPAPhZyagsW","outputId":"9d774279-989e-4ba7-a55a-3cfe18e269c0","trusted":true},"cell_type":"code","source":"skills = train[\"content_id\"].unique()\nn_skill = len(skills)\nprint(\"number skills\", len(skills))","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:53:39.384876Z","start_time":"2021-01-07T01:53:39.179529Z"},"id":"Hk1E3toaOdz_","trusted":true},"cell_type":"code","source":"n_part = len(train[\"part\"].unique())","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:53:39.446675Z","start_time":"2021-01-07T01:53:39.408507Z"},"executionInfo":{"elapsed":101682,"status":"ok","timestamp":1609255452545,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"},"user_tz":-480},"id":"TQ-Bchlz1jpg","outputId":"5ec2920e-6ab5-43e5-e5ec-f50a7dae2f04","trusted":true},"cell_type":"code","source":"del train_df \ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Transform training and validation data to sequential format as input as Transformer"},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:54:21.676915Z","start_time":"2021-01-07T01:53:39.44803Z"},"id":"2uVUt61bKosK","trusted":true},"cell_type":"code","source":"train_group = train[['user_id', 'content_id', 'answered_correctly', 'part', 'prior_question_elapsed_time', 'time_lag', 'prior_question_had_explanation']].groupby('user_id').apply(lambda r: (\n            r['content_id'].values,\n            r['answered_correctly'].values,\n            r['part'].values,\n            r['prior_question_elapsed_time'].values,\n            r['time_lag'].values,\n            r['prior_question_had_explanation'].values))\n","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:54:23.871454Z","start_time":"2021-01-07T01:54:21.677868Z"},"id":"otE1TPTWsMCP","trusted":true},"cell_type":"code","source":"val_group = val[['user_id', 'content_id', 'answered_correctly', 'part', 'prior_question_elapsed_time', 'time_lag', 'prior_question_had_explanation']].groupby('user_id').apply(lambda r: (\n            r['content_id'].values,\n            r['answered_correctly'].values,\n            r['part'].values,\n            r['prior_question_elapsed_time'].values,\n            r['time_lag'].values,\n            r['prior_question_had_explanation'].values))\n","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:54:23.873866Z","start_time":"2021-01-07T01:54:23.872302Z"},"id":"5gjQa8jlADTT","trusted":true},"cell_type":"code","source":"del train_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* We need to save the group data for inference. In training phase, I split the data to training/validation set. However, we have to use full data to make group data for inference.\n* [Transformer group data for inference](https://www.kaggle.com/m10515009/transformer-group-data-for-inference/)"},{"metadata":{},"cell_type":"markdown","source":"# SAINT Dataset"},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:54:23.902012Z","start_time":"2021-01-07T01:54:23.887649Z"},"id":"jndbpdAfMeNf","trusted":false},"cell_type":"code","source":"class SAINTDataset(Dataset):\n    def __init__(self, group, n_skill, max_seq=MAX_SEQ):\n        super(SAINTDataset, self).__init__()\n        self.max_seq = max_seq\n        self.n_skill = n_skill\n        self.samples = {}\n        \n        self.user_ids = []\n        for user_id in group.index:\n            q, qa, part, pri_elap, lag, pri_exp = group[user_id]\n            if len(q) < 2:\n                continue\n            \n            # Credit to https://www.kaggle.com/manikanthr5/riiid-sakt-model-training-public\n            if len(q) > self.max_seq:\n                total_questions = len(q)\n                initial = total_questions % self.max_seq\n                if initial >= 2:\n                    self.user_ids.append(f\"{user_id}_0\")\n                    self.samples[f\"{user_id}_0\"] = (q[:initial], qa[:initial], part[:initial], pri_elap[:initial], lag[:initial], pri_exp[:initial])\n                for seq in range(total_questions // self.max_seq):\n                    self.user_ids.append(f\"{user_id}_{seq+1}\")\n                    start = initial + seq * self.max_seq\n                    end = initial + (seq + 1) * self.max_seq\n                    self.samples[f\"{user_id}_{seq+1}\"] = (q[start:end], qa[start:end], part[start:end], pri_elap[start:end], lag[start:end], pri_exp[start:end])\n            else:\n                user_id = str(user_id)\n                self.user_ids.append(user_id)\n                self.samples[user_id] = (q, qa, part, pri_elap, lag, pri_exp)\n    \n    def __len__(self):\n        return len(self.user_ids)\n\n    def __getitem__(self, index):\n        user_id = self.user_ids[index]\n        q_, qa_, part_, pri_elap_, lag_, pri_exp_ = self.samples[user_id]\n        seq_len = len(q_)\n\n        ## for zero padding\n        q_ = q_+1\n        pri_exp_ = pri_exp_ + 1\n        res_ = qa_ + 1\n        \n        q = np.zeros(self.max_seq, dtype=int)\n        qa = np.zeros(self.max_seq, dtype=int)\n        res = np.zeros(self.max_seq, dtype=int)\n        part = np.zeros(self.max_seq, dtype=int)\n        pri_elap = np.zeros(self.max_seq, dtype=float)\n        lag = np.zeros(self.max_seq, dtype=float)\n        pri_exp = np.zeros(self.max_seq, dtype=int)\n\n        if seq_len == self.max_seq:\n\n            q[:] = q_\n            qa[:] = qa_\n            res[:] = res_\n            part[:] = part_\n            pri_elap[:] = pri_elap_\n            lag[:] = lag_\n            pri_exp[:] = pri_exp_\n            \n        else:\n            q[-seq_len:] = q_\n            qa[-seq_len:] = qa_\n            res[-seq_len:] = res_\n            part[-seq_len:] = part_\n            pri_elap[-seq_len:] = pri_elap_\n            lag[-seq_len:] = lag_\n            pri_exp[-seq_len:] = pri_exp_\n        \n        exercise = q[1:]\n        part = part[1:]\n        response = res[:-1]\n        label = qa[1:]\n        elap = pri_elap[1:]\n\n        ## It's different from paper. The lag time including present lag time have more information. \n        lag = lag[1:]\n        pri_exp = pri_exp[1:]\n\n\n        return exercise, part, response, elap, lag, pri_exp, label","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:54:26.844002Z","start_time":"2021-01-07T01:54:23.902697Z"},"id":"XSilTYWthGi8","trusted":false},"cell_type":"code","source":"train_dataset = SAINTDataset(train_group, n_skill)\ntrain_dataloader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=8)\n\n\nval_dataset = SAINTDataset(val_group, n_skill)\nval_dataloader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=8)\n","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:54:26.846685Z","start_time":"2021-01-07T01:54:26.844854Z"},"trusted":false},"cell_type":"code","source":"item = val_dataset.__getitem__(3)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:54:26.863002Z","start_time":"2021-01-07T01:54:26.84739Z"},"trusted":false},"cell_type":"code","source":"item","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Transformer model(SAINT+ like)"},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:54:26.877512Z","start_time":"2021-01-07T01:54:26.864208Z"},"id":"_snimMNaVQL0","trusted":false},"cell_type":"code","source":"class FFN(nn.Module):\n    def __init__(self, state_size=200):\n        super(FFN, self).__init__()\n        self.state_size = state_size\n\n        self.lr1 = nn.Linear(state_size, state_size)\n        self.relu = nn.ReLU()\n        self.lr2 = nn.Linear(state_size, state_size)\n        self.dropout = nn.Dropout(DROPOUT)\n    \n    def forward(self, x):\n        x = self.lr1(x)\n        x = self.relu(x)\n        x = self.lr2(x)\n        return self.dropout(x)\n\ndef future_mask(seq_length):\n    future_mask = np.triu(np.ones((seq_length, seq_length)), k=1).astype('bool')\n    return torch.from_numpy(future_mask)\n\n\nclass SAINTModel(nn.Module):\n    def __init__(self, n_skill, n_part, max_seq=MAX_SEQ, embed_dim= 128, time_cat_flag = True):\n        super(SAINTModel, self).__init__()\n\n        self.n_skill = n_skill\n        self.embed_dim = embed_dim\n        self.n_cat = n_part\n        self.time_cat_flag = time_cat_flag\n\n        self.e_embedding = nn.Embedding(self.n_skill+1, embed_dim) ## exercise\n        self.c_embedding = nn.Embedding(self.n_cat+1, embed_dim) ## category\n        self.pos_embedding = nn.Embedding(max_seq-1, embed_dim) ## position\n        self.res_embedding = nn.Embedding(2+1, embed_dim) ## response\n\n\n        if self.time_cat_flag == True:\n            self.elapsed_time_embedding = nn.Embedding(300+1, embed_dim) ## elapsed time (the maximum elasped time is 300)\n            self.lag_embedding1 = nn.Embedding(300+1, embed_dim) ## lag time1 for 300 seconds\n            self.lag_embedding2 = nn.Embedding(1440+1, embed_dim) ## lag time2 for 1440 minutes\n            self.lag_embedding3 = nn.Embedding(365+1, embed_dim) ## lag time3 for 365 days\n\n        else:\n            self.elapsed_time_embedding = nn.Linear(1, embed_dim, bias=False) ## elapsed time\n            self.lag_embedding = nn.Linear(1, embed_dim, bias=False) ## lag time\n\n\n        self.exp_embedding = nn.Embedding(2+1, embed_dim) ## user had explain\n\n        self.transformer = nn.Transformer(nhead=8, d_model = embed_dim, num_encoder_layers= N_LAYER, num_decoder_layers= N_LAYER, dropout = DROPOUT)\n\n        self.dropout = nn.Dropout(DROPOUT)\n        self.layer_normal = nn.LayerNorm(embed_dim) \n        self.ffn = FFN(embed_dim)\n        self.pred = nn.Linear(embed_dim, 1)\n    \n    def forward(self, question, part, response, elapsed_time, lag_time, exp):\n\n        device = question.device  \n\n        ## embedding layer\n        question = self.e_embedding(question)\n        part = self.c_embedding(part)\n        pos_id = torch.arange(question.size(1)).unsqueeze(0).to(device)\n        pos_id = self.pos_embedding(pos_id)\n        res = self.res_embedding(response)\n        exp = self.exp_embedding(exp)\n\n        if self.time_cat_flag == True:\n\n            ## feature engineering\n            ## elasped time\n            elapsed_time = torch.true_divide(elapsed_time, 1000)\n            elapsed_time = torch.round(elapsed_time)\n            elapsed_time = torch.where(elapsed_time.float() <= 300, elapsed_time, torch.tensor(300.0).to(device)).long()\n            elapsed_time = self.elapsed_time_embedding(elapsed_time)\n\n            ## lag_time1\n            lag_time = torch.true_divide(lag_time, 1000)\n            lag_time = torch.round(lag_time)\n            lag_time1 = torch.where(lag_time.float() <= 300, lag_time, torch.tensor(300.0).to(device)).long()\n\n            ## lag_time2\n            lag_time = torch.true_divide(lag_time, 60)\n            lag_time = torch.round(lag_time)\n            lag_time2 = torch.where(lag_time.float() <= 1440, lag_time, torch.tensor(1440.0).to(device)).long()\n\n            ## lag_time3\n            lag_time = torch.true_divide(lag_time, 1440)\n            lag_time = torch.round(lag_time)\n            lag_time3 = torch.where(lag_time.float() <= 365, lag_time, torch.tensor(365.0).to(device)).long()\n\n            ## lag time\n            lag_time1 = self.lag_embedding1(lag_time1) \n            lag_time2 = self.lag_embedding2(lag_time2) \n            lag_time3 = self.lag_embedding3(lag_time3)\n            \n            enc = question + part + pos_id + exp\n            dec = pos_id + res + elapsed_time + lag_time1 + lag_time2 + lag_time3\n  \n\n        else:\n\n            elapsed_time = elapsed_time.view(-1,1)\n            elapsed_time = self.elapsed_time_embedding(elapsed_time)\n            elapsed_time = elapsed_time.view(-1, MAX_SEQ-1, self.embed_dim)\n\n            lag_time = lag_time.view(-1,1)\n            lag_time = self.lag_embedding(lag_time)\n            lag_time = lag_time.view(-1, MAX_SEQ-1, self.embed_dim)\n\n            # elapsed_time = elapsed_time.view(-1, MAX_SEQ-1, 1)  ## [batch, s_len] => [batch, s_len, 1]\n            # elapsed_time = self.elapsed_time_embedding(elapsed_time)\n\n            enc = question + part + pos_id + exp\n            dec = pos_id + res + elapsed_time + lag_time\n        \n\n        enc = enc.permute(1, 0, 2) # x: [bs, s_len, embed] => [s_len, bs, embed]\n        dec = dec.permute(1, 0, 2)\n        mask = future_mask(enc.size(0)).to(device)\n\n        att_output = self.transformer(enc, dec, src_mask=mask, tgt_mask=mask, memory_mask = mask)\n        att_output = self.layer_normal(att_output)\n        att_output = att_output.permute(1, 0, 2) # att_output: [s_len, bs, embed] => [bs, s_len, embed]\n\n        x = self.ffn(att_output)\n        x = self.layer_normal(x + att_output)\n        x = self.pred(x)\n\n        return x.squeeze(-1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# New model or pretrained model"},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:54:28.214125Z","start_time":"2021-01-07T01:54:26.878138Z"},"executionInfo":{"elapsed":1186,"status":"ok","timestamp":1609256415603,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"},"user_tz":-480},"id":"Ah6y5XOoVi8B","outputId":"fdc9caef-638d-4df5-f9e4-6e78e4bfae9a","trusted":false},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n# device = torch.device(\"cpu\")\n\nmodel = SAINTModel(n_skill, n_part, embed_dim= D_MODEL, time_cat_flag = TIME_CAT_FLAG)\n\n# try:\n#     model.load_state_dict(torch.load(\"./SAINT_model/saint_plus_model_20210102_padding_v2.pt\"))\n# except:\n#     model.load_state_dict(torch.load(\"./SAINT_model/saint_plus_model_20210102_padding_v2.pt\", map_location='cpu'))\n\n## AdamW\noptimizer = torch.optim.AdamW(model.parameters(), lr=5e-4)\ncriterion = nn.BCEWithLogitsLoss()\n\nmodel.to(device)\ncriterion.to(device)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:54:28.216784Z","start_time":"2021-01-07T01:54:28.214776Z"},"executionInfo":{"elapsed":756,"status":"ok","timestamp":1609256416966,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"},"user_tz":-480},"id":"UZr9BuGAV2pf","outputId":"19252423-d279-41d5-a809-5bed6715e9be","trusted":false},"cell_type":"code","source":"print(model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model training"},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T01:54:28.25179Z","start_time":"2021-01-07T01:54:28.217467Z"},"id":"9N4lZK2oV58q","trusted":false},"cell_type":"code","source":"def train_epoch(model, train_dataloader, val_dataloader, optimizer, criterion, device=\"cpu\", time_cat_flag = True):\n    model.train()\n\n    train_loss = []\n    num_corrects = 0\n    num_total = 0\n    labels = []\n    outs = []\n\n    start_time = time.time()\n\n    ## training\n    for item in train_dataloader:\n        exercise = item[0].to(device).long()\n        part = item[1].to(device).long()\n        response = item[2].to(device).long()\n\n        if time_cat_flag == True:\n            elapsed_time = item[3].to(device).long()\n            lag_time = item[4].to(device).long()\n        else :\n            elapsed_time = item[3].to(device).float()\n            lag_time = item[4].to(device).float()\n\n        exp = item[5].to(device).long()\n        label = item[6].to(device).float()\n        target_mask = (exercise != 0)\n\n        optimizer.zero_grad()\n        output = model(exercise, part, response, elapsed_time, lag_time, exp)\n        \n        loss = criterion(output, label)\n        loss.backward()\n        optimizer.step()\n        train_loss.append(loss.item())\n        \n        # mask the output\n        output_mask = torch.masked_select(output, target_mask)\n        label_mask = torch.masked_select(label, target_mask)\n\n        labels.extend(label_mask.view(-1).data.cpu().numpy())\n        outs.extend(output_mask.view(-1).data.cpu().numpy())\n\n    train_auc = roc_auc_score(labels, outs)\n    train_loss = np.mean(train_loss)\n\n    labels = []\n    outs = []\n    val_loss = []\n\n    # validation\n    model.eval()\n    for item in val_dataloader:\n        exercise = item[0].to(device).long()\n        part = item[1].to(device).long()\n        response = item[2].to(device).long()\n\n        if time_cat_flag == True:\n            elapsed_time = item[3].to(device).long()\n            lag_time = item[4].to(device).long()\n        else :\n            elapsed_time = item[3].to(device).float()\n            lag_time = item[4].to(device).float()\n\n        exp = item[5].to(device).long()\n        label = item[6].to(device).float()\n        target_mask = (exercise != 0)\n        \n        output = model(exercise, part, response, elapsed_time, lag_time, exp)\n        \n        ## mask the output\n        output = torch.masked_select(output, target_mask)\n        label = torch.masked_select(label, target_mask)\n        \n        loss = criterion(output, label)\n        val_loss.append(loss.item())\n\n        labels.extend(label.view(-1).data.cpu().numpy())\n        outs.extend(output.view(-1).data.cpu().numpy())\n\n    val_auc = roc_auc_score(labels, outs)\n    val_loss = np.mean(val_loss)\n\n    elapsed_time = time.time() - start_time \n\n    return train_loss, train_auc, val_loss, val_auc, elapsed_time","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"start_time":"2021-01-07T01:53:36.512Z"},"trusted":false},"cell_type":"code","source":"epochs = 10\nfor epoch in range(epochs):\n    train_loss, train_auc, val_loss, val_auc, elapsed_time = train_epoch(model, train_dataloader, val_dataloader, optimizer, criterion, device, time_cat_flag = TIME_CAT_FLAG)\n    print(\"epoch - {} train_loss - {:.4f} train_auc - {:.4f} val_loss - {:.4f} val_auc - {:.4f} time={:.2f}s\".format(epoch, train_loss, train_auc, val_loss, val_auc, elapsed_time))\n#     logging.info(\"epoch - {} train_loss - {:.4f} train_auc - {:.4f} val_loss - {:.4f} val_auc - {:.4f} time={:.2f}s\".format(epoch, train_loss, train_auc, val_loss, val_auc, elapsed_time))","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-01-07T05:34:10.461339Z","start_time":"2021-01-07T05:34:10.403348Z"},"id":"p9fH59qxhmlu","trusted":false},"cell_type":"code","source":"torch.save(model.state_dict(), \"./saint_plus_model.pt\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}