{"cells":[{"metadata":{"_uuid":"e9ba902d-5dab-4613-bc90-fb651545d2b9","_cell_guid":"b84f37e1-0052-498c-a8b0-170497f5c773","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n \nimport psutil\nfrom collections import defaultdict\nimport datatable as dt\nimport lightgbm as lgb\nfrom matplotlib import pyplot as plt\nimport riiideducation\nimport random\nfrom sklearn.metrics import roc_auc_score\nimport gc\nfrom torch.utils.data import Dataset, DataLoader\nimport torch\nimport torch.nn as nn\nimport torch.nn.utils.rnn as rnn_utils\nfrom torch.autograd import Variable\nfrom torch.utils.data import Dataset, DataLoader\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"SAKT part learned from https://www.kaggle.com/mpware/sakt-fork "},{"metadata":{"_uuid":"e515f1b9-0fd2-4ce3-b5cd-23b2eca95a67","_cell_guid":"02885be4-1662-43db-9a43-8ca1d129dd94","trusted":true},"cell_type":"code","source":"train=pd.read_csv('../input/riiid-test-answer-prediction/train.csv',nrows=1000000)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"86365db4-6ce7-4bf2-8699-e0fa803c7ac9","_cell_guid":"d7f2847f-1522-465c-a94f-1a1203bcb604","trusted":true},"cell_type":"code","source":"##calculate the recency of a student\ntrain['lagtime'] = train.groupby('user_id')['timestamp'].shift()\ntrain['lagtime']=train['timestamp']-train['lagtime']\ntrain['lagtime'].fillna(0, inplace=True)\ntrain.lagtime=train.lagtime.astype('int32')\n#calculate mean time of prior questions\nprior_time_agg = train.groupby('user_id')['prior_question_elapsed_time'].agg(['mean','sum'])\ntrain['prior_time_mean_user'] = train['user_id'].map(prior_time_agg['mean'])\n#train.prior_time_mean_user=train.prior_time_mean_user.astype('int64')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"27c4c7ce-1755-4a98-b15f-314118555255","_cell_guid":"9cd2ab8c-8847-40be-bb98-9bcc78188799","trusted":true},"cell_type":"code","source":"#about prior question answered correctness\nprior_answer_correctness=train.groupby('prior_question_had_explanation')['answered_correctly'].agg(['sum'])\ntrain['prior_answer_correctness']=train['prior_question_had_explanation'].map(prior_answer_correctness['sum'])\ntrain.prior_answer_correctness=train.prior_answer_correctness.astype('float16')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"79b606a2-70f6-4d74-8f0d-9bfdb11de64d","_cell_guid":"a65c6a8c-dc39-40fa-a210-d7c82d2ac4e2","trusted":true},"cell_type":"code","source":"correct_agg = train[train['answered_correctly']==1].groupby('user_id').answered_correctly.agg(['sum'])\ntrain['correct_count_user'] = train['user_id'].map(correct_agg['sum'])\nfalse_agg = train[train['answered_correctly']==0].groupby('user_id').answered_correctly.agg(['count'])\ntrain['false_count_user'] = train['user_id'].map(false_agg['count'])\nnull_agg = train[train['answered_correctly']==-1].groupby('user_id').answered_correctly.agg(['count'])\ntrain['null_count_user'] = train['user_id'].map(null_agg['count'])\n#correctness\ntrain['lag_correctness'] = train.groupby('user_id')[\"answered_correctly\"].shift()\ncum = train.groupby('user_id')['lag_correctness'].agg(['cumsum', 'cumcount'])\ntrain['user_correctness'] = cum['cumsum'] / cum['cumcount']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9ab96675-3049-4fe1-832e-a884059e0b50","_cell_guid":"bcd29acf-c90b-434a-9c43-7c4281841226","trusted":true},"cell_type":"code","source":"#about expainled questions\nexplain_agg_user= train[train['prior_question_had_explanation']==True].groupby('user_id').prior_question_had_explanation.agg(['count'])\ntrain['explained_count_user'] = train['user_id'].map(explain_agg_user['count'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"451dec2c-6f29-490c-995e-21c3c90e14d8","_cell_guid":"9823834b-7053-457f-b889-294386630cec","trusted":true},"cell_type":"code","source":"##grouped by user\n#about time\n#train[\"correct_count_user\"]=train.groupby(train[\"user_id\"])[\"answered_correctly\"].count(1)\n#train[\"false_count_user\"]=train[\"answered_correctly\"].groupby(train[\"user_id\"]).count(0)\n#train[\"null_count_user\"]=train[\"answered_correctly\"].groupby(train[\"user_id\"]).count(-1)\ntrain['prior_time_sum_user'] = train['user_id'].map(prior_time_agg['sum'])\ntrain.prior_time_sum_user=train.prior_time_sum_user.astype('int32')\n#train[\"prior_mean_correct_user\"]=train[\"prior_time_sum_user\"]/train[\"correct_count_user\"]\n#train[\"prior_mean_false_user\"]=train[\"prior_time_sum_user\"]/train[\"false_count_user\"]\n#train[\"prior_mean_null_user\"]=train[\"prior_time_sum_user\"]/train[\"null_count_user\"]v","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"14f847c8-f138-45d0-8e1c-a6889dbe80ed","_cell_guid":"87d3468e-ae67-4417-b7bc-0c5f1f4060cc","trusted":true},"cell_type":"code","source":"#about content\ncontent_agg = train.groupby('content_id')['answered_correctly'].agg(['sum', 'count','var'])\ntrain['content_correctness'] = train['content_id'].map(content_agg['sum'] / content_agg['count'])\ntrain.content_correctness=train.content_correctness.astype('float16')\ntrain[\"attempt_no\"] = 1\ntrain.attempt_no=train.attempt_no.astype('int8')\ntrain[\"attempt_no\"] = train[[\"user_id\",\"content_id\",'attempt_no']].groupby([\"user_id\",\"content_id\"])[\"attempt_no\"].cumsum()\nexplain_agg_contain= train[train['prior_question_had_explanation']==True].groupby('content_id').prior_question_had_explanation.agg(['count'])\ntrain['explained_count_content'] = train['content_id'].map(explain_agg_contain['count'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"84a33bd9-012c-4d6e-9bbd-bc1f37290dca","_cell_guid":"b5fc0341-ad5f-4d4e-83f6-c4348d970ccc","trusted":true},"cell_type":"code","source":"##lectures watched & lecture_vs_questions\nlec_cum = train.groupby('user_id')['content_type_id'].agg(['cumsum', 'cumcount'])\ntrain['user_lecture_cumsum'] = lec_cum['cumsum'] \ntrain['user_lecture_cumavg'] = lec_cum['cumsum'] / lec_cum['cumcount']\ntrain.user_lecture_cumavg=train.user_lecture_cumavg.astype('float16')\ntrain.user_lecture_cumsum=train.user_lecture_cumsum.astype('int8')\nuser_lecture_agg = train.groupby('user_id')['content_type_id'].agg(['sum', 'count'])\ntrain['user_lecture_sum'] = train['user_id'].map(user_lecture_agg['sum'])\ntrain['user_lecture_avg'] = train['user_id'].map(user_lecture_agg['sum'] / user_lecture_agg['count'])\ntrain.user_lecture_avg=train.user_lecture_avg.astype('float16')\ntrain.user_lecture_sum=train.user_lecture_sum.astype('int8')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"788cfcf7-3ffe-47b0-8526-d883352e5537","_cell_guid":"6e25d804-b9e6-4c32-9f1d-6950d3753d56","trusted":true},"cell_type":"code","source":"train.prior_question_had_explanation=train.prior_question_had_explanation.astype('bool')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cf74364e-8aba-4a56-82a6-279111dd5cbf","_cell_guid":"ccf5040c-5430-4c48-8070-73dbc5ed8250","trusted":true},"cell_type":"code","source":"questions_df = pd.read_csv(\n '../input/riiid-test-answer-prediction/questions.csv', \n usecols=[0,1,3,4],\n dtype={'question_id': 'int16','bundle_id': 'int16', 'part': 'int8','tags': 'str'},\n nrows=200000000\n)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3904fb24-1c94-4234-98cb-3a71c1df08d3","_cell_guid":"d4cb58c6-129b-4ce2-bb6c-92e0ff425fe2","trusted":true},"cell_type":"code","source":"### TSVD decomposition\nfrom gensim import corpora,similarities,models\nfrom tqdm.notebook import tqdm\nimport warnings\nwarnings.filterwarnings(action='ignore',category=UserWarning,module='gensim')\n\nquestions_df['tags'].fillna(\"-1\",inplace=True)\n\nclass getwords(object):\n def __iter__(self):\n for index,doc in enumerate(questions_df['tags']):\n if index%5000000==0:\n print(index)\n yield doc.split(' ')\ncorp = getwords()\ndictionary=corpora.Dictionary(corp)\nclass doc2b(object):\n def __init__(self):\n print('MyCorpus')\n def __iter__(self):\n for index,doc in enumerate(questions_df['tags']):\n if index%5000000==0:\n print(index)\n yield dictionary.doc2bow(doc.split(' '))\ncorpus = doc2b()\ntfidf_model = models.TfidfModel(corpus, id2word=dictionary)\n\ntfidf_model.save(\"./tfidf.model\")\ndictionary.save(\"./dictionary.model\")\ncorpus_tfidf = tfidf_model[corpus]\nclass tfidfv(object):\n def __init__(self):\n print('MyCorpus')\n def __iter__(self):\n for index,doc in enumerate(corpus_tfidf):\n if index%1000000==0:\n print(index)\n yield doc\ncorpus3 = tfidfv()\nlsi_model = models.LsiModel(corpus3, id2word=dictionary,chunksize=2500000,num_topics=50)\nlsi_model.save(\"./lsi_all_query.lsi\")\n\ndef get_arg_max(single_list):\n max_index=0\n max_num=single_list[0][1]\n for index in range(len(single_list)-1):\n if max_num [s_len, bs, embed]\n e = e.permute(1, 0, 2)\n att_mask = future_mask(x.size(0)).to(device)\n att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)\n att_output = self.layer_normal(att_output + e)\n att_output = att_output.permute(1, 0, 2) # att_output: [s_len, bs, embed] => [bs, s_len, embed]\n\n x = self.ffn(att_output)\n x = self.layer_normal(x + att_output)\n x = self.pred(x)\n\n return x.squeeze(-1), att_weight","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SAKTDataset(Dataset):\n def __init__(self, group, n_skill, subset=\"train\", max_seq=100):\n super(SAKTDataset, self).__init__()\n self.max_seq = max_seq\n self.n_skill = n_skill \n self.samples = group\n self.subset = subset\n \n # self.user_ids = [x for x in group.index]\n self.user_ids = []\n for user_id in group.index:\n q, qa = group[user_id]\n if len(q) < 10: # 10 interactions minimum\n continue\n self.user_ids.append(user_id) # user_ids indexes\n\n def __len__(self):\n return len(self.user_ids)\n\n def __getitem__(self, index):\n user_id = self.user_ids[index] # Pick a user\n q_, qa_ = self.samples[user_id] # Pick full sequence for user\n seq_len = len(q_)\n\n q = np.zeros(self.max_seq, dtype=int)\n qa = np.zeros(self.max_seq, dtype=int)\n\n if seq_len >= self.max_seq:\n if self.subset == \"train\":\n if seq_len > self.max_seq:\n random_start_index = np.random.randint(seq_len - self.max_seq)\n q[:] = q_[random_start_index:random_start_index + self.max_seq] # Pick 100 questions from a random index\n qa[:] = qa_[random_start_index:random_start_index + self.max_seq] # Pick 100 answers from a random index\n else:\n q[:] = q_[-self.max_seq:]\n qa[:] = qa_[-self.max_seq:]\n else:\n q[:] = q_[-self.max_seq:] # Pick last 100 questions\n qa[:] = qa_[-self.max_seq:] # Pick last 100 answers\n else:\n q[-seq_len:] = q_ # Pick last N question with zero padding\n qa[-seq_len:] = qa_ # Pick last N answers with zero padding \n \n target_id = q[1:] # Ignore first item 1 to 99\n label = qa[1:] # Ignore first item 1 to 99\n\n # x = np.zeros(self.max_seq-1, dtype=int)\n x = q[:-1].copy() # 0 to 98\n x += (qa[:-1] == 1) * self.n_skill # y = et + rt x E\n\n return x, target_id, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_epoch(model_sakt, train_iterator, optim, criterion, device=\"cpu\"):\n model_sakt.train()\n\n train_loss = []\n num_corrects = 0\n num_total = 0\n labels = []\n outs = []\n\n # tbar = tqdm(train_iterator)\n for item in train_iterator: #tbar:\n x = item[0].to(device).long()\n target_id = item[1].to(device).long()\n label = item[2].to(device).float()\n\n optim.zero_grad()\n output, atten_weight = model_sakt(x, target_id)\n loss = criterion(output, label)\n loss.backward()\n optim.step()\n train_loss.append(loss.item())\n\n output = output[:, -1]\n label = label[:, -1] \n pred = (torch.sigmoid(output) >= 0.5).long()\n \n num_corrects += (pred == label).sum().item()\n num_total += len(label)\n\n labels.extend(label.view(-1).data.cpu().numpy())\n #outs.extend(output.view(-1).data.cpu().numpy())\n outs.extend(torch.sigmoid(output).view(-1).data.cpu().numpy())\n\n # tbar.set_description('loss - {:.4f}'.format(loss))\n \n acc = num_corrects / num_total\n auc = roc_auc_score(labels, outs)\n loss = np.mean(train_loss)\n\n return loss, acc, auc\ndef valid_epoch(model_sakt, valid_iterator, criterion, device=\"cpu\"):\n model_sakt.eval()\n\n valid_loss = []\n num_corrects = 0\n num_total = 0\n labels = []\n outs = []\n\n #tbar = tqdm(valid_iterator)\n for item in valid_iterator: # tbar:\n x = item[0].to(device).long()\n target_id = item[1].to(device).long()\n label = item[2].to(device).float()\n\n with torch.no_grad():\n output, atten_weight = model_sakt(x, target_id)\n loss = criterion(output, label)\n valid_loss.append(loss.item())\n\n output = output[:, -1] # (BS, 1)\n label = label[:, -1] \n pred = (torch.sigmoid(output) >= 0.5).long()\n \n num_corrects += (pred == label).sum().item()\n num_total += len(label)\n\n labels.extend(label.view(-1).data.cpu().numpy())\n #outs.extend(output.view(-1).data.cpu().numpy())\n outs.extend(torch.sigmoid(output).view(-1).data.cpu().numpy())\n\n #tbar.set_description('loss - {:.4f}'.format(loss))\n\n acc = num_corrects / num_total\n auc = roc_auc_score(labels, outs)\n loss = np.mean(valid_loss)\n\n return loss, acc, auc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"train_dataset = SAKTDataset(train_group, n_skill, subset=\"train\")\ntrain_dataloader = DataLoader(train_dataset, batch_size=conf.BATCH_SIZE, shuffle=True, num_workers=conf.WORKERS)\n\nvalid_dataset = SAKTDataset(valid_group, n_skill, subset=\"valid\")\nvalid_dataloader = DataLoader(valid_dataset, batch_size=conf.BATCH_SIZE, shuffle=False, num_workers=conf.WORKERS)\nitem = train_dataset.__getitem__(5)\n#print(\"x\", len(item[0]), item[0])\n#print(\"target_id\", len(item[1]), item[1])\n#print(\"label\", len(item[2]), item[2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = DEVICE\n\nmodel_sakt = SAKTModel(n_skill, embed_dim=conf.D_MODEL)\noptimizer = torch.optim.Adam(model_sakt.parameters(), lr=conf.lr)\ncriterion = nn.BCEWithLogitsLoss()\n\nmodel_sakt.to(device)\ncriterion.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 20\nauc_max = -np.inf\nhistory = []\n\nsnapshot_path = \"%s/fold%d/%s/snapshots\" % (MODEL_PATH, FOLD, STAGE)\nif not os.path.exists(snapshot_path):\n os.makedirs(snapshot_path)\n\nprint(\"Stage:\", STAGE, \"fold:\", FOLD, \"on:\", DEVICE, \"workers:\", conf.WORKERS, \"batch size:\", conf.BATCH_SIZE, \"metric_:\", conf.METRIC_, \n \"train dataset:\", len(train_dataset), \"valid dataset:\", len(valid_dataset))\n\nfor epoch in range(1, epochs+1):\n train_loss, train_acc, train_auc = train_epoch(model_sakt, train_dataloader, optimizer, criterion, device)\n print(\"\\nEpoch#{}, train_loss - {:.2f} acc - {:.4f} auc - {:.4f}\".format(epoch, train_loss, train_acc, train_auc))\n valid_loss, valid_acc, valid_auc = valid_epoch(model_sakt, valid_dataloader, criterion, device)\n print(\"Epoch#{}, valid_loss - {:.2f} acc - {:.4f} auc - {:.4f}\".format(epoch, valid_loss, valid_acc, valid_auc))\n lr = optimizer.param_groups[0]['lr']\n history.append({\"epoch\":epoch, \"lr\": lr, **{\"train_auc\": train_auc, \"train_acc\": train_acc}, **{\"valid_auc\": valid_auc, \"valid_acc\": valid_acc}})\n if valid_auc > auc_max:\n print(\"Epoch#%s, valid loss %.4f, Metric loss improved from %.4f to %.4f, saving model ...\" % (epoch, valid_loss, auc_max, valid_auc))\n auc_max = valid_auc\n torch.save(model_sakt.state_dict(), os.path.join(snapshot_path, MODEL_BEST))\n\nif history:\n metric = \"auc\"\n # Plot training history\n history_pd = pd.DataFrame(history[1:]).set_index(\"epoch\")\n train_history_pd = history_pd[[c for c in history_pd.columns if \"train_\" in c]]\n valid_history_pd = history_pd[[c for c in history_pd.columns if \"valid_\" in c]]\n lr_history_pd = history_pd[[c for c in history_pd.columns if \"lr\" in c]]\n fig, ax = plt.subplots(1,2, figsize=(DEFAULT_FIG_WIDTH, 6))\n t_epoch = train_history_pd[\"train_%s\" % metric].argmin() if conf.METRIC_ == \"min\" else train_history_pd[\"train_%s\" % metric].argmax()\n v_epoch = valid_history_pd[\"valid_%s\" % metric].argmin() if conf.METRIC_ == \"min\" else valid_history_pd[\"valid_%s\" % metric].argmax()\n d = train_history_pd.plot(kind=\"line\", ax=ax[0], title=\"Epoch: %d, Train: %.3f\" % (t_epoch, train_history_pd.iloc[t_epoch,:][\"train_%s\" % metric]))\n d = lr_history_pd.plot(kind=\"line\", ax=ax[0], secondary_y=True)\n d = valid_history_pd.plot(kind=\"line\", ax=ax[1], title=\"Epoch: %d, Valid: %.3f\" % (v_epoch, valid_history_pd.iloc[v_epoch,:][\"valid_%s\" % metric]))\n d = lr_history_pd.plot(kind=\"line\", ax=ax[1], secondary_y=True)\n plt.savefig(\"%s/train.png\" % snapshot_path, bbox_inches='tight')\n plt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}