{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# A self-Attentive model for Knowledge Tracing\n\n\n## update:\n\n### 2020-12-11: auc -> 0.765\n\n- add state for test dataset\n\nthanks for HDKIM wordk: https://www.kaggle.com/leadbest/sakt-with-state-updates","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-01T02:56:46.469693Z","iopub.execute_input":"2023-11-01T02:56:46.470020Z","iopub.status.idle":"2023-11-01T02:56:46.506808Z","shell.execute_reply.started":"2023-11-01T02:56:46.469989Z","shell.execute_reply":"2023-11-01T02:56:46.506008Z"},"jupyter":{"source_hidden":true,"outputs_hidden":true},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport random\nfrom tqdm import tqdm\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import train_test_split\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.utils.rnn as rnn_utils\nfrom torch.autograd import Variable\nfrom torch.utils.data import Dataset, DataLoader","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2023-11-01T02:56:46.508585Z","iopub.execute_input":"2023-11-01T02:56:46.508832Z","iopub.status.idle":"2023-11-01T02:56:48.684222Z","shell.execute_reply.started":"2023-11-01T02:56:46.508807Z","shell.execute_reply":"2023-11-01T02:56:48.683498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load data","metadata":{}},{"cell_type":"code","source":"%%time\ndtype = {'timestamp': 'int64', 'user_id': 'int32' ,'content_id': 'int16','content_type_id': 'int8','answered_correctly':'int8'}\n\ntrain_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', usecols=[1, 2, 3,4,7], dtype=dtype)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T02:56:48.685385Z","iopub.execute_input":"2023-11-01T02:56:48.685664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df[train_df.content_type_id == False]\n\n#arrange by timestamp\ntrain_df = train_df.sort_values(['timestamp'], ascending=True).reset_index(drop = True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocess","metadata":{}},{"cell_type":"code","source":"skills = train_df[\"content_id\"].unique()\nn_skill = len(skills)\nprint(\"number skills\", len(skills))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"group = train_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id').apply(lambda r: (\n            r['content_id'].values,\n            r['answered_correctly'].values))\n\ndel train_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SAKTDataset(Dataset):\n    def __init__(self, group, n_skill, max_seq=100):\n        super(SAKTDataset, self).__init__()\n        self.max_seq = max_seq\n        self.n_skill = n_skill\n        self.samples = group\n        \n#         self.user_ids = [x for x in group.index]\n        self.user_ids = []\n        for user_id in group.index:\n            q, qa = group[user_id]\n            if len(q) < 10:\n                continue\n            self.user_ids.append(user_id)\n\n    def __len__(self):\n        return len(self.user_ids)\n\n    def __getitem__(self, index):\n        user_id = self.user_ids[index]\n        q_, qa_ = self.samples[user_id]\n        seq_len = len(q_)\n\n        q = np.zeros(self.max_seq, dtype=int)\n        qa = np.zeros(self.max_seq, dtype=int)\n        if seq_len >= self.max_seq:\n            q[:] = q_[-self.max_seq:]\n            qa[:] = qa_[-self.max_seq:]\n        else:\n            q[-seq_len:] = q_\n            qa[-seq_len:] = qa_\n        \n        target_id = q[1:]\n        label = qa[1:]\n\n        x = np.zeros(self.max_seq-1, dtype=int)\n        x = q[:-1].copy()\n        x += (qa[:-1] == 1) * self.n_skill\n\n        return x, target_id, label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train, val = train_test_split(group, test_size=0.2)\n\ntrain_dataset = SAKTDataset(train, n_skill)\ntrain_dataloader = DataLoader(train_dataset, batch_size=2048, shuffle=True, num_workers=8)\ndel train\n\nval_dataset = SAKTDataset(val, n_skill)\nval_dataloader = DataLoader(val_dataset, batch_size=2048, shuffle=True, num_workers=8)\ndel val\n# print(item[0])\n# print(item[1])\n# print(item[2])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define model","metadata":{}},{"cell_type":"code","source":"class FFN(nn.Module):\n    def __init__(self, state_size=200):\n        super(FFN, self).__init__()\n        self.state_size = state_size\n\n        self.lr1 = nn.Linear(state_size, state_size)\n        self.relu = nn.ReLU()\n        self.lr2 = nn.Linear(state_size, state_size)\n        self.dropout = nn.Dropout(0.2)\n    \n    def forward(self, x):\n        x = self.lr1(x)\n        x = self.relu(x)\n        x = self.lr2(x)\n        return self.dropout(x)\n\ndef future_mask(seq_length):\n    future_mask = np.triu(np.ones((seq_length, seq_length)), k=1).astype('bool')\n    return torch.from_numpy(future_mask)\n\n\nclass SAKTModel(nn.Module):\n    def __init__(self, n_skill, max_seq=100, embed_dim=128):\n        super(SAKTModel, self).__init__()\n        self.n_skill = n_skill\n        self.embed_dim = embed_dim\n\n        self.embedding = nn.Embedding(2*n_skill+1, embed_dim)\n        self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)\n        self.e_embedding = nn.Embedding(n_skill+1, embed_dim)\n\n        self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)\n\n        self.dropout = nn.Dropout(0.2)\n        self.layer_normal = nn.LayerNorm(embed_dim) \n\n        self.ffn = FFN(embed_dim)\n        self.pred = nn.Linear(embed_dim, 1)\n    \n    def forward(self, x, question_ids):\n        device = x.device        \n        x = self.embedding(x)\n        pos_id = torch.arange(x.size(1)).unsqueeze(0).to(device)\n\n        pos_x = self.pos_embedding(pos_id)\n        x = x + pos_x\n\n        e = self.e_embedding(question_ids)\n\n        x = x.permute(1, 0, 2) # x: [bs, s_len, embed] => [s_len, bs, embed]\n        e = e.permute(1, 0, 2)\n        att_mask = future_mask(x.size(0)).to(device)\n        att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)\n        att_output = self.layer_normal(att_output + e)\n        att_output = att_output.permute(1, 0, 2) # att_output: [s_len, bs, embed] => [bs, s_len, embed]\n\n        x = self.ffn(att_output)\n        x = self.layer_normal(x + att_output)\n        x = self.pred(x)\n\n        return x.squeeze(-1), att_weight","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n# device = torch.device(\"cpu\")\n\n\nmodel = SAKTModel(n_skill, embed_dim=128)\n# optimizer = torch.optim.SGD(model.parameters(), lr=1e-3, momentum=0.99, weight_decay=0.005)\noptimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\ncriterion = nn.BCEWithLogitsLoss()\n\nmodel.to(device)\ncriterion.to(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_epoch(model, train_iterator, optim, criterion, device=\"cpu\"):\n    model.train()\n\n    train_loss = []\n    num_corrects = 0\n    num_total = 0\n    labels = []\n    outs = []\n\n    tbar = tqdm(train_iterator)\n    for item in tbar:\n        x = item[0].to(device).long()\n        target_id = item[1].to(device).long()\n        label = item[2].to(device).float()\n        target_mask = (target_id != 0)\n\n        optim.zero_grad()\n        output, atten_weight = model(x, target_id)\n        \n        output = torch.masked_select(output, target_mask)\n        label = torch.masked_select(label, target_mask)\n        \n        loss = criterion(output, label)\n        loss.backward()\n        optim.step()\n        train_loss.append(loss.item())\n        pred = (torch.sigmoid(output) >= 0.5).long()\n        \n        num_corrects += (pred == label).sum().item()\n        num_total += len(label)\n\n        labels.extend(label.view(-1).data.cpu().numpy())\n        outs.extend(output.view(-1).data.cpu().numpy())\n\n        tbar.set_description('loss - {:.4f}'.format(loss))\n\n    acc = num_corrects / num_total\n    auc = roc_auc_score(labels, outs)\n    loss = np.average(train_loss)\n\n    return loss, acc, auc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def val_epoch(model, val_iterator, criterion, device=\"cpu\"):\n    model.eval()\n\n    train_loss = []\n    num_corrects = 0\n    num_total = 0\n    labels = []\n    outs = []\n\n    tbar = tqdm(val_iterator)\n    for item in tbar:\n        x = item[0].to(device).long()\n        target_id = item[1].to(device).long()\n        label = item[2].to(device).float()\n        target_mask = (target_id != 0)\n\n        with torch.no_grad():\n            output, atten_weight = model(x, target_id)\n        \n        output = torch.masked_select(output, target_mask)\n        label = torch.masked_select(label, target_mask)\n\n        loss = criterion(output, label)\n        train_loss.append(loss.item())\n\n        pred = (torch.sigmoid(output) >= 0.5).long()\n        \n        num_corrects += (pred == label).sum().item()\n        num_total += len(label)\n\n        labels.extend(label.view(-1).data.cpu().numpy())\n        outs.extend(output.view(-1).data.cpu().numpy())\n\n        tbar.set_description('loss - {:.4f}'.format(loss))\n\n    acc = num_corrects / num_total\n    auc = roc_auc_score(labels, outs)\n    loss = np.average(train_loss)\n\n    return loss, acc, auc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 20\n\nover_fit = 0\nlast_auc = 0\nfor epoch in range(epochs):\n    train_loss, train_acc, train_auc = train_epoch(model, train_dataloader, optimizer, criterion, device)\n    print(\"epoch - {} train_loss - {:.2f} acc - {:.3f} auc - {:.3f}\".format(epoch, train_loss, train_acc, train_auc))\n    \n    val_loss, avl_acc, val_auc = val_epoch(model, val_dataloader, criterion, device)\n    print(\"epoch - {} val_loss - {:.2f} acc - {:.3f} auc - {:.3f}\".format(epoch, val_loss, avl_acc, val_auc))\n    \n    if val_auc > last_auc:\n        last_auc = val_auc\n        over_fit = 0\n    else:\n        over_fit += 1\n        \n    \n    if over_fit >= 2:\n        print(\"early stop epoch \", epoch)\n        break","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test","metadata":{"trusted":true}},{"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, samples, test_df, skills, max_seq=100):\n        super(TestDataset, self).__init__()\n        self.samples = samples\n        self.user_ids = [x for x in test_df[\"user_id\"].unique()]\n        self.test_df = test_df\n        self.skills = skills\n        self.n_skill = len(skills)\n        self.max_seq = max_seq\n\n    def __len__(self):\n        return self.test_df.shape[0]\n\n    def __getitem__(self, index):\n        test_info = self.test_df.iloc[index]\n\n        user_id = test_info[\"user_id\"]\n        target_id = test_info[\"content_id\"]\n\n        q = np.zeros(self.max_seq, dtype=int)\n        qa = np.zeros(self.max_seq, dtype=int)\n\n        if user_id in self.samples.index:\n            q_, qa_ = self.samples[user_id]\n            \n            seq_len = len(q_)\n\n            if seq_len >= self.max_seq:\n                q = q_[-self.max_seq:]\n                qa = qa_[-self.max_seq:]\n            else:\n                q[-seq_len:] = q_\n                qa[-seq_len:] = qa_          \n        \n        x = np.zeros(self.max_seq-1, dtype=int)\n        x = q[1:].copy()\n        x += (qa[1:] == 1) * self.n_skill\n        \n        questions = np.append(q[2:], [target_id])\n        \n        return x, questions","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()\niter_test = env.iter_test()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import psutil\n\nmodel.eval()\n\nprev_test_df = None\nMAX_SEQ = 100\n\nfor (test_df, sample_prediction_df) in tqdm(iter_test):\n    #HDKIM\n    if (prev_test_df is not None) & (psutil.virtual_memory().percent<90):\n        print(psutil.virtual_memory().percent)\n        prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])\n        prev_test_df = prev_test_df[prev_test_df.content_type_id == False]\n        prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id').apply(lambda r: (\n            r['content_id'].values,\n            r['answered_correctly'].values))\n        for prev_user_id in prev_group.index:\n            prev_group_content = prev_group[prev_user_id][0]\n            prev_group_ac = prev_group[prev_user_id][1]\n            if prev_user_id in group.index:\n                group[prev_user_id] = (np.append(group[prev_user_id][0],prev_group_content), \n                                       np.append(group[prev_user_id][1],prev_group_ac))\n \n            else:\n                group[prev_user_id] = (prev_group_content,prev_group_ac)\n            if len(group[prev_user_id][0])>MAX_SEQ:\n                new_group_content = group[prev_user_id][0][-MAX_SEQ:]\n                new_group_ac = group[prev_user_id][1][-MAX_SEQ:]\n                group[prev_user_id] = (new_group_content,new_group_ac)\n\n    prev_test_df = test_df.copy()\n    \n    test_df = test_df[test_df.content_type_id == False]\n    \n    test_dataset = TestDataset(group, test_df, skills)\n    test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)\n    \n    outs = []\n\n    for item in tqdm(test_dataloader):\n        x = item[0].to(device).long()\n        target_id = item[1].to(device).long()\n\n        with torch.no_grad():\n            output, att_weight = model(x, target_id)\n        \n        \n        output = torch.sigmoid(output)\n        output = output[:, -1]\n\n        # pred = (output >= 0.5).long()\n        # loss = criterion(output, label)\n\n        # val_loss.append(loss.item())\n        # num_corrects += (pred == label).sum().item()\n        # num_total += len(label)\n\n        # labels.extend(label.squeeze(-1).data.cpu().numpy())\n        outs.extend(output.view(-1).data.cpu().numpy())\n        \n    test_df['answered_correctly'] =  outs\n    \n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}