{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Self-Attentive model for Knowledge Tracing model (SAKT)\n\nThis is a fork of:\nhttps://www.kaggle.com/wangsg/a-self-attentive-model-for-knowledge-tracing from @[wangsg](https://www.kaggle.com/wangsg)\n\nWhich is an implementation of this paper:\nhttps://arxiv.org/pdf/1907.06837.pdf\n\nWith the following improvements:\n* Pytorch random fixed to be reproductible\n* Random sequence added during training\n* torch.sigmoid added/fixed to train loop\n* Training plot\n* Train/Valid simple split to save best model"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import gc, sys, os\nimport random, math\nfrom tqdm.notebook import tqdm\n\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import train_test_split\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.utils.rnn as rnn_utils\nfrom torch.autograd import Variable\nfrom torch.utils.data import Dataset, DataLoader\n\nimport seaborn as sns\nsns.set()\nDEFAULT_FIG_WIDTH = 20\nsns.set_context(\"paper\", font_scale=1.2) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Python     : ' + sys.version.split('\\n')[0])\nprint('Numpy      : ' + np.__version__)\nprint('Pandas     : ' + pd.__version__)\nprint('PyTorch    : ' + torch.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') # if IS_TPU == False else xm.xla_device()\nprint('Running on device: {}'.format(DEVICE))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(s):\n    random.seed(s)\n    os.environ['PYTHONHASHSEED'] = str(s)\n    np.random.seed(s)\n    # Torch\n    torch.manual_seed(s)\n    torch.cuda.manual_seed(s)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(s)\n\nseed = 2020\nseed_everything(seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"HOME =  \"./\"\nDATA_HOME = \"/kaggle/input/riiid-test-answer-prediction/\"\nMODEL_NAME = \"SAKT-v1\"\nMODEL_PATH = HOME + MODEL_NAME\nSTAGE = \"stage1\"\nMODEL_BEST = 'model_best.pt'\nFOLD = 1\n\nif not os.path.exists(MODEL_PATH):\n    os.makedirs(MODEL_PATH)\n    \nCONTENT_TYPE_ID = \"content_type_id\"\nCONTENT_ID = \"content_id\"\nTARGET = \"answered_correctly\"\nUSER_ID = \"user_id\"\nTASK_CONTAINER_ID = \"task_container_id\"\nTIMESTAMP = \"timestamp\" ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load data"},{"metadata":{"trusted":true},"cell_type":"code","source":"#%%time\ndtype = {TIMESTAMP: 'int64', USER_ID: 'int32', CONTENT_ID: 'int16',CONTENT_TYPE_ID: 'bool',TARGET:'int8'}\ntrain_df = pd.read_csv(DATA_HOME + 'train.csv', usecols=[1, 2, 3, 4, 7], dtype=dtype)\ntrain_df = train_df[train_df[CONTENT_TYPE_ID] == False].reset_index(drop = True)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Valid with last 100 interactions (must be improved to balance new users and be around 2.5M rows only)\n# However, valid_df includes all users' history needed for testing.\nvalid_df = train_df.groupby([USER_ID]).tail(100)\nprint(\"valid:\", valid_df.shape, \"users:\", valid_df[USER_ID].nunique())\n# Train\ntrain_df.drop(valid_df.index, inplace = True)\nprint(\"train:\", train_df.shape, \"users:\", train_df[USER_ID].nunique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[train_df[USER_ID] == 2147482216].head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_df[valid_df[USER_ID] == 115].head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preprocess"},{"metadata":{"trusted":true},"cell_type":"code","source":"skills = train_df[CONTENT_ID].unique()\nn_skill = 13523 # len(skills)\nprint(\"Number of skills\", n_skill)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Index by user_id\n#valid_df.sort_values([USER_ID, TIMESTAMP], ascending=[True, True], inplace=True) # Already sorted\nvalid_df = valid_df.reset_index(drop = True)\nvalid_group = valid_df[[USER_ID, CONTENT_ID, TARGET]].groupby(USER_ID).apply(lambda r: (r[CONTENT_ID].values, r[TARGET].values))\ndel valid_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Index by user_id\n#train_df.sort_values([USER_ID, TIMESTAMP], ascending=[True, True], inplace=True) # Already sorted\ntrain_df = train_df.reset_index(drop = True)\ntrain_group = train_df[[USER_ID, CONTENT_ID, TARGET]].groupby(USER_ID).apply(lambda r: (r[CONTENT_ID].values, r[TARGET].values))\ndel train_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SAKTDataset(Dataset):\n    def __init__(self, group, n_skill, subset=\"train\", max_seq=100):\n        super(SAKTDataset, self).__init__()\n        self.max_seq = max_seq\n        self.n_skill = n_skill # 13523\n        self.samples = group\n        self.subset = subset\n        \n        # self.user_ids = [x for x in group.index]\n        self.user_ids = []\n        for user_id in group.index:\n            q, qa = group[user_id]\n            if len(q) < 10: # 10 interactions minimum\n                continue\n            self.user_ids.append(user_id) # user_ids indexes\n\n    def __len__(self):\n        return len(self.user_ids)\n\n    def __getitem__(self, index):\n        user_id = self.user_ids[index] # Pick a user\n        q_, qa_ = self.samples[user_id] # Pick full sequence for user\n        seq_len = len(q_)\n\n        q = np.zeros(self.max_seq, dtype=int)\n        qa = np.zeros(self.max_seq, dtype=int)\n\n        if seq_len >= self.max_seq:\n            if self.subset == \"train\":\n                if seq_len > self.max_seq:\n                    random_start_index = np.random.randint(seq_len - self.max_seq)\n                    q[:] = q_[random_start_index:random_start_index + self.max_seq] # Pick 100 questions from a random index\n                    qa[:] = qa_[random_start_index:random_start_index + self.max_seq] # Pick 100 answers from a random index\n                else:\n                    q[:] = q_[-self.max_seq:]\n                    qa[:] = qa_[-self.max_seq:]\n            else:\n                q[:] = q_[-self.max_seq:] # Pick last 100 questions\n                qa[:] = qa_[-self.max_seq:] # Pick last 100 answers\n        else:\n            q[-seq_len:] = q_ # Pick last N question with zero padding\n            qa[-seq_len:] = qa_ # Pick last N answers with zero padding        \n                \n        target_id = q[1:] # Ignore first item 1 to 99\n        label = qa[1:] # Ignore first item 1 to 99\n\n        # x = np.zeros(self.max_seq-1, dtype=int)\n        x = q[:-1].copy() # 0 to 98\n        x += (qa[:-1] == 1) * self.n_skill # y = et + rt x E\n\n        return x, target_id, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class FFN(nn.Module):\n    def __init__(self, state_size=200):\n        super(FFN, self).__init__()\n        self.state_size = state_size\n\n        self.lr1 = nn.Linear(state_size, state_size)\n        self.relu = nn.ReLU()\n        self.lr2 = nn.Linear(state_size, state_size)\n        self.dropout = nn.Dropout(0.2)\n    \n    def forward(self, x):\n        x = self.lr1(x)\n        x = self.relu(x)\n        x = self.lr2(x)\n        return self.dropout(x)\n\ndef future_mask(seq_length):\n    future_mask = np.triu(np.ones((seq_length, seq_length)), k=1).astype('bool')\n    return torch.from_numpy(future_mask)\n\n\nclass SAKTModel(nn.Module):\n    def __init__(self, n_skill, max_seq=100, embed_dim=128):\n        super(SAKTModel, self).__init__()\n        self.n_skill = n_skill\n        self.embed_dim = embed_dim\n\n        self.embedding = nn.Embedding(2*n_skill+1, embed_dim)\n        self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)\n        self.e_embedding = nn.Embedding(n_skill+1, embed_dim)\n\n        self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)\n\n        self.dropout = nn.Dropout(0.2)\n        self.layer_normal = nn.LayerNorm(embed_dim) \n\n        self.ffn = FFN(embed_dim)\n        self.pred = nn.Linear(embed_dim, 1)\n    \n    def forward(self, x, question_ids):\n        device = x.device        \n        x = self.embedding(x)\n        pos_id = torch.arange(x.size(1)).unsqueeze(0).to(device)\n\n        pos_x = self.pos_embedding(pos_id)\n        x = x + pos_x\n\n        e = self.e_embedding(question_ids)\n\n        x = x.permute(1, 0, 2) # x: [bs, s_len, embed] => [s_len, bs, embed]\n        e = e.permute(1, 0, 2)\n        att_mask = future_mask(x.size(0)).to(device)\n        att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)\n        att_output = self.layer_normal(att_output + e)\n        att_output = att_output.permute(1, 0, 2) # att_output: [s_len, bs, embed] => [bs, s_len, embed]\n\n        x = self.ffn(att_output)\n        x = self.layer_normal(x + att_output)\n        x = self.pred(x)\n\n        return x.squeeze(-1), att_weight","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_epoch(model, train_iterator, optim, criterion, device=\"cpu\"):\n    model.train()\n\n    train_loss = []\n    num_corrects = 0\n    num_total = 0\n    labels = []\n    outs = []\n\n    # tbar = tqdm(train_iterator)\n    for item in train_iterator: #tbar:\n        x = item[0].to(device).long()\n        target_id = item[1].to(device).long()\n        label = item[2].to(device).float()\n\n        optim.zero_grad()\n        output, atten_weight = model(x, target_id)\n        loss = criterion(output, label)\n        loss.backward()\n        optim.step()\n        train_loss.append(loss.item())\n\n        output = output[:, -1]\n        label = label[:, -1] \n        pred = (torch.sigmoid(output) >= 0.5).long()\n        \n        num_corrects += (pred == label).sum().item()\n        num_total += len(label)\n\n        labels.extend(label.view(-1).data.cpu().numpy())\n        #outs.extend(output.view(-1).data.cpu().numpy())\n        outs.extend(torch.sigmoid(output).view(-1).data.cpu().numpy())\n\n        # tbar.set_description('loss - {:.4f}'.format(loss))\n    \n    acc = num_corrects / num_total\n    auc = roc_auc_score(labels, outs)\n    loss = np.mean(train_loss)\n\n    return loss, acc, auc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def valid_epoch(model, valid_iterator, criterion, device=\"cpu\"):\n    model.eval()\n\n    valid_loss = []\n    num_corrects = 0\n    num_total = 0\n    labels = []\n    outs = []\n\n    #tbar = tqdm(valid_iterator)\n    for item in valid_iterator: # tbar:\n        x = item[0].to(device).long()\n        target_id = item[1].to(device).long()\n        label = item[2].to(device).float()\n\n        with torch.no_grad():\n            output, atten_weight = model(x, target_id)\n        loss = criterion(output, label)\n        valid_loss.append(loss.item())\n\n        output = output[:, -1] # (BS, 1)\n        label = label[:, -1] \n        pred = (torch.sigmoid(output) >= 0.5).long()\n        \n        num_corrects += (pred == label).sum().item()\n        num_total += len(label)\n\n        labels.extend(label.view(-1).data.cpu().numpy())\n        #outs.extend(output.view(-1).data.cpu().numpy())\n        outs.extend(torch.sigmoid(output).view(-1).data.cpu().numpy())\n\n        #tbar.set_description('loss - {:.4f}'.format(loss))\n\n    acc = num_corrects / num_total\n    auc = roc_auc_score(labels, outs)\n    loss = np.mean(valid_loss)\n\n    return loss, acc, auc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class conf:\n    METRIC_ = \"max\"\n    WORKERS = 4 # 0\n    BATCH_SIZE = 2048\n    lr = 1e-3\n    D_MODEL = 128\n\n    if torch.cuda.is_available():\n        map_location=lambda storage, loc: storage.cuda()\n    else:\n        map_location='cpu'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = SAKTDataset(train_group, n_skill, subset=\"train\")\ntrain_dataloader = DataLoader(train_dataset, batch_size=conf.BATCH_SIZE, shuffle=True, num_workers=conf.WORKERS)\n\nvalid_dataset = SAKTDataset(valid_group, n_skill, subset=\"valid\")\nvalid_dataloader = DataLoader(valid_dataset, batch_size=conf.BATCH_SIZE, shuffle=False, num_workers=conf.WORKERS)\n\nitem = train_dataset.__getitem__(5)\n\nprint(\"x\", len(item[0]), item[0])\nprint(\"target_id\", len(item[1]), item[1])\nprint(\"label\", len(item[2]), item[2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = DEVICE\n\nmodel = SAKTModel(n_skill, embed_dim=conf.D_MODEL)\noptimizer = torch.optim.Adam(model.parameters(), lr=conf.lr)\ncriterion = nn.BCEWithLogitsLoss()\n\nmodel.to(device)\ncriterion.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 48\nauc_max = -np.inf\nhistory = []\n\nsnapshot_path = \"%s/fold%d/%s/snapshots\" % (MODEL_PATH, FOLD, STAGE)\nif not os.path.exists(snapshot_path):\n    os.makedirs(snapshot_path)\n\nprint(\"Stage:\", STAGE, \"fold:\", FOLD, \"on:\", DEVICE, \"workers:\", conf.WORKERS, \"batch size:\", conf.BATCH_SIZE, \"metric_:\", conf.METRIC_, \n      \"train dataset:\", len(train_dataset), \"valid dataset:\", len(valid_dataset))\n\nfor epoch in range(1, epochs+1):\n    train_loss, train_acc, train_auc = train_epoch(model, train_dataloader, optimizer, criterion, device)\n    print(\"\\nEpoch#{}, train_loss - {:.2f} acc - {:.4f} auc - {:.4f}\".format(epoch, train_loss, train_acc, train_auc))\n    valid_loss, valid_acc, valid_auc = valid_epoch(model, valid_dataloader, criterion, device)\n    print(\"Epoch#{}, valid_loss - {:.2f} acc - {:.4f} auc - {:.4f}\".format(epoch, valid_loss, valid_acc, valid_auc))\n    lr = optimizer.param_groups[0]['lr']\n    history.append({\"epoch\":epoch, \"lr\": lr, **{\"train_auc\": train_auc, \"train_acc\": train_acc}, **{\"valid_auc\": valid_auc, \"valid_acc\": valid_acc}})\n    if valid_auc > auc_max:\n        print(\"Epoch#%s, valid loss %.4f, Metric loss improved from %.4f to %.4f, saving model ...\" % (epoch, valid_loss, auc_max, valid_auc))\n        auc_max = valid_auc\n        torch.save(model.state_dict(), os.path.join(snapshot_path, MODEL_BEST))\n\nif history:\n    metric = \"auc\"\n    # Plot training history\n    history_pd = pd.DataFrame(history[1:]).set_index(\"epoch\")\n    train_history_pd = history_pd[[c for c in history_pd.columns if \"train_\" in c]]\n    valid_history_pd = history_pd[[c for c in history_pd.columns if \"valid_\" in c]]\n    lr_history_pd = history_pd[[c for c in history_pd.columns if \"lr\" in c]]\n    fig, ax = plt.subplots(1,2, figsize=(DEFAULT_FIG_WIDTH, 6))\n    t_epoch = train_history_pd[\"train_%s\" % metric].argmin() if conf.METRIC_ == \"min\" else train_history_pd[\"train_%s\" % metric].argmax()\n    v_epoch = valid_history_pd[\"valid_%s\" % metric].argmin() if conf.METRIC_ == \"min\" else valid_history_pd[\"valid_%s\" % metric].argmax()\n    d = train_history_pd.plot(kind=\"line\", ax=ax[0], title=\"Epoch: %d, Train: %.3f\" % (t_epoch, train_history_pd.iloc[t_epoch,:][\"train_%s\" % metric]))\n    d = lr_history_pd.plot(kind=\"line\", ax=ax[0], secondary_y=True)\n    d = valid_history_pd.plot(kind=\"line\", ax=ax[1], title=\"Epoch: %d, Valid: %.3f\" % (v_epoch, valid_history_pd.iloc[v_epoch,:][\"valid_%s\" % metric]))\n    d = lr_history_pd.plot(kind=\"line\", ax=ax[1], secondary_y=True)\n    plt.savefig(\"%s/train.png\" % snapshot_path, bbox_inches='tight')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## Test"},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, samples, test_df, skills, max_seq=100):\n        super(TestDataset, self).__init__()\n        self.samples = samples\n        self.user_ids = [x for x in test_df[\"user_id\"].unique()]\n        self.test_df = test_df\n        self.skills = skills\n        self.n_skill = len(skills)\n        self.max_seq = max_seq\n\n    def __len__(self):\n        return self.test_df.shape[0]\n\n    def __getitem__(self, index):\n        test_info = self.test_df.iloc[index]\n\n        user_id = test_info[\"user_id\"]\n        target_id = test_info[\"content_id\"]\n\n        q = np.zeros(self.max_seq, dtype=int)\n        qa = np.zeros(self.max_seq, dtype=int)\n\n        if user_id in self.samples.index:\n            q_, qa_ = self.samples[user_id]\n            \n            seq_len = len(q_)\n\n            if seq_len >= self.max_seq:\n                q = q_[-self.max_seq:]\n                qa = qa_[-self.max_seq:]\n            else:\n                q[-seq_len:] = q_\n                qa[-seq_len:] = qa_          \n        \n        x = np.zeros(self.max_seq-1, dtype=int)\n        x = q[1:].copy()\n        x += (qa[1:] == 1) * self.n_skill\n        \n        questions = np.append(q[2:], [target_id])\n        \n        return x, questions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reload model with best weights\nmodel = SAKTModel(n_skill, embed_dim=conf.D_MODEL)\nresume_path = os.path.join(snapshot_path, MODEL_BEST)\nif os.path.exists(resume_path):\n    model.load_state_dict(torch.load(resume_path, map_location=conf.map_location))\n    print(\"Resuming, model weights loaded: %s\" % resume_path)\nmodel.to(device)\n_ = model.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# It does not take into account the new history in each test_df\nfor (test_df, sample_prediction_df) in iter_test:\n    test_df = test_df[test_df.content_type_id == False]\n    \n    test_dataset = TestDataset(valid_group, test_df, skills)\n    test_dataloader = DataLoader(test_dataset, batch_size=conf.BATCH_SIZE, shuffle=False, drop_last=False)\n    \n    outs = []\n\n    for item in test_dataloader:\n        x = item[0].to(device).long()\n        target_id = item[1].to(device).long()\n\n        with torch.no_grad():\n            output, _ = model(x, target_id)\n                \n        output = torch.sigmoid(output)\n        output = output[:, -1]\n\n        outs.extend(output.view(-1).data.cpu().numpy())\n        \n    test_df['answered_correctly'] = outs\n    \n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}