{"cells":[{"metadata":{},"cell_type":"markdown","source":"[KDD'2020: Context-Aware Attentive Knowledge Tracing (Pytorch implementation for AKT).](https://github.com/arghosh/AKT)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import sys\nimport os\nimport os.path\nimport glob\nimport logging\nimport argparse\nimport numpy as np\nimport torch\nimport numpy as np\nimport torch\nimport math\nfrom sklearn import metrics\nimport torch\nfrom torch import nn\nfrom torch.nn.init import xavier_uniform_\nfrom torch.nn.init import constant_\nfrom torch.nn.init import xavier_normal_\nimport math\nimport torch.nn.functional as F\nfrom enum import IntEnum\nimport numpy as np\n\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass Dim(IntEnum):\n    batch = 0\n    seq = 1\n    feature = 2\n\n\nclass AKT(nn.Module):\n    def __init__(self, n_question, n_pid, d_model, n_blocks,\n                 kq_same, dropout, model_type, final_fc_dim=512, n_heads=8, d_ff=2048,  l2=1e-5, separate_qa=False):\n        super().__init__()\n        \"\"\"\n        Input:\n            d_model: dimension of attention block\n            final_fc_dim: dimension of final fully connected net before prediction\n            n_heads: number of heads in multi-headed attention\n            d_ff : dimension for fully conntected net inside the basic block\n        \"\"\"\n        self.n_question = n_question\n        self.dropout = dropout\n        self.kq_same = kq_same\n        self.n_pid = n_pid\n        self.l2 = l2\n        self.model_type = model_type\n        self.separate_qa = separate_qa\n        embed_l = d_model\n        if self.n_pid > 0:\n            self.difficult_param = nn.Embedding(self.n_pid+1, 1)\n            self.q_embed_diff = nn.Embedding(self.n_question+1, embed_l)\n            self.qa_embed_diff = nn.Embedding(2 * self.n_question + 1, embed_l)\n        # n_question+1 ,d_model\n        self.q_embed = nn.Embedding(self.n_question+1, embed_l)\n        if self.separate_qa:\n            self.qa_embed = nn.Embedding(2*self.n_question+1, embed_l)\n        else:\n            self.qa_embed = nn.Embedding(2, embed_l)\n        # Architecture Object. It contains stack of attention block\n        self.model = Architecture(n_question=n_question, n_blocks=n_blocks, n_heads=n_heads, dropout=dropout,\n                                    d_model=d_model, d_feature=d_model / n_heads, d_ff=d_ff,  kq_same=self.kq_same, model_type=self.model_type)\n\n        self.out = nn.Sequential(\n            nn.Linear(d_model + embed_l,\n                      final_fc_dim), nn.ReLU(), nn.Dropout(self.dropout),\n            nn.Linear(final_fc_dim, 256), nn.ReLU(\n            ), nn.Dropout(self.dropout),\n            nn.Linear(256, 1)\n        )\n        self.reset()\n\n    def reset(self):\n        for p in self.parameters():\n            if p.size(0) == self.n_pid+1 and self.n_pid > 0:\n                torch.nn.init.constant_(p, 0.)\n\n    def forward(self, q_data, qa_data, target, pid_data=None):\n        # Batch First\n        q_embed_data = self.q_embed(q_data)  # BS, seqlen,  d_model# c_ct\n        if self.separate_qa:\n            # BS, seqlen, d_model #f_(ct,rt)\n            qa_embed_data = self.qa_embed(qa_data)\n        else:\n            qa_data = (qa_data-q_data)//self.n_question  # rt\n            # BS, seqlen, d_model # c_ct+ g_rt =e_(ct,rt)\n            qa_embed_data = self.qa_embed(qa_data)+q_embed_data\n\n        if self.n_pid > 0:\n            q_embed_diff_data = self.q_embed_diff(q_data)  # d_ct\n            pid_embed_data = self.difficult_param(pid_data)  # uq\n            q_embed_data = q_embed_data + pid_embed_data * \\\n                q_embed_diff_data  # uq *d_ct + c_ct\n            qa_embed_diff_data = self.qa_embed_diff(\n                qa_data)  # f_(ct,rt) or #h_rt\n            if self.separate_qa:\n                qa_embed_data = qa_embed_data + pid_embed_data * \\\n                    qa_embed_diff_data  # uq* f_(ct,rt) + e_(ct,rt)\n            else:\n                qa_embed_data = qa_embed_data + pid_embed_data * \\\n                    (qa_embed_diff_data+q_embed_diff_data)  # + uq *(h_rt+d_ct)\n            c_reg_loss = (pid_embed_data ** 2.).sum() * self.l2\n        else:\n            c_reg_loss = 0.\n\n        # BS.seqlen,d_model\n        # Pass to the decoder\n        # output shape BS,seqlen,d_model or d_model//2\n        d_output = self.model(q_embed_data, qa_embed_data)  # 211x512\n\n        concat_q = torch.cat([d_output, q_embed_data], dim=-1)\n        output = self.out(concat_q)\n        labels = target.reshape(-1)\n        m = nn.Sigmoid()\n        preds = (output.reshape(-1))  # logit\n        mask = labels > -0.9\n        masked_labels = labels[mask].float()\n        masked_preds = preds[mask]\n        loss = nn.BCEWithLogitsLoss(reduction='none')\n        output = loss(masked_preds, masked_labels)\n        return output.sum()+c_reg_loss, m(preds), mask.sum()\n\n\nclass Architecture(nn.Module):\n    def __init__(self, n_question,  n_blocks, d_model, d_feature,\n                 d_ff, n_heads, dropout, kq_same, model_type):\n        super().__init__()\n        \"\"\"\n            n_block : number of stacked blocks in the attention\n            d_model : dimension of attention input/output\n            d_feature : dimension of input in each of the multi-head attention part.\n            n_head : number of heads. n_heads*d_feature = d_model\n        \"\"\"\n        self.d_model = d_model\n        self.model_type = model_type\n\n        if model_type in {'akt'}:\n            self.blocks_1 = nn.ModuleList([\n                TransformerLayer(d_model=d_model, d_feature=d_model // n_heads,\n                                 d_ff=d_ff, dropout=dropout, n_heads=n_heads, kq_same=kq_same)\n                for _ in range(n_blocks)\n            ])\n            self.blocks_2 = nn.ModuleList([\n                TransformerLayer(d_model=d_model, d_feature=d_model // n_heads,\n                                 d_ff=d_ff, dropout=dropout, n_heads=n_heads, kq_same=kq_same)\n                for _ in range(n_blocks*2)\n            ])\n\n    def forward(self, q_embed_data, qa_embed_data):\n        # target shape  bs, seqlen\n        seqlen, batch_size = q_embed_data.size(1), q_embed_data.size(0)\n\n        qa_pos_embed = qa_embed_data\n        q_pos_embed = q_embed_data\n\n        y = qa_pos_embed\n        seqlen, batch_size = y.size(1), y.size(0)\n        x = q_pos_embed\n\n        # encoder\n        for block in self.blocks_1:  # encode qas\n            y = block(mask=1, query=y, key=y, values=y)\n        flag_first = True\n        for block in self.blocks_2:\n            if flag_first:  # peek current question\n                x = block(mask=1, query=x, key=x,\n                          values=x, apply_pos=False)\n                flag_first = False\n            else:  # dont peek current response\n                x = block(mask=0, query=x, key=x, values=y, apply_pos=True)\n                flag_first = True\n        return x\n\n\nclass TransformerLayer(nn.Module):\n    def __init__(self, d_model, d_feature,\n                 d_ff, n_heads, dropout,  kq_same):\n        super().__init__()\n        \"\"\"\n            This is a Basic Block of Transformer paper. It containts one Multi-head attention object. Followed by layer norm and postion wise feedforward net and dropout layer.\n        \"\"\"\n        kq_same = kq_same == 1\n        # Multi-Head Attention Block\n        self.masked_attn_head = MultiHeadAttention(\n            d_model, d_feature, n_heads, dropout, kq_same=kq_same)\n\n        # Two layer norm layer and two droput layer\n        self.layer_norm1 = nn.LayerNorm(d_model)\n        self.dropout1 = nn.Dropout(dropout)\n\n        self.linear1 = nn.Linear(d_model, d_ff)\n        self.activation = nn.ReLU()\n        self.dropout = nn.Dropout(dropout)\n        self.linear2 = nn.Linear(d_ff, d_model)\n\n        self.layer_norm2 = nn.LayerNorm(d_model)\n        self.dropout2 = nn.Dropout(dropout)\n\n    def forward(self, mask, query, key, values, apply_pos=True):\n        \"\"\"\n        Input:\n            block : object of type BasicBlock(nn.Module). It contains masked_attn_head objects which is of type MultiHeadAttention(nn.Module).\n            mask : 0 means, it can peek only past values. 1 means, block can peek only current and pas values\n            query : Query. In transformer paper it is the input for both encoder and decoder\n            key : Keys. In transformer paper it is the input for both encoder and decoder\n            Values. In transformer paper it is the input for encoder and  encoded output for decoder (in masked attention part)\n        Output:\n            query: Input gets changed over the layer and returned.\n        \"\"\"\n\n        seqlen, batch_size = query.size(1), query.size(0)\n        nopeek_mask = np.triu(\n            np.ones((1, 1, seqlen, seqlen)), k=mask).astype('uint8')\n        src_mask = (torch.from_numpy(nopeek_mask) == 0).to(device)\n        if mask == 0:  # If 0, zero-padding is needed.\n            # Calls block.masked_attn_head.forward() method\n            query2 = self.masked_attn_head(\n                query, key, values, mask=src_mask, zero_pad=True)\n        else:\n            # Calls block.masked_attn_head.forward() method\n            query2 = self.masked_attn_head(\n                query, key, values, mask=src_mask, zero_pad=False)\n\n        query = query + self.dropout1((query2))\n        query = self.layer_norm1(query)\n        if apply_pos:\n            query2 = self.linear2(self.dropout(\n                self.activation(self.linear1(query))))\n            query = query + self.dropout2((query2))\n            query = self.layer_norm2(query)\n        return query\n\n\nclass MultiHeadAttention(nn.Module):\n    def __init__(self, d_model, d_feature, n_heads, dropout, kq_same, bias=True):\n        super().__init__()\n        \"\"\"\n        It has projection layer for getting keys, queries and values. Followed by attention and a connected layer.\n        \"\"\"\n        self.d_model = d_model\n        self.d_k = d_feature\n        self.h = n_heads\n        self.kq_same = kq_same\n\n        self.v_linear = nn.Linear(d_model, d_model, bias=bias)\n        self.k_linear = nn.Linear(d_model, d_model, bias=bias)\n        if kq_same is False:\n            self.q_linear = nn.Linear(d_model, d_model, bias=bias)\n        self.dropout = nn.Dropout(dropout)\n        self.proj_bias = bias\n        self.out_proj = nn.Linear(d_model, d_model, bias=bias)\n        self.gammas = nn.Parameter(torch.zeros(n_heads, 1, 1))\n        torch.nn.init.xavier_uniform_(self.gammas)\n\n        self._reset_parameters()\n\n    def _reset_parameters(self):\n        xavier_uniform_(self.k_linear.weight)\n        xavier_uniform_(self.v_linear.weight)\n        if self.kq_same is False:\n            xavier_uniform_(self.q_linear.weight)\n\n        if self.proj_bias:\n            constant_(self.k_linear.bias, 0.)\n            constant_(self.v_linear.bias, 0.)\n            if self.kq_same is False:\n                constant_(self.q_linear.bias, 0.)\n            constant_(self.out_proj.bias, 0.)\n\n    def forward(self, q, k, v, mask, zero_pad):\n\n        bs = q.size(0)\n\n        # perform linear operation and split into h heads\n\n        k = self.k_linear(k).view(bs, -1, self.h, self.d_k)\n        if self.kq_same is False:\n            q = self.q_linear(q).view(bs, -1, self.h, self.d_k)\n        else:\n            q = self.k_linear(q).view(bs, -1, self.h, self.d_k)\n        v = self.v_linear(v).view(bs, -1, self.h, self.d_k)\n\n        # transpose to get dimensions bs * h * sl * d_model\n\n        k = k.transpose(1, 2)\n        q = q.transpose(1, 2)\n        v = v.transpose(1, 2)\n        # calculate attention using function we will define next\n        gammas = self.gammas\n        scores = attention(q, k, v, self.d_k,\n                           mask, self.dropout, zero_pad, gammas)\n\n        # concatenate heads and put through final linear layer\n        concat = scores.transpose(1, 2).contiguous()\\\n            .view(bs, -1, self.d_model)\n\n        output = self.out_proj(concat)\n\n        return output\n\n\ndef attention(q, k, v, d_k, mask, dropout, zero_pad, gamma=None):\n    \"\"\"\n    This is called by Multi-head atention object to find the values.\n    \"\"\"\n    scores = torch.matmul(q, k.transpose(-2, -1)) / \\\n        math.sqrt(d_k)  # BS, 8, seqlen, seqlen\n    bs, head, seqlen = scores.size(0), scores.size(1), scores.size(2)\n\n    x1 = torch.arange(seqlen).expand(seqlen, -1).to(device)\n    x2 = x1.transpose(0, 1).contiguous()\n\n    with torch.no_grad():\n        scores_ = scores.masked_fill(mask == 0, -1e32)\n        scores_ = F.softmax(scores_, dim=-1)  # BS,8,seqlen,seqlen\n        scores_ = scores_ * mask.float().to(device)\n        distcum_scores = torch.cumsum(scores_, dim=-1)  # bs, 8, sl, sl\n        disttotal_scores = torch.sum(\n            scores_, dim=-1, keepdim=True)  # bs, 8, sl, 1\n        position_effect = torch.abs(\n            x1-x2)[None, None, :, :].type(torch.FloatTensor).to(device)  # 1, 1, seqlen, seqlen\n        # bs, 8, sl, sl positive distance\n        dist_scores = torch.clamp(\n            (disttotal_scores-distcum_scores)*position_effect, min=0.)\n        dist_scores = dist_scores.sqrt().detach()\n    m = nn.Softplus()\n    gamma = -1. * m(gamma).unsqueeze(0)  # 1,8,1,1\n    # Now after do exp(gamma*distance) and then clamp to 1e-5 to 1e5\n    total_effect = torch.clamp(torch.clamp(\n        (dist_scores*gamma).exp(), min=1e-5), max=1e5)\n    scores = scores * total_effect\n\n    scores.masked_fill_(mask == 0, -1e32)\n    scores = F.softmax(scores, dim=-1)  # BS,8,seqlen,seqlen\n    if zero_pad:\n        pad_zero = torch.zeros(bs, head, 1, seqlen).to(device)\n        scores = torch.cat([pad_zero, scores[:, :, 1:, :]], dim=2)\n    scores = dropout(scores)\n    output = torch.matmul(scores, v)\n    return output\n\n\nclass LearnablePositionalEmbedding(nn.Module):\n    def __init__(self, d_model, max_len=512):\n        super().__init__()\n        # Compute the positional encodings once in log space.\n        pe = 0.1 * torch.randn(max_len, d_model)\n        pe = pe.unsqueeze(0)\n        self.weight = nn.Parameter(pe, requires_grad=True)\n\n    def forward(self, x):\n        return self.weight[:, :x.size(Dim.seq), :]  # ( 1,seq,  Feature)\n\n\nclass CosinePositionalEmbedding(nn.Module):\n    def __init__(self, d_model, max_len=512):\n        super().__init__()\n        # Compute the positional encodings once in log space.\n        pe = 0.1 * torch.randn(max_len, d_model)\n        position = torch.arange(0, max_len).unsqueeze(1).float()\n        div_term = torch.exp(torch.arange(0, d_model, 2).float() *\n                             -(math.log(10000.0) / d_model))\n        pe[:, 0::2] = torch.sin(position * div_term)\n        pe[:, 1::2] = torch.cos(position * div_term)\n        pe = pe.unsqueeze(0)\n        self.weight = nn.Parameter(pe, requires_grad=False)\n\n    def forward(self, x):\n        return self.weight[:, :x.size(Dim.seq), :]  # ( 1,seq,  Feature)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def try_makedirs(path_):\n    if not os.path.isdir(path_):\n        try:\n            os.makedirs(path_)\n        except FileExistsError:\n            pass\n\n\ndef get_file_name_identifier(params):\n    words = params.model.split('_')\n    model_type = words[0]\n    if model_type == 'dkt':\n        file_name = [['_b', params.batch_size], ['_gn', params.maxgradnorm], ['_lr', params.lr],\n                     ['_s', params.seed], ['_sl', params.seqlen], ['_dm', params.d_model], ['_ts', params.train_set],  ['_h', params.hidden_dim], ['_do', params.dropout], ['_l2', params.l2]]\n    elif model_type == 'dktplus':\n        file_name = [['_b', params.batch_size], ['_gn', params.maxgradnorm], ['_lr', params.lr],\n                     ['_s', params.seed], ['_sl', params.seqlen], ['_dm', params.d_model], ['_ts', params.train_set],  ['_h', params.hidden_dim], ['_do', params.dropout], ['_l2', params.l2], ['_r', params.lamda_r], ['_w1', params.lamda_w1], ['_w2', params.lamda_w2]]\n    elif model_type == 'dkvmn':\n        file_name = [['_b', params.batch_size], ['_gn', params.maxgradnorm], ['_lr', params.lr],\n                     ['_s', params.seed], ['_sl', params.seqlen], ['_q', params.q_embed_dim], ['_qa', params.qa_embed_dim], ['_ts', params.train_set], ['_m', params.memory_size], ['_l2', params.l2]]\n    elif model_type in {'akt', 'sakt'}:\n        file_name = [['_b', params.batch_size], ['_nb', params.n_block], ['_gn', params.maxgradnorm], ['_lr', params.lr],\n                     ['_s', params.seed], ['_sl', params.seqlen], ['_do', params.dropout], ['_dm', params.d_model], ['_ts', params.train_set], ['_kq', params.kq_same], ['_l2', params.l2]]\n    return file_name\n\n\ndef model_isPid_type(model_name):\n    words = model_name.split('_')\n    is_pid = True if 'pid' in words else False\n    return is_pid, words[0]\n\n\ndef load_model(params):\n    words = params.model.split('_')\n    model_type = words[0]\n    is_cid = words[1] == 'cid'\n    if is_cid:\n        params.n_pid = -1\n\n    if model_type in {'akt'}:\n        model = AKT(n_question=params.n_question, n_pid=params.n_pid, n_blocks=params.n_block, d_model=params.d_model,\n                    dropout=params.dropout, kq_same=params.kq_same, model_type=model_type, l2=params.l2).to(device)\n    else:\n        model = None\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport math","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transpose_data_model = {'akt'}\n\n\ndef binaryEntropy(target, pred, mod=\"avg\"):\n    loss = target * np.log(np.maximum(1e-10, pred)) + \\\n        (1.0 - target) * np.log(np.maximum(1e-10, 1.0-pred))\n    if mod == 'avg':\n        return np.average(loss)*(-1.0)\n    elif mod == 'sum':\n        return - loss.sum()\n    else:\n        assert False\n\n\ndef compute_auc(all_target, all_pred):\n    #fpr, tpr, thresholds = metrics.roc_curve(all_target, all_pred, pos_label=1.0)\n    return metrics.roc_auc_score(all_target, all_pred)\n\n\ndef compute_accuracy(all_target, all_pred):\n    all_pred[all_pred > 0.5] = 1.0\n    all_pred[all_pred <= 0.5] = 0.0\n    return metrics.accuracy_score(all_target, all_pred)\n\n\ndef train(net, params,  optimizer,  q_data, qa_data, pid_data,  label):\n    net.train()\n    pid_flag, model_type = model_isPid_type(params.model)\n    N = int(math.ceil(len(q_data) / params.batch_size))\n    q_data = q_data.T  # Shape: (200,3633)\n    qa_data = qa_data.T  # Shape: (200,3633)\n    # Shuffle the data\n    shuffled_ind = np.arange(q_data.shape[1])\n    np.random.shuffle(shuffled_ind)\n    q_data = q_data[:, shuffled_ind]\n    qa_data = qa_data[:, shuffled_ind]\n\n    if pid_flag:\n        pid_data = pid_data.T\n        pid_data = pid_data[:, shuffled_ind]\n\n    pred_list = []\n    target_list = []\n\n    element_count = 0\n    true_el = 0\n    for idx in range(N):\n        optimizer.zero_grad()\n\n        q_one_seq = q_data[:, idx*params.batch_size:(idx+1)*params.batch_size]\n        if pid_flag:\n            pid_one_seq = pid_data[:, idx *\n                                   params.batch_size:(idx+1) * params.batch_size]\n\n        qa_one_seq = qa_data[:, idx *\n                             params.batch_size:(idx+1) * params.batch_size]\n\n        if model_type in transpose_data_model:\n            input_q = np.transpose(q_one_seq[:, :])  # Shape (bs, seqlen)\n            input_qa = np.transpose(qa_one_seq[:, :])  # Shape (bs, seqlen)\n            target = np.transpose(qa_one_seq[:, :])\n            if pid_flag:\n                # Shape (seqlen, batch_size)\n                input_pid = np.transpose(pid_one_seq[:, :])\n        else:\n            input_q = (q_one_seq[:, :])  # Shape (seqlen, batch_size)\n            input_qa = (qa_one_seq[:, :])  # Shape (seqlen, batch_size)\n            target = (qa_one_seq[:, :])\n            if pid_flag:\n                input_pid = (pid_one_seq[:, :])  # Shape (seqlen, batch_size)\n        target = (target - 1) / params.n_question\n        target_1 = np.floor(target)\n        el = np.sum(target_1 >= -.9)\n        element_count += el\n\n        input_q = torch.from_numpy(input_q).long().to(device)\n        input_qa = torch.from_numpy(input_qa).long().to(device)\n        target = torch.from_numpy(target_1).float().to(device)\n        if pid_flag:\n            input_pid = torch.from_numpy(input_pid).long().to(device)\n\n        if pid_flag:\n            loss, pred, true_ct = net(input_q, input_qa, target, input_pid)\n        else:\n            loss, pred, true_ct = net(input_q, input_qa, target)\n        pred = pred.detach().cpu().numpy()  # (seqlen * batch_size, 1)\n        loss.backward()\n        true_el += true_ct.cpu().numpy()\n\n        if params.maxgradnorm > 0.:\n            torch.nn.utils.clip_grad_norm_(\n                net.parameters(), max_norm=params.maxgradnorm)\n\n        optimizer.step()\n\n        # correct: 1.0; wrong 0.0; padding -1.0\n        target = target_1.reshape((-1,))\n\n        nopadding_index = np.flatnonzero(target >= -0.9)\n        nopadding_index = nopadding_index.tolist()\n        pred_nopadding = pred[nopadding_index]\n        target_nopadding = target[nopadding_index]\n\n        pred_list.append(pred_nopadding)\n        target_list.append(target_nopadding)\n\n    all_pred = np.concatenate(pred_list, axis=0)\n    all_target = np.concatenate(target_list, axis=0)\n\n    loss = binaryEntropy(all_target, all_pred)\n    auc = compute_auc(all_target, all_pred)\n    accuracy = compute_accuracy(all_target, all_pred)\n\n    return loss, accuracy, auc\n\n\ndef test(net, params, optimizer, q_data, qa_data, pid_data, label):\n    # dataArray: [ array([[],[],..])] Shape: (3633, 200)\n    pid_flag, model_type = model_isPid_type(params.model)\n    net.eval()\n    N = int(math.ceil(float(len(q_data)) / float(params.batch_size)))\n    q_data = q_data.T  # Shape: (200,3633)\n    qa_data = qa_data.T  # Shape: (200,3633)\n    if pid_flag:\n        pid_data = pid_data.T\n    seq_num = q_data.shape[1]\n    pred_list = []\n    target_list = []\n\n    count = 0\n    true_el = 0\n    element_count = 0\n    for idx in range(N):\n\n        q_one_seq = q_data[:, idx*params.batch_size:(idx+1)*params.batch_size]\n        if pid_flag:\n            pid_one_seq = pid_data[:, idx *\n                                   params.batch_size:(idx+1) * params.batch_size]\n        input_q = q_one_seq[:, :]  # Shape (seqlen, batch_size)\n        qa_one_seq = qa_data[:, idx *\n                             params.batch_size:(idx+1) * params.batch_size]\n        input_qa = qa_one_seq[:, :]  # Shape (seqlen, batch_size)\n\n        # print 'seq_num', seq_num\n        if model_type in transpose_data_model:\n            # Shape (seqlen, batch_size)\n            input_q = np.transpose(q_one_seq[:, :])\n            # Shape (seqlen, batch_size)\n            input_qa = np.transpose(qa_one_seq[:, :])\n            target = np.transpose(qa_one_seq[:, :])\n            if pid_flag:\n                input_pid = np.transpose(pid_one_seq[:, :])\n        else:\n            input_q = (q_one_seq[:, :])  # Shape (seqlen, batch_size)\n            input_qa = (qa_one_seq[:, :])  # Shape (seqlen, batch_size)\n            target = (qa_one_seq[:, :])\n            if pid_flag:\n                input_pid = (pid_one_seq[:, :])\n        target = (target - 1) / params.n_question\n        target_1 = np.floor(target)\n        #target = np.random.randint(0,2, size = (target.shape[0],target.shape[1]))\n\n        input_q = torch.from_numpy(input_q).long().to(device)\n        input_qa = torch.from_numpy(input_qa).long().to(device)\n        target = torch.from_numpy(target_1).float().to(device)\n        if pid_flag:\n            input_pid = torch.from_numpy(input_pid).long().to(device)\n\n        with torch.no_grad():\n            if pid_flag:\n                loss, pred, ct = net(input_q, input_qa, target, input_pid)\n            else:\n                loss, pred, ct = net(input_q, input_qa, target)\n        pred = pred.cpu().numpy()  # (seqlen * batch_size, 1)\n        true_el += ct.cpu().numpy()\n        #target = target.cpu().numpy()\n        if (idx + 1) * params.batch_size > seq_num:\n            real_batch_size = seq_num - idx * params.batch_size\n            count += real_batch_size\n        else:\n            count += params.batch_size\n\n        # correct: 1.0; wrong 0.0; padding -1.0\n        target = target_1.reshape((-1,))\n        #nopadding_index = np.flatnonzero(target >= -0.9)\n        #nopadding_index = nopadding_index.tolist()\n        pred_nopadding = pred[nopadding_index]\n        target_nopadding = target[nopadding_index]\n\n        element_count += pred_nopadding.shape[0]\n        # print avg_loss\n        pred_list.append(pred_nopadding)\n        target_list.append(target_nopadding)\n\n    assert count == seq_num, \"Seq not matching\"\n\n    all_pred = np.concatenate(pred_list, axis=0)\n    all_target = np.concatenate(target_list, axis=0)\n    loss = binaryEntropy(all_target, all_pred)\n    auc = compute_auc(all_target, all_pred)\n    accuracy = compute_accuracy(all_target, all_pred)\n\n    return loss, accuracy, auc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_one_dataset(params, file_name, train_q_data, train_qa_data, train_pid, valid_q_data, valid_qa_data, valid_pid):\n    # ================================== model initialization ==================================\n\n    model = load_model(params)\n    optimizer = torch.optim.Adam(\n        model.parameters(), lr=params.lr, betas=(0.9, 0.999), eps=1e-8)\n\n    print(\"\\n\")\n\n    # ================================== start training ==================================\n    all_train_loss = {}\n    all_train_accuracy = {}\n    all_train_auc = {}\n    all_valid_loss = {}\n    all_valid_accuracy = {}\n    all_valid_auc = {}\n    best_valid_auc = 0\n\n    for idx in range(params.max_iter):\n        # Train Model\n        train_loss, train_accuracy, train_auc = train(\n            model, params, optimizer, train_q_data, train_qa_data, train_pid,  label='Train')\n        # Validation step\n        valid_loss, valid_accuracy, valid_auc = test(\n            model,  params, optimizer, valid_q_data, valid_qa_data, valid_pid, label='Valid')\n\n        print('epoch', idx + 1)\n        print(\"valid_auc\\t\", valid_auc, \"\\ttrain_auc\\t\", train_auc)\n        print(\"valid_accuracy\\t\", valid_accuracy,\n              \"\\ttrain_accuracy\\t\", train_accuracy)\n        print(\"valid_loss\\t\", valid_loss, \"\\ttrain_loss\\t\", train_loss)\n\n        try_makedirs('model')\n        try_makedirs(os.path.join('model', params.model))\n        try_makedirs(os.path.join('model', params.model, params.save))\n\n        all_valid_auc[idx + 1] = valid_auc\n        all_train_auc[idx + 1] = train_auc\n        all_valid_loss[idx + 1] = valid_loss\n        all_train_loss[idx + 1] = train_loss\n        all_valid_accuracy[idx + 1] = valid_accuracy\n        all_train_accuracy[idx + 1] = train_accuracy\n\n        # output the epoch with the best validation auc\n        if valid_auc > best_valid_auc:\n            path = os.path.join('model', params.model,\n                                params.save,  file_name) + '_*'\n            for i in glob.glob(path):\n                os.remove(i)\n            best_valid_auc = valid_auc\n            best_epoch = idx+1\n            torch.save({'epoch': idx,\n                        'model_state_dict': model.state_dict(),\n                        'optimizer_state_dict': optimizer.state_dict(),\n                        'loss': train_loss,\n                        },\n                       os.path.join('model', params.model, params.save,\n                                    file_name)+'_' + str(idx+1)\n                       )\n        if idx-best_epoch > 40:\n            break   \n\n    try_makedirs('result')\n    try_makedirs(os.path.join('result', params.model))\n    try_makedirs(os.path.join('result', params.model, params.save))\n    f_save_log = open(os.path.join(\n        'result', params.model, params.save, file_name), 'w')\n    f_save_log.write(\"valid_auc:\\n\" + str(all_valid_auc) + \"\\n\\n\")\n    f_save_log.write(\"train_auc:\\n\" + str(all_train_auc) + \"\\n\\n\")\n    f_save_log.write(\"valid_loss:\\n\" + str(all_valid_loss) + \"\\n\\n\")\n    f_save_log.write(\"train_loss:\\n\" + str(all_train_loss) + \"\\n\\n\")\n    f_save_log.write(\"valid_accuracy:\\n\" + str(all_valid_accuracy) + \"\\n\\n\")\n    f_save_log.write(\"train_accuracy:\\n\" + str(all_train_accuracy) + \"\\n\\n\")\n    f_save_log.close()\n    return best_epoch\n\n\ndef test_one_dataset(params, file_name, test_q_data, test_qa_data, test_pid,  best_epoch):\n    print(\"\\n\\nStart testing ......................\\n Best epoch:\", best_epoch)\n    model = load_model(params)\n\n    checkpoint = torch.load(os.path.join(\n        'model', params.model, params.save, file_name) + '_'+str(best_epoch))\n    model.load_state_dict(checkpoint['model_state_dict'])\n\n    test_loss, test_accuracy, test_auc = test(\n        model, params, None, test_q_data, test_qa_data, test_pid, label='Test')\n    print(\"\\ntest_auc\\t\", test_auc)\n    print(\"test_accuracy\\t\", test_accuracy)\n    print(\"test_loss\\t\", test_loss)\n\n    # Now Delete all the models\n    path = os.path.join('model', params.model, params.save,  file_name) + '_*'\n    for i in glob.glob(path):\n        os.remove(i)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class params:\n    max_iter=300\n    train_set=1\n    seed=224\n    optim='adam'\n    batch_size=24\n    lr=1e-5\n    maxgradnorm=-1\n    final_fc_dim=512\n    d_model=256\n    d_ff=1024\n    dropout=0.05\n    n_block=1\n    n_head=8\n    kq_same=1\n    l2=1e-5\n    q_embed_dim=50\n    qa_embed_dim=256\n    memory_size=50\n    init_std=0.1\n    hidden_dim=512\n    lamda_r=0.1\n    lamda_w1=0.1\n    lamda_w2=0.1\n    model='akt_pid'\n    dataset=\"assist2009_pid\"\n\n    \ndataset = params.dataset\n\nparams.n_question = 1223\nparams.batch_size = 24\nparams.seqlen = 200\nparams.data_dir = 'data/'+dataset\nparams.data_name = dataset\n\nparams.save = params.data_name\nparams.load = params.data_name\n\nseedNum = params.seed\nnp.random.seed(seedNum)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\ntorch.manual_seed(seedNum)\nnp.random.seed(seedNum)\nfile_name_identifier = get_file_name_identifier(params)\n\n###Train- Test\nd = vars(params)\nfor key in d:\n    print('\\t', key, '\\t', d[key])\nfile_name = ''\nfor item_ in file_name_identifier:\n    file_name = file_name+item_[0] + str(item_[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"used_data_types_dict = {\n        'row_id': 'int64', \n        'user_id': 'int32', \n        'content_id': 'int16',\n        'content_type_id': 'int8',\n        'answered_correctly': 'int8', \n}\ntrain_df = pd.read_csv(\n    '/kaggle/input/riiid-test-answer-prediction/train.csv',\n    usecols = used_data_types_dict.keys(),\n    dtype=used_data_types_dict,\n    index_col = 0\n)\n\nq_used_data_types_dict = {\n        'question_id': 'int64', \n        'part': 'int8',\n}\nquestions_df = pd.read_csv(\n    '/kaggle/input/riiid-test-answer-prediction/questions.csv',\n    usecols = q_used_data_types_dict.keys(),\n    dtype=q_used_data_types_dict, \n    index_col = 0\n)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_qtype = {}\nuser_ans = {}\nuser_pid = {}\nuser = set()\nfor row in train_df.iterrows():\n    if(row[1]['content_type_id']==1):\n        continue\n    user.add(row[1]['user_id'])\n    if(user_qtype.get(row[1]['user_id'])== None):\n        user_qtype[row[1]['user_id']]=[]\n    if(user_ans.get(row[1]['user_id'])== None):\n        user_ans[row[1]['user_id']]=[]\n    if(user_pid.get(row[1]['user_id'])== None):\n        user_pid[row[1]['user_id']]=[]\n    user_qtype[row[1]['user_id']].append(questions_df.iloc[row[1]['content_id']]['part'])\n    user_ans[row[1]['user_id']].append(row[1]['answered_correctly'])\n    user_pid[row[1]['user_id']].append(row[1]['content_id'])\nseqlen=0\nfor u in user:\n    seqlen =max(len(user_ans[user]), seqlen)\n\ntrain_q_data= np.zeros((len(user), seqlen))\ntrain_qa_data= np.zeros((len(user), seqlen))\ntrain_pid = np.zeros((len(user), seqlen))\nvalid_q_data= np.zeros((len(user), seqlen))\nvalid_qa_data= np.zeros((len(user), seqlen))\nvalid_pid= np.zeros((len(user), seqlen))\ncnt =0\nfor j in user:\n    if cnt <= 0.8 * len(user):\n        dat = user_qtype[j]\n        train_q_data[j, :len(dat)] = dat\n\n        dat = user_ans[j]\n        train_qa_data[j, :len(dat)] = dat   \n\n        dat = user_pid[j]\n        train_q_data[j, :len(dat)] = dat\n    else:\n        dat = user_qtype[j]\n        valid_q_data[j, :len(dat)] = dat\n\n        dat = user_ans[j]\n        valid_qa_data[j, :len(dat)] = dat   \n\n        dat = user_pid[j]\n        valid_pid[j, :len(dat)] = dat","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"\\n\")\nprint(\"train_q_data.shape\", train_q_data.shape)\nprint(\"train_qa_data.shape\", train_qa_data.shape)\nprint(\"valid_q_data.shape\", valid_q_data.shape)  # (1566, 200)\nprint(\"valid_qa_data.shape\", valid_qa_data.shape)  # (1566, 200)\nprint(\"\\n\")\n# Train and get the best episode\nbest_epoch = train_one_dataset(\n    params, file_name, train_q_data, train_qa_data, train_pid, valid_q_data, valid_qa_data, valid_pid)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()\niter_test = env.iter_test()\n\nfor (test_df, sample_prediction_df) in iter_test:\n\ntest_q_data, test_qa_data, test_index = dat.load_data(test_data_path)\n    \ntest_one_dataset(params, file_name, test_q_data,\n                 test_qa_data, test_index, best_epoch)\n\ntest_df['answered_correctly'] = model.predict_proba(test_df[features])[:,1]\nenv.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}