{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Main changes and references\n\n1. Version 1: A warm-up scheduler is added as it is known that Transformer-based model usually needs a warm-up phase.\nhttps://ufal.mff.cuni.cz/pbml/110/art-popel-bojar.pdf\nThe learning rate will be some magnitudes smaller than the targeted learning rate, gradually increase to that level, then comes back down in various fashion. Sometime in later epochs, the learning rate will increase again (called restart).\n\n2. Version 2: @gilfernandes 's modification of stacked attention layers with non-shared weights are added: https://www.kaggle.com/gilfernandes/riiid-self-attention-transformer, a simple function checking model's size is added as well.\n\n**Reference**: \n- https://www.kaggle.com/leadbest/sakt-with-randomization-state-updates \n- https://www.kaggle.com/wangsg/a-self-attentive-model-for-knowledge-tracing\n- https://www.kaggle.com/manikanthr5/riiid-sakt-model-training-public\n- https://www.kaggle.com/marisakamozz/cv-strategy-in-the-kaggle-environment\n- https://huggingface.co/transformers/v1.2.0/_modules/pytorch_transformers/optimization.html\n"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import gc\nimport psutil\nimport random\nfrom tqdm.notebook import tqdm\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set()\n\nfrom sklearn.metrics import roc_auc_score\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"`WARMUP_EPOCHS` is the warm-up epochs, where the learning rate linearly increases from 0 to the desired learning rate. Then the learning rate will start to decay, the `NUM_RESTARTS` controls the number of restarts happened in all the epochs."},{"metadata":{"trusted":true},"cell_type":"code","source":"WARMUP_EPOCHS = 10\nNUM_RESTARTS = 2\nEPOCHS = 40\n\nBATCH_SIZE = 256\nVAL_BATCH_SIZE = 2048\nMAX_SEQ = 160\nNUM_HEADS = 8\nNUM_EMBED = 128\nNUM_LAYERS = 2\n\nACCEPTED_USER_CONTENT_SIZE = 3\nn_skill = 13523\n\nLEARNING_RATE = 5e-4","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data\nHere we use @marisakamozz 's almost unbiased CV file: https://www.kaggle.com/marisakamozz/cv-strategy-in-the-kaggle-environment"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ndtypes = {'timestamp': 'int64', \n          'user_id': 'int32' ,\n          'content_id': 'int16',\n          'content_type_id': 'int8',\n          'answered_correctly':'int8'}\ntrain_cols = ['timestamp', \n              'user_id', \n              'content_id', \n              'content_type_id', \n              'answered_correctly']\ntrain_df = pd.read_parquet('../input/cv-strategy-in-the-kaggle-environment/cv4_train.parquet')\ntrain_df = train_df[train_cols]\ntrain_df = train_df.astype(dtypes)\n\n# for col, dtype in dtypes.items():\n#     train_df[col] = train_df[col].astype(dtype)\nprint(train_df.dtypes)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preprocess"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df[train_df.content_type_id == False]\ntrain_df = train_df.sort_values(['timestamp'], ascending=True).reset_index(drop=True)\ngroup = train_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id')\\\n    .apply(lambda r: (r['content_id'].values,\n                      r['answered_correctly'].values))\ndel train_df\ngc.collect();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nvalid_df = pd.read_parquet('../input/cv-strategy-in-the-kaggle-environment/cv4_valid.parquet')\nvalid_df = valid_df[train_cols]\nvalid_df = valid_df.astype(dtypes)\n\nvalid_df = valid_df[valid_df.content_type_id == False]\nval_group = valid_df[['user_id', 'content_id', 'answered_correctly']]\\\n    .groupby('user_id')\\\n    .apply(lambda r: (r['content_id'].values,\n                      r['answered_correctly'].values))\ndel valid_df\ngc.collect();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SAKTDataset(Dataset):\n    def __init__(self, group, n_skill, max_seq=MAX_SEQ):\n        super(SAKTDataset, self).__init__()\n        self.max_seq = max_seq\n        self.n_skill = n_skill\n        self.samples = {}\n        \n        self.user_ids = []\n        for user_id in group.index:\n            q, qa = group[user_id]\n            if len(q) < ACCEPTED_USER_CONTENT_SIZE:\n                continue\n            \n            # Main Contribution by Manikanth Reddy\n            if len(q) > self.max_seq:\n                total_questions = len(q)\n                initial = total_questions % self.max_seq\n                if initial >= ACCEPTED_USER_CONTENT_SIZE:\n                    self.user_ids.append(f\"{user_id}_0\")\n                    self.samples[f\"{user_id}_0\"] = (q[:initial], qa[:initial])\n                for seq in range(total_questions // self.max_seq):\n                    self.user_ids.append(f\"{user_id}_{seq+1}\")\n                    start = initial + seq * self.max_seq\n                    end = initial + (seq + 1) * self.max_seq\n                    self.samples[f\"{user_id}_{seq+1}\"] = (q[start:end], qa[start:end])\n            else:\n                user_id = str(user_id)\n                self.user_ids.append(user_id)\n                self.samples[user_id] = (q, qa)\n    \n    def __len__(self):\n        return len(self.user_ids)\n\n    def __getitem__(self, index):\n        user_id = self.user_ids[index]\n        q_, qa_ = self.samples[user_id]\n        seq_len = len(q_)\n\n        q = np.zeros(self.max_seq, dtype=int)\n        qa = np.zeros(self.max_seq, dtype=int)\n        if seq_len == self.max_seq:\n            q[:] = q_\n            qa[:] = qa_\n        else:\n            q[-seq_len:] = q_\n            qa[-seq_len:] = qa_\n        \n        target_id = q[1:]\n        label = qa[1:]\n\n        x = np.zeros(self.max_seq-1, dtype=int)\n        x = q[:-1].copy()\n        x += (qa[:-1] == 1) * self.n_skill\n\n        return x, target_id, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = SAKTDataset(group, n_skill)\ndataloader = DataLoader(dataset, \n                        batch_size=BATCH_SIZE, \n                        shuffle=True, \n                        num_workers=4)\n\n\nvalid_dataset = SAKTDataset(val_group, n_skill)\nval_loader = DataLoader(valid_dataset, \n                          batch_size=VAL_BATCH_SIZE, \n                          shuffle=False, \n                          drop_last=False,\n                          num_workers=4)\n\nitem = next(iter(dataloader))\nval_item = next(iter(val_loader))\n\nfor i in range(len(item)):\n    print(item[i].shape, val_item[i].shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class FFN(nn.Module):\n    def __init__(self, \n                 state_size=200, \n                 bn_size=MAX_SEQ-1,\n                 dropout=0.2):\n        super(FFN, self).__init__()\n        self.state_size = state_size\n\n        self.lr1 = nn.Linear(state_size, state_size)\n        self.relu = nn.ReLU()\n        self.lr2 = nn.Linear(state_size, state_size)\n        self.dropout = nn.Dropout(dropout)\n        self.bn = nn.BatchNorm1d(bn_size)\n        \n    def forward(self, x):\n        x = self.lr1(x)\n        x = self.relu(x)\n        x = self.bn(x)\n        x = self.lr2(x)\n        return self.dropout(x)\n\ndef future_mask(seq_length):\n    future_mask = np.triu(np.ones((seq_length, seq_length)), k=1).astype('bool')\n    return torch.from_numpy(future_mask)\n\n\nclass TransformerBlock(nn.Module):\n    def __init__(self, embed_dim, \n                 heads = 8, \n                 dropout = 0.2):\n        super(TransformerBlock, self).__init__()\n        self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=heads, dropout=dropout)\n        self.dropout = nn.Dropout(dropout)\n        self.layer_normal = nn.LayerNorm(embed_dim)\n        self.ffn = FFN(embed_dim, dropout=dropout)\n        self.layer_normal_2 = nn.LayerNorm(embed_dim)\n        \n\n    def forward(self, value, key, query, att_mask):\n        att_output, att_weight = self.multi_att(value, key, query, attn_mask=att_mask)\n        att_output = self.dropout(self.layer_normal(att_output + value))\n        att_output = att_output.permute(1, 0, 2) # att_output: [s_len, bs, embed] => [bs, s_len, embed]\n        x = self.ffn(att_output)\n        x = self.dropout(self.layer_normal_2(x + att_output))\n        return x.squeeze(-1), att_weight\n    \nclass Encoder(nn.Module):\n    def __init__(self, n_skill, \n                 max_seq=100, \n                 embed_dim=128, \n                 dropout = 0.2, \n                 num_layers=1, \n                 heads = 8):\n        super(Encoder, self).__init__()\n        self.n_skill, self.embed_dim = n_skill, embed_dim\n        self.embedding = nn.Embedding(2 * n_skill + 1, embed_dim)\n        self.pos_embedding = nn.Embedding(max_seq - 1, embed_dim)\n        self.e_embedding = nn.Embedding(n_skill+1, embed_dim)\n        self.layers = nn.ModuleList([TransformerBlock(embed_dim)\\\n                                     for _ in range(num_layers)])\n        self.dropout = nn.Dropout(dropout)\n        \n    def forward(self, x, question_ids):\n        device = x.device\n        x = self.embedding(x)\n        pos_id = torch.arange(x.size(1)).unsqueeze(0).to(device)\n        pos_x = self.pos_embedding(pos_id)\n        x = self.dropout(x + pos_x)\n        x = x.permute(1, 0, 2) # x: [bs, s_len, embed] => [s_len, bs, embed]\n        e = self.e_embedding(question_ids)\n        e = e.permute(1, 0, 2)\n        for layer in self.layers:\n            att_mask = future_mask(e.size(0)).to(device)\n            x, att_weight = layer(e, x, x, att_mask=att_mask)\n            x = x.permute(1, 0, 2)\n        x = x.permute(1, 0, 2)\n        return x, att_weight\n\nclass SAKTModel(nn.Module):\n    def __init__(self, n_skill, \n                 max_seq=100, \n                 embed_dim=128, \n                 dropout = 0.2, \n                 enc_layers=1, \n                 heads = 8):\n        super(SAKTModel, self).__init__()\n        self.encoder = Encoder(n_skill, \n                               max_seq, \n                               embed_dim, \n                               dropout, \n                               num_layers=enc_layers,\n                               heads=heads)\n        self.pred = nn.Linear(embed_dim, 1)\n        \n    def forward(self, x, question_ids):\n        x, att_weight = self.encoder(x, question_ids)\n        x = self.pred(x)\n        return x.squeeze(-1), att_weight","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Scheduler\n\nThis is the scheduler adapted from https://huggingface.co/transformers/v2.0.0/main_classes/optimizer_schedules.html#schedules"},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.optim.lr_scheduler import LambdaLR\nimport math\n\nclass WarmupCosineSchedule(LambdaLR):\n    \"\"\" Linear warmup and then cosine decay.\n        Linearly increases learning rate from 0 to 1 over `warmup_steps` training steps.\n        Decreases learning rate from 1. to 0. over remaining `t_total - warmup_steps` steps following a cosine curve.\n        If `cycles` (default=0.5) is different from default, learning rate follows cosine function after warmup.\n    \"\"\"\n    def __init__(self, optimizer, warmup_steps, t_total, cycles=.5, last_epoch=-1):\n        self.warmup_steps = warmup_steps\n        self.t_total = t_total\n        self.cycles = cycles\n        self.name = 'Warmup Cosine'\n        super(WarmupCosineSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch)\n\n    def lr_lambda(self, step):\n        if step < self.warmup_steps:\n            return float(step) / float(max(1.0, self.warmup_steps))\n        # progress after warmup\n        progress = float(step - self.warmup_steps) / float(max(1, self.t_total - self.warmup_steps))\n        return max(0.0, 0.5 * (1. + math.cos(math.pi * float(self.cycles) * 2.0 * progress)))\n    \nclass WarmupCosineWithHardRestartsSchedule(LambdaLR):\n    \"\"\" Linear warmup and then cosine cycles with hard restarts.\n        Linearly increases learning rate from 0 to 1 over `warmup_steps` training steps.\n        If `cycles` (default=1.) is different from default, learning rate follows `cycles` times a cosine decaying\n        learning rate (with hard restarts).\n    \"\"\"\n    def __init__(self, optimizer, warmup_steps, t_total, cycles=1., last_epoch=-1):\n        self.warmup_steps = warmup_steps\n        self.t_total = t_total\n        self.cycles = cycles\n        self.name = 'Warmup Cosine With Hard Restarts'\n        super(WarmupCosineWithHardRestartsSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch)\n\n    def lr_lambda(self, step):\n        if step < self.warmup_steps:\n            return float(step) / float(max(1, self.warmup_steps))\n        # progress after warmup\n        progress = float(step - self.warmup_steps) / float(max(1, self.t_total - self.warmup_steps))\n        if progress >= 1.0:\n            return 0.0\n        return max(0.0, 0.5 * (1. + math.cos(math.pi * ((float(self.cycles) * progress) % 1.0))))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create the model\nHere we added a function of checking model's size, just making sure the model does not grow too large."},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_num_params(model):\n    model_parameters = filter(lambda p: p.requires_grad, model.parameters())\n    n_params = sum([np.prod(p.size()) for p in model_parameters])\n    return n_params","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nmodel = SAKTModel(n_skill, embed_dim=NUM_EMBED, max_seq=MAX_SEQ, \n                  enc_layers=NUM_LAYERS, heads=NUM_HEADS, dropout=0.1)\n\ncriterion = nn.BCEWithLogitsLoss()\n\nmodel.to(device)\ncriterion.to(device);\nn_params = get_num_params(model)\nprint(f\"Current model has {n_params} parameters.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def train_epoch(model, dataloader, optimizer, criterion, device=\"cuda\"):\n    model.train()\n\n    train_loss = []\n    num_corrects = 0\n    num_total = 0\n    labels = []\n    outs = []\n    len_loader = len(dataloader)\n#     with tqdm(total=len_loader) as pbar:\n    for idx, item in enumerate(dataloader):\n        x = item[0].to(device).long()\n        target_id = item[1].to(device).long()\n        label = item[2].to(device).float()\n\n        optimizer.zero_grad()\n        output, atten_weight = model(x, target_id)\n        loss = criterion(output, label)\n        loss.backward()\n        optimizer.step()\n        train_loss.append(loss.item())\n\n        output = output[:, -1]\n        label = label[:, -1] \n        pred = (torch.sigmoid(output) >= 0.5).long()\n\n        num_corrects += (pred == label).sum().item()\n        num_total += len(label)\n\n        labels.extend(label.view(-1).data.cpu().numpy())\n        outs.extend(output.view(-1).data.cpu().numpy())\n            \n#             if idx % 8==0: pbar.update(8)\n\n    acc = num_corrects / num_total\n    auc = roc_auc_score(labels, outs)\n    loss = np.mean(train_loss)\n\n    return loss, acc, auc\n\ndef valid_epoch(model, valid_iterator, criterion, device=\"cuda\"):\n    model.eval()\n    \n    valid_loss = []\n    num_corrects = 0\n    num_total = 0\n    labels = []\n    outs = []\n    len_dataset = len(valid_iterator)\n    \n    for idx, item in enumerate(valid_iterator): \n        x = item[0].to(device).long()\n        target_id = item[1].to(device).long()\n        label = item[2].to(device).float()\n\n        with torch.no_grad():\n            output, _ = model(x, target_id)\n        loss = criterion(output, label)\n        valid_loss.append(loss.item())\n\n        output = output[:, -1] # (BS, 1)   \n        \n        output = torch.sigmoid(output)\n        \n        label = label[:, -1] \n        pred = (output >= 0.5).long()\n\n        num_corrects += (pred == label).sum().item()\n        num_total += len(label)\n\n        labels.extend(label.view(-1).data.cpu().numpy())\n        outs.extend(output.view(-1).data.cpu().numpy())\n\n    acc = num_corrects / num_total\n    auc = roc_auc_score(labels, outs)\n    loss = np.mean(valid_loss)\n\n    return loss, acc, auc","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Scheduler is here\nAfter each epoch, `scheduler.step()` needs to be called."},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)\n# scheduler = WarmupCosineSchedule(optimizer, \n#                         warmup_steps=WARMUP_EPOCHS, \n#                         t_total=EPOCHS)\nscheduler = WarmupCosineWithHardRestartsSchedule(optimizer, \n                                                 warmup_steps=WARMUP_EPOCHS, \n                                                 cycles=NUM_RESTARTS, \n                                                 t_total=EPOCHS)\nlr = []\nbest_auc = 0\nbest_epoch = 0\nsaving_threshold =  0.7675","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for epoch in range(EPOCHS):\n    loss, acc, auc = train_epoch(model, \n                                 dataloader, \n                                 optimizer, \n                                 criterion, \n                                 device)\n    scheduler.step()\n    lr.append(optimizer.param_groups[0]['lr'])\n    print(f\"\\nEpoch - [{epoch+1}/{EPOCHS}]\")\n    print(f\"Learning rate : {lr[-1]:.4e} \")\n    print(f\"train: loss - {loss:.3f} acc - {acc:.3f} auc - {auc:.3f}\")\n    val_loss, val_acc, val_auc = valid_epoch(model, \n                                             val_loader, \n                                             criterion, \n                                             device)\n    print(f\"valid: loss - {val_loss:.3f} acc - {val_acc:.3f} auc - {val_auc:.3f}\")\n    \n    if best_auc < val_auc:\n        print(f'epoch - {epoch + 1} best model with val auc: {val_auc}')\n        best_auc = val_auc\n        best_epoch = epoch\n        if best_auc > saving_threshold:\n            model_name = f\"sakt_layer_{NUM_LAYERS}_head_{NUM_HEADS}\"\n            model_name += f\"_embed_{NUM_EMBED}_seq_{MAX_SEQ}\"\n            model_name += f\"_auc_{val_auc:.4f}.pt\"\n            torch.save(model.state_dict(), model_name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nplt.plot(lr)\nplt.xlabel('epoch')\nplt.ylabel('Learning rate')\nplt.suptitle(f'{scheduler.name}')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del dataset, valid_dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Test"},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, samples, test_df, n_skill, max_seq=MAX_SEQ): #HDKIM 100\n        super(TestDataset, self).__init__()\n        self.samples = samples\n        self.user_ids = [x for x in test_df[\"user_id\"].unique()]\n        self.test_df = test_df\n        self.n_skill = n_skill\n        self.max_seq = max_seq\n\n    def __len__(self):\n        return self.test_df.shape[0]\n\n    def __getitem__(self, index):\n        test_info = self.test_df.iloc[index]\n\n        user_id = test_info[\"user_id\"]\n        target_id = test_info[\"content_id\"]\n\n        q = np.zeros(self.max_seq, dtype=int)\n        qa = np.zeros(self.max_seq, dtype=int)\n\n        if user_id in self.samples.index:\n            q_, qa_ = self.samples[user_id]\n            \n            seq_len = len(q_)\n\n            if seq_len >= self.max_seq:\n                q = q_[-self.max_seq:]\n                qa = qa_[-self.max_seq:]\n            else:\n                q[-seq_len:] = q_\n                qa[-seq_len:] = qa_          \n        \n        x = np.zeros(self.max_seq-1, dtype=int)\n        x = q[1:].copy()\n        x += (qa[1:] == 1) * self.n_skill\n        \n        questions = np.append(q[2:], [target_id])\n        \n        return x, questions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def find_files(name, path):\n#     result = []\n#     for root, dirs, files in os.walk(path):\n#         for _file in files:\n#             if name in _file:\n#                 result.append(os.path.join(root, _file))\n#     return result\n\n\n# try: \n#     model = SAKTModel(n_skill, \n#                   embed_dim=NUM_EMBED, \n#                   max_seq=MAX_SEQ, \n#                   enc_layers=NUM_LAYERS, \n#                   heads=NUM_HEADS, dropout=0.1)\n\n#     model.to(device)\n#     print(\"Loading the best AUC model trained in the current notebook\")\n#     model_name = f\"../working/sakt_layer_{NUM_LAYERS}_head_{NUM_HEADS}\"\n#     model_name += f\"_embed_{NUM_EMBED}_seq_{MAX_SEQ}\"\n#     model_name += f\"_auc_{val_auc:.4f}.pt\"\n#     model.load_state_dict(torch.load(model_name, map_location=device))\n# except:\n# #     model_files = find_files('sakt', '../working/')\n# #     print(f\"Loading {model_files[-1]}\")\n# #     model.load_state_dict(torch.load(model_files[-1], map_location=device))\n#     print(\"Use current trained model.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nmodel.eval()\nprev_test_df = None\n\nfor (test_df, sample_prediction_df) in tqdm(iter_test):\n    if (prev_test_df is not None) & (psutil.virtual_memory().percent < 95):\n        prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])\n        prev_test_df = prev_test_df[prev_test_df.content_type_id == False]\n        \n        prev_group = prev_test_df[['user_id', \n                                   'content_id', \n                                   'answered_correctly']]\\\n            .groupby('user_id').apply(lambda r: (\n                                            r['content_id'].values,\n                                            r['answered_correctly'].values))\n        for prev_user_id in prev_group.index:\n            if prev_user_id in group.index:\n                group[prev_user_id] = (\n                    np.append(group[prev_user_id][0], \n                              prev_group[prev_user_id][0])[-MAX_SEQ:], \n                    np.append(group[prev_user_id][1], \n                              prev_group[prev_user_id][1])[-MAX_SEQ:]\n                )\n \n            else:\n                group[prev_user_id] = (\n                    prev_group[prev_user_id][0], \n                    prev_group[prev_user_id][1]\n                )\n\n    prev_test_df = test_df.copy()\n    \n    test_df = test_df[test_df.content_type_id == False]\n    test_dataset = TestDataset(group, test_df, n_skill)\n    test_dataloader = DataLoader(test_dataset, batch_size=len(test_df), shuffle=False)\n    \n    outs = []\n\n    for item in tqdm(test_dataloader):\n        x = item[0].to(device).long()\n        target_id = item[1].to(device).long()\n\n        with torch.no_grad():\n            output, att_weight = model(x, target_id)\n        outs.extend(torch.sigmoid(output)[:, -1].view(-1).data.cpu().numpy())\n        \n    test_df['answered_correctly'] = outs\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, \n                            ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../working/submission.csv')\nsub['answered_correctly'].hist(bins=15)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}