{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Transformer in PyTorch\n\nThis is a debugged version of https://www.kaggle.com/adityaecdrid/pytorch-demystifying-transformers that added inference. In the original one the author mistakenly sorted the dataframe during feature engineering which result the CV being okay but upon submission, you only get 0.5 AUC like a random guess. \n\nFor inference, I added a function to add new user, however, it is somewhat slow....\n\nA scheduler is also added to the template. There are changes here and there to fix the original kernel's hardcoded hyperparameters.\n\nCV is fixed as well, however, apparently there is a leakage as the CV is much higher than LB even in the debugged version.\n\nReference:\n\n- CV: https://www.kaggle.com/marisakamozz/cv-strategy-in-the-kaggle-environment\n- @adityaecdrid 's kernel: https://www.kaggle.com/adityaecdrid/pytorch-demystifying-transformers\n- Host's arXiv preprints https://arxiv.org/abs/2002.07033, https://arxiv.org/abs/2010.12042\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pickle\nimport gc\n\nfrom time import time, sleep\nimport itertools\n\nfrom collections import deque, Counter\n\nfrom tqdm import tqdm\nfrom sklearn.metrics import roc_auc_score\nfrom typing import List\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom pathlib import Path\n\nRANDOM_SEED = 42\nnp.random.seed(RANDOM_SEED)\ngc.enable()\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport math\n\nfrom torch.autograd import Variable\nfrom torch.nn import TransformerEncoder, TransformerEncoderLayer\nfrom torch.nn.modules.dropout import Dropout\nfrom torch.nn.modules.activation import MultiheadAttention\nfrom torch.nn.modules.normalization import LayerNorm\nfrom torch.utils.data import DataLoader, Dataset\n\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\n\nTRAIN_DTYPES = {\n    # 'row_id': np.uint32,\n    'timestamp': np.uint64,\n    'user_id': np.uint32,\n    'content_id': np.uint16,\n    'content_type_id': np.uint8,\n    'task_container_id': np.uint16,\n    'user_answer': np.int8,\n    'answered_correctly': np.int8,\n    'prior_question_elapsed_time': np.float32,\n    'prior_question_had_explanation': 'boolean'\n}\nTRAIN_COLS = list(TRAIN_DTYPES.keys())\n\nDATA_DIR = Path('../input/riiid-test-answer-prediction')\nTRAIN_PATH = DATA_DIR / 'train.csv'\nQUESTIONS_PATH = DATA_DIR / 'questions.csv'\nLECTURES_PATH = DATA_DIR / 'lectures.csv'\n\n# Some global variables\n# this parameter denotes how many last seen content_ids I am going to consider <aka the max_seq_len or the window size>.\nMAX_SEQ = 100 \nTQDM_INT = 4 # interval for tqdm to update the pbar\nPAD = 0\nFILLNA_VAL = 14_000\nn_skill = 13523\nEPOCHS = 20\nBATCH_SIZE = 512\nVAL_BATCH_SIZE = 2048\nTEST_BATCH_SIZE = 4096\n\nDEBUG = False\nNROWS_TRAIN, NROWS_VAL = (20_000_000, 200_000) if not DEBUG else (2_000_000, 20_000)\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load the Dataset\n\nWe load the train, cv split directly from @marisakamozz 's awesome kernel: https://www.kaggle.com/marisakamozz/cv-strategy-in-the-kaggle-environment"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_questions = pd.read_csv('../input/riiid-test-answer-prediction/questions.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndf_train = pd.read_parquet('../input/cv-strategy-in-the-kaggle-environment/cv3_train.parquet')\ndf_train = df_train[TRAIN_COLS]\ndf_train = df_train[:NROWS_TRAIN]\ngc.collect();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndf_valid = pd.read_parquet('../input/cv-strategy-in-the-kaggle-environment/cv3_valid.parquet')\ndf_valid = df_valid[TRAIN_COLS]\ndf_valid = df_valid[:NROWS_VAL]\ngc.collect();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_feats(data_df, question_df, max_seq=MAX_SEQ):\n    '''\n    Using a deque as it automatically limits the max_size as per the Data Strucutre's defination itself\n    so we don't need to manage that...\n    '''\n    data_df = data_df.copy()\n\n    data_df['prior_question_had_explanation'] = \\\n            data_df['prior_question_had_explanation'].astype(np.float16).fillna(0).astype(np.int8)\n    data_df = data_df.loc[data_df['content_type_id'] == 0]\n    \n    part_ids_map = dict(zip(question_df['question_id'], question_df['part']))\n    data_df['part_id'] = data_df['content_id'].map(part_ids_map)\n    \n    data_df[\"prior_question_elapsed_time\"].fillna(FILLNA_VAL, inplace=True) \n    # FILLNA_VAL different than all current values\n    data_df[\"prior_question_elapsed_time\"] = data_df[\"prior_question_elapsed_time\"] // 1000\n\n    \n    df = {}\n    user_id_to_idx = {}\n    # the sort needs to be False here for the test\n    grp = data_df.groupby(\"user_id\", sort=False).tail(max_seq) # Select MAX_SEQ rows of each user.\n    grp_user = grp.groupby(\"user_id\", sort=False)\n    num_user_id_grp = len(grp_user)\n    # with tqdm(total=num_user_id_grp) as pbar:\n    for idx, row in grp_user.agg({\n        \"content_id\":list, \n        \"answered_correctly\":list, \n        \"task_container_id\":list, \n        \"part_id\":list, \n        \"prior_question_elapsed_time\":list\n        }).reset_index().iterrows():\n        # here we make a split whether a user has more than equal to 100 entries or less than that\n        # if it's less than max_seq, then we need to PAD it using the PAD token defined as 0 by me in this cell block\n        # also, padded will be True where we have done padding obviously, rest places it's False.\n        if len(row[\"content_id\"]) >= max_seq:\n            df[idx] = {\n                \"user_id\": row[\"user_id\"],\n                \"content_id\" : deque(row[\"content_id\"], maxlen=max_seq),\n                \"answered_correctly\" : deque(row[\"answered_correctly\"], maxlen=max_seq),\n                \"task_container_id\" : deque(row[\"task_container_id\"], maxlen=max_seq),\n                \"prior_question_elapsed_time\" : deque(row[\"prior_question_elapsed_time\"], maxlen=max_seq),\n                \"part_id\": deque(row[\"part_id\"], maxlen=max_seq),\n                \"padded\" : deque([False]*max_seq, maxlen=max_seq)\n            }\n        else:\n            # we have to pad...\n            num_padding = max_seq-len(row[\"content_id\"])\n            padding = [PAD]*num_padding\n            df[idx] = {\n            \"user_id\": row[\"user_id\"],\n            \"content_id\" : deque(row[\"content_id\"] + padding, maxlen=max_seq),\n            \"answered_correctly\" : deque(row[\"answered_correctly\"] + padding, maxlen=max_seq),\n            \"task_container_id\" : deque(row[\"task_container_id\"] + padding, maxlen=max_seq),\n            \"prior_question_elapsed_time\" : deque(row[\"prior_question_elapsed_time\"] + padding, maxlen=max_seq),\n            \"part_id\": deque(row[\"part_id\"] + padding, maxlen=max_seq),\n            \"padded\" : deque([False]*len(row[\"content_id\"]) + [True]*num_padding, maxlen=max_seq)\n            }\n        user_id_to_idx[row[\"user_id\"]] = idx\n        # if in future a new user comes, we will just increase the counts as of now... <WIP>\n    return df, user_id_to_idx","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nprint(f\"train rows: {len(df_train)}    valid rows: {len(df_valid)} \")\nd, user_id_to_idx_train = get_feats(df_train, df_questions)\nd_val, _ = get_feats(df_valid, df_questions)\nif not DEBUG:\n    del df_train, df_valid\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# A Minimal Transformer Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class TransformerModel(nn.Module):\n\n    def __init__(self, ninp:int=32, nhead:int=2, nhid:int=64, nlayers:int=2, dropout:float=0.3):\n        '''\n        nhead -> number of heads in the transformer multi attention thing.\n        nhid -> the number of hidden dimension neurons in the model.\n        nlayers -> how many layers we want to stack.\n        '''\n        super(TransformerModel, self).__init__()\n        self.src_mask = None\n        encoder_layers = TransformerEncoderLayer(d_model=ninp, \n                                                 nhead=nhead, \n                                                 dim_feedforward=nhid, \n                                                 dropout=dropout, \n                                                 activation='relu')\n        self.transformer_encoder = TransformerEncoder(encoder_layer=encoder_layers, num_layers=nlayers)\n        self.exercise_embeddings = nn.Embedding(num_embeddings=13523, embedding_dim=ninp) # exercise_id\n        self.pos_embedding = nn.Embedding(ninp, ninp) # positional embeddings\n        self.part_embeddings = nn.Embedding(num_embeddings=7+1, embedding_dim=ninp) # part_id_embeddings\n        self.prior_question_elapsed_time = nn.Embedding(num_embeddings=301, embedding_dim=ninp) # prior_question_elapsed_time\n        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n        self.ninp = ninp\n        self.decoder = nn.Linear(ninp, 2)\n        self.init_weights()\n        self.name = 'transformer'\n\n    def init_weights(self):\n        initrange = 0.1\n        # init embeddings\n        self.exercise_embeddings.weight.data.uniform_(-initrange, initrange)\n        self.part_embeddings.weight.data.uniform_(-initrange, initrange)\n        self.prior_question_elapsed_time.weight.data.uniform_(-initrange, initrange)\n        self.decoder.bias.data.zero_()\n        self.decoder.weight.data.uniform_(-initrange, initrange)\n\n    def forward(self, content_id, part_id, prior_question_elapsed_time=None, mask_src=None):\n        '''\n        S is the sequence length, N the batch size and E the Embedding Dimension (number of features).\n        src: (S, N, E)\n        src_mask: (S, S)\n        src_key_padding_mask: (N, S)\n        padding mask is (N, S) with boolean True/False.\n        SRC_MASK is (S, S) with float(’-inf’) and float(0.0).\n        '''\n\n        embedded_src = self.exercise_embeddings(content_id) + \\\n        self.pos_embedding(torch.arange(0, content_id.shape[1]).to(self.device).unsqueeze(0).repeat(content_id.shape[0], 1)) + \\\n        self.part_embeddings(part_id) + self.prior_question_elapsed_time(prior_question_elapsed_time) # (N, S, E)\n        embedded_src = embedded_src.transpose(0, 1) # (S, N, E)\n        \n        _src = embedded_src * np.sqrt(self.ninp)\n        \n        output = self.transformer_encoder(src=_src, src_key_padding_mask=mask_src)\n        output = self.decoder(output)\n        output = output.transpose(1, 0)\n        return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Riiid(Dataset):\n    def __init__(self, d):\n        super(Riiid, self).__init__()\n        self.d = d\n    \n    def __len__(self):\n        return len(self.d)\n    \n    def __getitem__(self, idx):\n        # you can return a dict of these as well etc etc...\n        # remember the order\n        return idx, self.d[idx][\"content_id\"], self.d[idx][\"task_container_id\"], \\\n    self.d[idx][\"part_id\"], self.d[idx][\"prior_question_elapsed_time\"], self.d[idx][\"padded\"], \\\n    self.d[idx][\"answered_correctly\"]\n\n    \ndef collate_fn(batch):\n    _, content_id, task_id, part_id, prior_question_elapsed_time, padded, labels = zip(*batch)\n    content_id = torch.Tensor(content_id).long()\n    task_id = torch.Tensor(task_id).long()\n    part_id = torch.Tensor(part_id).long()\n    prior_question_elapsed_time = torch.Tensor(prior_question_elapsed_time).long()\n    padded = torch.Tensor(padded).bool()\n    labels = torch.Tensor(labels)\n    # remember the order\n    return content_id, task_id, part_id, prior_question_elapsed_time, padded, labels\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_train = Riiid(d=d)\ndataset_val = Riiid(d=d_val)\n\ntrain_loader = DataLoader(dataset=dataset_train, \n                          shuffle=True,\n                          batch_size=BATCH_SIZE, \n                          collate_fn=collate_fn)\n\nval_loader = DataLoader(dataset=dataset_val, shuffle=False,\n                        batch_size=VAL_BATCH_SIZE, \n                        collate_fn=collate_fn, \n                        drop_last=False)\nprint(f'Train iters: {len(train_loader)};  val iters: {len(val_loader)}')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true},"cell_type":"code","source":"sample = next(iter(train_loader)) # next returns the next element in an iterator\n# dummy check\nprint(sample)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true},"cell_type":"code","source":"# createing the mdoel\n\ndef get_num_params(model):\n    model_parameters = filter(lambda p: p.requires_grad, model.parameters())\n    params = sum([np.prod(p.size()) for p in model_parameters])\n    return params\n\nmodel = TransformerModel(ninp=128, nhead=8, nhid=128, nlayers=3, dropout=0.3)\n\nprint(f\"Number of parameters: {get_num_params(model)} \\n\")\nprint(model) # look into it!","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train models"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_epoch(model, train_iterator, optimizer, criterion):\n    model.train()\n\n    train_loss = []\n    num_corrects = 0\n    num_total = 0\n    label_all = []\n    pred_all = []\n    len_dataset = len(train_iterator)\n\n    with tqdm(total=len_dataset) as pbar:\n        for idx,batch in enumerate(train_iterator):\n            content_id, _, part_id, prior_question_elapsed_time, mask, labels = batch\n            content_id = Variable(content_id.to(device))\n            part_id = Variable(part_id.to(device))\n            prior_question_elapsed_time = Variable(prior_question_elapsed_time.to(device))\n            mask = Variable(mask.to(device))\n            labels = Variable(labels.to(device).long())\n            optimizer.zero_grad()\n            \n            with torch.set_grad_enabled(mode=True):\n                output = model(content_id, part_id, prior_question_elapsed_time, mask)\n                # output is (N,S,2) # i am working on it\n                \n                # loss = criterion(output[:,:,1], labels) # BCEWithLogitsLoss\n                loss = criterion(output.reshape(-1, 2), labels.reshape(-1)) # Flatten and use crossEntropy\n                loss.backward()\n                optimizer.step()\n\n                train_loss.append(loss.cpu().detach().data.numpy())\n\n            pred_probs = torch.softmax(output[~mask], dim=1)\n            pred = torch.argmax(pred_probs, dim=1)\n            labels = labels[~mask]\n            num_corrects += (pred == labels).sum().item()\n            num_total += len(labels)\n\n            label_all.extend(labels.reshape(-1).data.cpu().numpy())\n            # pred_all.extend(pred.reshape(-1).data.cpu().numpy())\n            pred_all.extend(pred_probs[:,1].reshape(-1).data.cpu().numpy()) # use probability to do auc\n\n            if idx % TQDM_INT == 0:\n                pbar.set_description(f'loss - {train_loss[-1]:.4f}')\n                pbar.update(TQDM_INT)\n\n    acc = num_corrects / num_total\n    auc = roc_auc_score(label_all, pred_all)\n    loss = np.mean(train_loss)\n\n    return loss, acc, auc\n\n\ndef valid_epoch(model, valid_iterator, criterion):\n    model.eval()\n    valid_loss = []\n    num_corrects = 0\n    num_total = 0\n    label_all = []\n    pred_all = []\n    len_dataset = len(valid_iterator)\n\n    for idx, batch in enumerate(valid_iterator):\n        content_id, _, part_id, prior_question_elapsed_time, mask, labels = batch\n        content_id = Variable(content_id.to(device))\n        part_id = Variable(part_id.to(device))\n        prior_question_elapsed_time = Variable(prior_question_elapsed_time.to(device))\n        mask = Variable(mask.to(device))\n        labels = Variable(labels.to(device).long())\n        with torch.set_grad_enabled(mode=False):\n            output = model(content_id, part_id, prior_question_elapsed_time, mask)\n            loss = criterion(output.reshape(-1, 2), labels.reshape(-1)) # Flatten and use crossEntropy\n\n        # New: crossEntropy loss\n        valid_loss.append(loss.cpu().detach().data.numpy())\n        pred_probs = torch.softmax(output[~mask], dim=1)\n        pred = torch.argmax(pred_probs, dim=1)\n\n        # Old: BCE loss\n        # output_prob = output[:,:,1]\n        # pred = (output_prob >= 0.50)\n        # print(output.shape, labels.shape) # torch.Size([N, S, 2]) torch.Size([N, S])\n        # _, predicted_classes = torch.max(output[:,:,].data, 1)\n\n        labels = labels[~mask]\n        num_corrects += (pred == labels).sum().item()\n        num_total += len(labels)\n        label_all.extend(labels.reshape(-1).data.cpu().numpy())\n        pred_all.extend(pred_probs[:,1].reshape(-1).data.cpu().numpy()) # use probability to do auc\n\n    acc = num_corrects / num_total\n    auc = roc_auc_score(label_all, pred_all)\n    loss = np.mean(valid_loss)\n\n    return loss, acc, auc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"losses = []\nhistory = []\nauc_max = 0\nmodel.to(device)\n# criterion = nn.BCEWithLogitsLoss().to(device)\ncriterion = nn.CrossEntropyLoss().to(device)\nlr = 1e-3 \noptimizer = torch.optim.Adam(model.parameters(), lr=lr)\nscheduler = ReduceLROnPlateau(optimizer, 'min', patience=5, threshold=1e-5)\n\nfor epoch in range(EPOCHS):\n    train_loss, train_acc, train_auc = train_epoch(model, train_loader, optimizer, criterion)\n    valid_loss, valid_acc, valid_auc = valid_epoch(model, val_loader, criterion)\n    \n    if epoch >= 1:\n        scheduler.step(valid_auc)\n        \n    print(f\"\\n\\n[Epoch {epoch}/{EPOCHS}]\")\n    print(f\"\\nTrain: loss - {train_loss:.4f} acc - {train_acc:.4f} auc - {train_auc:.4f}\")\n    print(f\"\\nValid: loss - {valid_loss:.4f} acc - {valid_acc:.4f} auc - {valid_auc:.4f}\")\n    lr = optimizer.param_groups[0]['lr']\n    history.append({\"epoch\":epoch, \"lr\": lr, \n                    **{\"train_auc\": train_auc, \"train_acc\": train_acc}, \n                    **{\"valid_auc\": valid_auc, \"valid_acc\": valid_acc}})\n\n    if valid_auc > auc_max:\n        print(f\"\\n[Epoch {epoch}/{EPOCHS}] auc improved from {auc_max:.4f} to {valid_auc:.4f}\") \n        print(\"saving model ...\")\n        auc_max = valid_auc\n        torch.save(model.state_dict(), f\"{model.name}_auc_{valid_auc:.4f}.pt\")\n\nwith open(f'history_auc_{valid_auc:.4f}.pickle', 'wb') as handle:\n    pickle.dump(history, handle, protocol=pickle.HIGHEST_PROTOCOL)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_feats_test(data_df, question_df, max_seq=MAX_SEQ):\n\n    data_df = data_df.copy()\n    \n    data_df['prior_question_had_explanation'] = \\\n            data_df['prior_question_had_explanation'].astype(np.float16).fillna(0).astype(np.int8)\n    data_df = data_df.loc[data_df['content_type_id'] == 0]\n    \n    part_ids_map = dict(zip(question_df['question_id'], question_df['part']))\n    data_df['part_id'] = data_df['content_id'].map(part_ids_map)\n    \n    data_df[\"prior_question_elapsed_time\"].fillna(FILLNA_VAL, inplace=True) \n    # FILLNA_VAL different than all current values\n    data_df[\"prior_question_elapsed_time\"] = data_df[\"prior_question_elapsed_time\"] // 1000\n\n    \n    df = {}\n    user_id_to_idx = {}\n     # the sort needs to be False here for the test\n    grp = data_df.groupby(\"user_id\", sort=False).tail(max_seq) # Select MAX_SEQ rows of each user.\n    grp_user = grp.groupby(\"user_id\", sort=False)\n    num_user_id_grp = len(grp_user)\n\n    for idx, row in grp_user.agg({\n        \"content_id\":list, \n#         \"answered_correctly\":list, \n        \"task_container_id\":list, \n        \"part_id\":list, \n        \"prior_question_elapsed_time\":list\n        }).reset_index().iterrows():\n        # here we make a split whether a user has more than equal to 100 entries or less than that\n        # if it's less than max_seq, then we need to PAD it using the PAD token defined as 0 by me in this cell block\n        # also, padded will be True where we have done padding obviously, rest places it's False.\n        if len(row[\"content_id\"]) >= max_seq:\n            df[idx] = {\n                \"user_id\": row[\"user_id\"],\n                \"content_id\" : deque(row[\"content_id\"], maxlen=max_seq),\n#                 \"answered_correctly\" : deque(row[\"answered_correctly\"], maxlen=max_seq),\n                \"task_container_id\" : deque(row[\"task_container_id\"], maxlen=max_seq),\n                \"prior_question_elapsed_time\" : deque(row[\"prior_question_elapsed_time\"], maxlen=max_seq),\n                \"part_id\": deque(row[\"part_id\"], maxlen=max_seq),\n                \"padded\" : deque([False]*max_seq, maxlen=max_seq),\n                \"pred_mask\": deque([True]*max_seq, maxlen=max_seq),\n            }\n        else:\n            # we have to pad...\n            num_padding = max_seq-len(row[\"content_id\"])\n            padding = [PAD]*num_padding\n            df[idx] = {\n            \"user_id\": row[\"user_id\"],\n            \"content_id\" : deque(row[\"content_id\"] + padding, maxlen=max_seq),\n#             \"answered_correctly\" : deque(row[\"answered_correctly\"] + padding, maxlen=max_seq),\n            \"task_container_id\" : deque(row[\"task_container_id\"] + padding, maxlen=max_seq),\n            \"prior_question_elapsed_time\" : deque(row[\"prior_question_elapsed_time\"] + padding, maxlen=max_seq),\n            \"part_id\": deque(row[\"part_id\"] + padding, maxlen=max_seq),\n            \"padded\" : deque([False]*len(row[\"content_id\"]) + [True]*num_padding, maxlen=max_seq),\n            \"pred_mask\" : deque([True]*len(row[\"content_id\"]) + [False]*num_padding, maxlen=max_seq),\n            }\n        user_id_to_idx[row[\"user_id\"]] = idx\n        # if in future a new user comes, we will just increase the counts as of now... <WIP>\n    return df, user_id_to_idx\n\nclass RiiidTest(Dataset):\n    \n    def __init__(self, d):\n        super(RiiidTest, self).__init__()\n        self.d = d\n    \n    def __len__(self):\n        return len(self.d)\n    \n    def __getitem__(self, idx):\n        # you can return a dict of these as well etc etc...\n        # remember the order\n        return idx, self.d[idx][\"content_id\"], self.d[idx][\"task_container_id\"], \\\n    self.d[idx][\"part_id\"], self.d[idx][\"prior_question_elapsed_time\"], \\\n    self.d[idx][\"padded\"], self.d[idx][\"pred_mask\"]\n\ndef collate_fn_test(batch):\n    _, content_id, task_id, part_id, prior_question_elapsed_time, padded, pred_mask = zip(*batch)\n    content_id = torch.Tensor(content_id).long()\n    task_id = torch.Tensor(task_id).long()\n    part_id = torch.Tensor(part_id).long()\n    prior_question_elapsed_time = torch.Tensor(prior_question_elapsed_time).long()\n    padded = torch.Tensor(padded).bool()\n    pred_mask = torch.Tensor(pred_mask).bool()\n    # remember the order\n    return content_id, task_id, part_id, prior_question_elapsed_time, padded, pred_mask","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Function for update user\nThis is the function for updating the users, however, it is slow...."},{"metadata":{"trusted":true},"cell_type":"code","source":"def update_users(d, d_new, uid_to_idx, uid_to_idx_new, test_flag=False):\n    '''\n    Add the user's features from d to d_new\n    During inference:\n    1. add user's feature from previous df to train df (old=prev test, new=train)\n    2. after reading current test df, add user's from train df to test df (old=train, new=current test)\n\n    '''\n    feature_cols =  ['content_id', \n#             'answered_correctly', \n            'task_container_id', \n            'prior_question_elapsed_time',\n            'part_id',\n            ]\n    mask_cols = ['padded',\n            'pred_mask']\n    for uid_test, idx_test in uid_to_idx_new.items():\n        if uid_test in uid_to_idx.keys():\n            idx_train = uid_to_idx[uid_test]\n            old_user_mask = [not s for s in d[idx_train]['padded']]\n\n            old_user = []\n            for col in feature_cols:\n                old_user.append(np.array(d[idx_train][col])[old_user_mask])\n            \n            new_user_mask = [not s for s in d_new[idx_test]['padded']]\n            len_user_pred = sum(new_user_mask)\n            # print(len_user_pred)\n            for idx_feat, feat in enumerate(feature_cols):\n                new_user_update = np.append(old_user[idx_feat],\n                                            np.array(d_new[idx_test][feat])[new_user_mask])\n                len_user = len(new_user_update) # the length of the current user after update\n                # print(len_user)\n                if len_user >= MAX_SEQ:\n                    d_new[idx_test][feat] = deque(new_user_update[-MAX_SEQ:], maxlen=MAX_SEQ)\n                    \n                else:\n                    num_padding = MAX_SEQ - len_user\n                    d_new[idx_test][feat] = deque(np.append(new_user_update, \n                    np.zeros(num_padding, dtype=int)), maxlen=MAX_SEQ)\n                    \n            if test_flag:\n                assert MAX_SEQ >= len_user_pred\n                if len_user >= MAX_SEQ:\n                    d_new[idx_test]['padded'] = deque([False]*MAX_SEQ, maxlen=MAX_SEQ)\n                    d_new[idx_test]['pred_mask'] = \\\n                        deque([False]*(MAX_SEQ-len_user_pred) + [True]*len_user_pred, maxlen=MAX_SEQ)\n                else:       \n                    num_padding = MAX_SEQ - len_user\n                    \n                    d_new[idx_test]['padded'] = deque([False]*len_user + [True]*num_padding, maxlen=MAX_SEQ)\n\n                    d_new[idx_test]['pred_mask'] = \\\n                                deque([False]*(len_user-len_user_pred) + [True]*len_user_pred + [False]*num_padding, \n                                maxlen=MAX_SEQ)\n    return d_new\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nprev_test_df = None\n\nfor idx, (current_test, current_prediction_df) in enumerate(iter_test):\n\n    '''\n    concised iter_env\n    '''\n    if prev_test_df is not None:\n        '''Making use of answers to previous questions'''\n        answers = eval(current_test[\"prior_group_answers_correct\"].iloc[0])\n        responses = eval(current_test[\"prior_group_responses\"].iloc[0])\n        prev_test_df['answered_correctly'] = answers\n        prev_test_df['user_answer'] = responses\n\n        prev_test_df = prev_test_df[prev_test_df['content_type_id'] == False]\n        d_prev, user_id_to_idx_prev = get_feats(prev_test_df, df_questions)\n        d = update_users(d_prev, d, user_id_to_idx_prev, user_id_to_idx_train)\n\n    prev_test_df = current_test.copy()\n\n    d_test, user_id_to_idx_test = get_feats_test(current_test, df_questions, max_seq=MAX_SEQ)\n    d_test = update_users(d, d_test, user_id_to_idx_train, user_id_to_idx_test, test_flag=True)\n    dataset_test = RiiidTest(d=d_test)\n    test_loader = DataLoader(dataset=dataset_test, \n                                batch_size=TEST_BATCH_SIZE, \n                                collate_fn=collate_fn_test, shuffle=False, drop_last=False)\n\n    # the problem with current feature gen is that \n    # using groupby user_id sorts the user_id and makes it different from the \n    # test_df's order\n\n    output_all = []\n    for _, batch in enumerate(test_loader):\n        content_id, _, part_id, prior_question_elapsed_time, mask, pred_mask = batch\n\n        content_id = Variable(content_id.cuda())\n        part_id = Variable(part_id.cuda())\n        prior_question_elapsed_time = Variable(prior_question_elapsed_time.cuda())\n        mask = Variable(mask.cuda())\n\n        with torch.no_grad():\n            output = model(content_id, part_id, prior_question_elapsed_time, mask)\n\n        pred_probs = torch.softmax(output[pred_mask], dim=1)\n        output_all.extend(pred_probs[:,1].reshape(-1).data.cpu().numpy())\n    '''prediction code ends'''\n\n    current_test['answered_correctly'] = output_all\n    env.predict(current_test[['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.set()\nsub = pd.read_csv('../working/submission.csv')\nsub['answered_correctly'].hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if DEBUG:\n    (current_test, current_prediction_df) = next(iter_test)\n\n    if prev_test_df is not None:\n        '''Making use of answers to previous questions'''\n        answers = eval(current_test[\"prior_group_answers_correct\"].iloc[0])\n        responses = eval(current_test[\"prior_group_responses\"].iloc[0])\n        prev_test_df['answered_correctly'] = answers\n        prev_test_df['user_answer'] = responses\n\n        prev_test_df = prev_test_df[prev_test_df['content_type_id'] == False]\n        d_prev, user_id_to_idx_prev = get_feats(prev_test_df, df_questions)\n        d = update_users(d_prev, d, user_id_to_idx_prev, user_id_to_idx_train)\n\n    prev_test_df = current_test.copy()\n\n    d_test, user_id_to_idx_test = get_feats_test(current_test, df_questions, max_seq=MAX_SEQ)\n    d_test = update_users(d, d_test, user_id_to_idx_train, user_id_to_idx_test, test_flag=True)\n    dataset_test = RiiidTest(d=d_test)\n    test_loader = DataLoader(dataset=dataset_test, \n                                batch_size=TEST_BATCH_SIZE, \n                                collate_fn=collate_fn_test, shuffle=False, drop_last=False)\n\n    # the problem with current feature gen is that \n    # using groupby user_id sorts the user_id and makes it different from the \n    # test_df's order\n\n    output_all = []\n    for _, batch in enumerate(test_loader):\n        content_id, _, part_id, prior_question_elapsed_time, mask, pred_mask = batch\n\n        content_id = Variable(content_id.cuda())\n        part_id = Variable(part_id.cuda())\n        prior_question_elapsed_time = Variable(prior_question_elapsed_time.cuda())\n        mask = Variable(mask.cuda())\n\n        with torch.no_grad():\n            output = model(content_id, part_id, prior_question_elapsed_time, mask)\n\n        pred_probs = torch.softmax(output[pred_mask], dim=1)\n        output_all.extend(pred_probs[:,1].reshape(-1).data.cpu().numpy())\n    '''prediction code ends'''\n\n    current_test['answered_correctly'] = output_all\n    env.predict(current_test[['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}