{"cells":[{"metadata":{},"cell_type":"markdown","source":"###### * Base Source: https://www.kaggle.com/wangsg/a-self-attentive-model-for-knowledge-tracing\n* My First Work: https://www.kaggle.com/leadbest/sakt-self-attentive-knowledge-tracing-submitter\n\n1. Version 1: State Updates -> LB 0.765\n2. Version 3: Random Selection of User Interactions -> LB 0.768\n3. Version 6: Small Optimization -> LB 0.771?"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import gc\nimport random\nfrom tqdm import tqdm\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import train_test_split\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.utils.rnn as rnn_utils\nfrom torch.autograd import Variable\nfrom torch.utils.data import Dataset, DataLoader","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"import datatable as dt\n\ndata_types_dict = {\n    'row_id':'int64',\n #   'timestamp': 'int64',\n    'user_id': 'int32', \n    'content_id': 'int16', \n    'content_type_id':'int8', \n    #'task_container_id': 'int16',\n    #'user_answer': 'int8',\n    'answered_correctly': 'int8', \n    #'prior_question_elapsed_time': 'float32', \n    #'prior_question_had_explanation': 'bool'\n}\ntarget = 'answered_correctly'\ndata_df = dt.fread('../input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"INPUT = '../../input/riiid-test-answer-prediction'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#HDKIM\nMAX_SEQ = 100\n#HDKIMHDKIM\nseq_length =100","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load data"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"skills = data_df[\"content_id\"].unique()\nn_skill = len(skills)+1\nprint(\"number skills\", len(skills))"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"\ndel data_df\ngc.collect()"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom numba import jit","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preprocess"},{"metadata":{"trusted":true},"cell_type":"code","source":"#HDKIM\nimport random\nrandom.seed(1)\n#HDKIMHDKIM","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_SEQ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define model"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass SAINTPLUSModel(nn.Module):\n    def __init__(self, n_skill,n_correctness, max_seq=MAX_SEQ, num_heads=8, embed_dim=128): #HDKIM 100->MAX_SEQ\n        super(SAINTPLUSModel, self).__init__()\n        \n        \n        self.n_skill = n_skill\n        self.n_part = 9\n        self.n_et = 302\n        self.n_pos=101\n        self.n_correctness=4\n        \n        \n        self.embed_dim = embed_dim\n\n        self.num_heads = num_heads\n\n        self.correctness_embedding = nn.Embedding(self.n_correctness, embed_dim)\n       \n        self.pos_embedding = nn.Embedding(self.n_pos, embed_dim)\n        \n        self.ex_embedding = nn.Embedding(self.n_skill, embed_dim)\n        \n        self.part_embedding = nn.Embedding(self.n_part, embed_dim)\n        \n\n#         self.SAKT1 = SAKTmodule(embed_dim=embed_dim)\n#         self.SAKT2 = SAKTmodule(embed_dim=embed_dim)\n#         self.SAKT3 = SAKTmodule(embed_dim=embed_dim)\n\n        self.encoder = nn.TransformerEncoder(nn.TransformerEncoderLayer(d_model=embed_dim,nhead=self.num_heads),num_layers=4)\n        \n        self.decoder = nn.TransformerDecoder(nn.TransformerDecoderLayer(d_model=embed_dim,nhead=self.num_heads),num_layers=4)\n        \n        #self.dropout = nn.Dropout(0.2)\n        \n       # self.elapT_weight = nn.Linear(1,embed_dim)\n        \n        self.pred = nn.Linear(embed_dim, 1)\n    \n    def forward(self,excercise,correctness, parts, src_pad_mask , tgt_pad_mask,src_mask, tgt_mask ,pos_id_e ,pos_id_d):\n        \n        \n    \n        \n     #  # pos_id_e = (torch.arange(x.size(1)) - torch.array(cum_id) ).unsqueeze(0).to(device)\n        \n    #  #  pos_id_d = (torch.arange(x.size(1)) - torch.array([0]+cum_id[:-1]) ).unsqueeze(0).to(device)\n\n        \n        \n     #   lag_embedding\n        \n      #  interaction_embedding\n        \n      #  self.lag_embeding()\n     #   self.elapsed_embedding()\n        \n       # print(elap_t.float().unsqueeze(- 1).shape)\n        #print(pos_id_d.min())\n        #print(pos_id_d.max())\n        \n        d = (self.correctness_embedding(correctness ) + self.pos_embedding(pos_id_d)).permute(1, 0, 2)\n        \n        e = (self.ex_embedding(excercise ) + self.pos_embedding(pos_id_e)+self.part_embedding(parts)).permute(1, 0, 2)\n\n       \n    \n        #print(e.shape)\n        \n    #    e  = self.encoder(e,mask=src_mask.repeat(self.num_heads,1,1),src_key_padding_mask =src_pad_mask)\n        \n       # print(e)\n        \n    #    d = self.decoder(d,e,tgt_mask = tgt_mask.repeat(self.num_heads,1,1),memory_mask = src_mask.repeat(self.num_heads,1,1),tgt_key_padding_mask = tgt_pad_mask, memory_key_padding_mask = src_pad_mask  )\n\n        \n       # print(e)\n        e  = self.encoder(e,mask=src_mask.repeat_interleave(self.num_heads,dim=0),src_key_padding_mask =src_pad_mask)\n        \n     #   print(e)\n        \n        d = self.decoder(d,e,tgt_mask = tgt_mask.repeat_interleave(self.num_heads,dim=0),memory_mask = src_mask.repeat_interleave(self.num_heads,dim=0),tgt_key_padding_mask = tgt_pad_mask, memory_key_padding_mask = src_pad_mask  )\n    # train_loss: 0.20374205638760184, lr:0.00064275000000000    \n    # valid_loss: 0.0952739624484875, auc: 0.9894335898260332 \n     \n        \n        \n        \n\n        return self.pred(e).transpose(1,0).squeeze(-1) , src_pad_mask\n    \n    \n    \n    \n    \n    \n\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#torch.repeat_interleave()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n#model = SAKTModel(n_skill, embed_dim=128)\n# optimizer = torch.optim.SGD(model.parameters(), lr=1e-3, momentum=0.99, weight_decay=0.005)\n#optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\n#criterion = nn.BCEWithLogitsLoss()\n\n#model.to(device)\n#criterion.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):\n    model.train()\n    final_loss = 0\n    \n    \n    tbar = tqdm(dataloader)\n    \n    for item in tbar:\n        optimizer.zero_grad()\n        \n        inp = item[0]\n        label = item[1].to(device)#.float().to(device)\n        \n        for k, v in inp.items():\n          \n            inp[k] = v.to(device)\n        \n       # print(inp)\n      \n        output, msk = model( **inp)\n        \n        #print(output)\n     #   print(output.dtype)\n    \n    \n       # print(msk.shape)\n      #  print(label.shape)\n     #   print(output.shape)\n        \n     #   print(inp['src_pad_mask'][0])\n     #   print(inp['src_mask'][0])\n     #   print(inp['src_mask'][:,0])\n     #   print(inp['tgt_pad_mask'][0])\n     #   print(inp['correctness'][0])\n        \n        #print((label > ))\n        \n      #  print(label.dtype)\n       # print(torch.any(label.flatten()[~msk.flatten()]<0))\n        \n       # print( (~msk).int().float())\n    \n      #  print(output.shape)\n     #   print((output * (~msk).int().float()).shape)\n     #   print((output * (~msk).int().float())[0])\n        \n        \n      #  print(((label * (~msk).int()).float() >=0).all())\n      \n       # loss = loss_fn(output * (~msk).int().float() , (label * (~msk).int()).float() )\n    \n        print( ((label * (~msk).int()).float()>=0).all())\n        \n        print((label * (~msk).int()).float()[0])\n        print(inp['correctness'][0])\n        print(label[0][(~msk[0])])\n        \n        loss = loss_fn(output , (label * (~msk).int()).float() , weight = (~msk).int())\n        \n        print(loss)\n        \n        loss.backward()\n        \n        optimizer.step()\n        \n        if not  scheduler.__class__ ==  torch.optim.lr_scheduler.ReduceLROnPlateau:\n            scheduler.step()\n        \n        final_loss += loss.item()\n        \n    final_loss /= len(dataloader)\n    \n    return final_loss\n\n\ndef valid_fn(model, scheduler, loss_fn, dataloader, device):\n    model.eval()\n    final_loss = 0\n    valid_preds = []\n    labels = []\n    \n    tbar = tqdm(dataloader)\n    \n    for item in tbar:\n     \n        inp = item[0]\n        label = item[1].to(device)\n        \n        for k, v in inp.items():\n          \n            inp[k] = v.to(device)\n\n        \n      \n        output, msk = model( **inp)\n        \n        loss = loss_fn(output , (label * (~msk).int()).float() , weight = (~msk).int())\n        \n        output = torch.sigmoid(output)\n        \n         \n        output = output.flatten()[(~msk).flatten()]\n        label = label.flatten()[(~msk).flatten()] \n        \n        \n        #labels.extend(label.view(-1).data.cpu().numpy())\n        #valid_preds.extend(output.view(-1).data.cpu().numpy())\n       \n        labels.extend(label.data.cpu().numpy())\n        valid_preds.extend(output.data.cpu().numpy())\n        \n        #valid_preds.append(output.sigmoid().detach().cpu().numpy())\n       # labels.append(label.detach().cpu().numpy()[0])\n        \n        if scheduler.__class__ ==  torch.optim.lr_scheduler.ReduceLROnPlateau:\n            scheduler.step(final_loss)\n    \n        \n        final_loss += loss.item()\n        \n    final_loss /= len(dataloader)\n    auc = roc_auc_score(labels, valid_preds)\n    \n    return final_loss,valid_preds,auc\n    \n\ndef inference_fn(model, dataloader, device):\n    model.eval()\n    outs = []\n    \n    tbar = tqdm(dataloader)\n    \n    for item in tbar:\n        optimizer.zero_grad()\n        \n        x = item[0].to(device).long()\n        target_id = item[1].to(device).long()\n        label = item[2].to(device).float()\n\n        with torch.no_grad():\n            output, _ = model(x, target_id)\n            \n        output = torch.sigmoid(output)\n        output = output[:, -1]\n        \n        outs.extend(output.data.cpu().numpy())\n    \n    return outs\n\n\ndef infer_features_fn(model, dataloader, device):\n    model.eval()\n    preds = []\n    \n    for item in dataloader:\n        inputs = data['x'].to(device)\n\n        with torch.no_grad():\n            outputs = model(inputs,mode='get_features')\n        \n#         print(len(outputs))\n        \n#         for i in range(len(outputs)):\n#             print(outputs[i].shape)\n            \n#         print(torch.cat(outputs,axis=1).shape)\n        \n        preds.append(torch.cat(outputs,axis=1).detach().cpu().numpy())\n        \n        \n        \n    preds = np.concatenate(preds)\n    \n    return preds\n   \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def accuracy_score():\n    \n    pred = (torch.sigmoid(output) >= 0.5).long()\n\n    num_corrects += (pred == label).sum().item()\n    num_total += len(label)\n\n    auc = roc_auc_score(labels, outs)\n    acc = num_corrects / num_total\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#chequear q quc no sea solo en ultima pregunta","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MaskedMSELoss(torch.nn.Module):\n    def __init__(self):\n        super(MaskedMSELoss, self).__init__()\n\n    def forward(self, input, target, mask):\n        diff2 = (torch.flatten(input) - torch.flatten(target)) ** 2.0\n        sum2 = 0.0\n        num = 0\n\n        flat_mask = torch.flatten(mask)\n        assert(len(flat_mask) == len(diff2))\n        for i in range(len(diff2)):\n            if flat_mask[i] == 1:\n                sum2 += diff2[i]\n                num += 1\n\n        return sum2 / num","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nEPOCHS = 2\n\nLR = 6e-4\n\nBATCH_SIZE = 128\n\nEARLY_STOPPING_STEPS = 5\n\nEARLY_STOP = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=42):\n    random.seed(seed)\n    #os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim.lr_scheduler import _LRScheduler\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class NoamLR(_LRScheduler):\n    \"\"\"\n    Implements the Noam Learning rate schedule. This corresponds to increasing the learning rate\n    linearly for the first ``warmup_steps`` training steps, and decreasing it thereafter proportionally\n    to the inverse square root of the step number, scaled by the inverse square root of the\n    dimensionality of the model. Time will tell if this is just madness or it's actually important.\n    Parameters\n    ----------\n    warmup_steps: ``int``, required.\n        The number of steps to linearly increase the learning rate.\n    \"\"\"\n    def __init__(self, optimizer, warmup_steps):\n        self.warmup_steps = warmup_steps\n        super().__init__(optimizer)\n\n    def get_lr(self):\n        last_epoch = max(1, self.last_epoch)\n        scale = self.warmup_steps ** 0.5 * min(last_epoch ** (-0.5), last_epoch * self.warmup_steps ** (-1.5))\n        return [base_lr * scale for base_lr in self.base_lrs]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"    \ndef train_pipeline(train_dataset,valid_dataset, seed,fold,verbose=False,**kwargs):  \n    \n    seed_everything(seed)\n    \n   \n    trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n    validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)\n    \n   \n    model = SAINTPLUSModel(n_skill,n_correctness=3, embed_dim=128)\n    \n    model.to(DEVICE)\n    \n    #initialize_from_past_model(model, f\"../results/original_torch_moa_smoothed_lrplateau_5_folds_AUX_SEED{seed}_FOLD{fold}.pth\")#,freeze_first_layer=True)\n    \n    optimizer = torch.optim.Adam(model.parameters(), lr=1e-3,weight_decay=1e-6)\n    #optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)\n    #scheduler = optim.lr_scheduler.OneCycleLR(optimizer, pct_start=0.1, div_factor=1e3, \n     #                                         max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))\n    \n    scheduler = NoamLR(optimizer,12000) #optim.lr_scheduler.ReduceLROnPlateau(optimizer,patience=3)\n    \n    \n    \n   # loss_val = nn.BCEWithLogitsLoss().to(DEVICE)\n   # loss_tr =  nn.BCEWithLogitsLoss().to(DEVICE)\n    \n    loss_val = nn.functional.binary_cross_entropy_with_logits\n    loss_tr =  nn.functional.binary_cross_entropy_with_logits\n    \n    early_stopping_steps = EARLY_STOPPING_STEPS\n    early_step = 0\n    \n    #todo el guardado de los resultados se puede mover a kfold que si tiene info de los indices\n    #oof = np.zeros((len(train), target.iloc[:, 1:].shape[1]))\n    best_loss = np.inf\n    \n    best_auc = 0\n    \n    \n    \n    \n    for epoch in range(EPOCHS):\n        \n        train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)\n        if verbose:\n            print(f\"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}, lr:{scheduler.get_last_lr()[0]}\")\n        valid_loss, valid_preds ,auc = valid_fn(model,scheduler, loss_val, validloader, DEVICE)\n        if verbose:\n            print(f\"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}, auc: {auc} \")\n\n    #         if valid_loss < best_loss:\n\n    #             best_loss = valid_loss\n    #             oof = valid_preds\n    #             torch.save(model.state_dict(), \"SAKT-HDKIM.pt\")\n\n        if auc > best_auc:\n\n            best_auc = auc\n            oof = valid_preds\n            torch.save(model.state_dict(), \"SAKT-HDKIM.pt\")\n\n\n\n       #     torch.save(model.state_dict(), f\"../results/{exp_name}_SEED{seed}_FOLD{fold}.pth\")\n\n        elif(EARLY_STOP == True):\n\n            early_step += 1\n            if (early_step >= early_stopping_steps):\n                break\n\n    \n    #--------------------- PREDICTION---------------------\n   \n  #  testdataset = TestDataset(X_test)\n  #  testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)\n    \n#     model = Model(\n#          num_features= X_train.shape[1] ,\n#         num_targets=  y_train.shape[1],\n#         hidden_size=hidden_size,**kwargs\n#     )\n    \n#     model.load_state_dict(torch.load(f\"../results/FOLD{fold}_{exp_name}.pth\"))\n   # model.to(DEVICE)\n    \n    #predictions = np.zeros((len(test_), target.iloc[:, 1:].shape[1]))\n    \n    #predictions = inference_fn(model, testloader, DEVICE)\n    \n    return oof\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pickle as pk","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group = pd.read_pickle('../input/saintex1inference/saintEX1_groups_LAST_100_with_cum_ap.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"(group.apply(len)==3).all()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#samples = .set_index('user_id')#groupby('user_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SAINTDataset(Dataset):\n    def __init__(self, group, n_skill, max_seq=MAX_SEQ): #HDKIM 100\n        super(SAINTDataset, self).__init__()\n        self.max_seq = max_seq\n        self.n_skill = n_skill\n        self.samples = group\n        \n        self.dummy_correctness = 2\n        self.dummy_elapsed_t = 0\n        self.dummy_pos = 100\n        \n        self.forward_peak_mask = self._forward_peak_mask()\n\n        #         self.user_ids = [x for x in group.index]\n      # self.user_ids = []\n      #  for user_id in group.index:\n      #      q, qa = group[user_id]\n      #      if len(q) < 2: #HDKIM 10\n      #           continue\n       #     self.user_ids.append(user_id)\n\n        #HDKIM Memory reduction\n        #if len(q)>self.max_seq:\n        #    group[user_id] = (q[-self.max_seq:],qa[-self.max_seq:])\n        \n    def _forward_peak_mask(self):\n        \n        future_mask = np.triu(np.ones((self.max_seq, self.max_seq)), k=1).astype('bool')\n        return torch.from_numpy(future_mask)\n\n        \n        \n        \n        \n    def _get_tgt_msk_from_src_msk(self,tgt_msk):\n       \n        \n        d_future_mask = torch.cat( (torch.ones((1,self.max_seq),dtype=torch.bool),torch.cat((torch.zeros((self.max_seq-1,1),dtype=torch.bool),tgt_msk[0:-1,0:-1]),dim=1)),dim=0)\n\n        d_future_mask[0,0] = False\n        \n        return d_future_mask\n   \n    def _add_container_info_to_mask(self,res_ar,rep_list):\n      \n        for i,num in enumerate(rep_list):\n    \n            res_ar[i,i-num:i] = True\n\n        return res_ar\n        \n        \n\n    def __len__(self):\n        return len(self.samples)\n        #return len(self.samples)\n\n    def __getitem__(self, index):\n        #user_id = self.user_ids[index]\n        cum_id_,q_, qa_ = self.samples.iloc[index]\n        #q_, qa_ = self.samples.iloc[index]\n        seq_len = len(q_)\n\n        q = np.ones(self.max_seq, dtype=int) *-1\n        qa = np.ones(self.max_seq, dtype=int)*-1\n        cum_id = np.zeros(self.max_seq, dtype=int)\n        \n        if seq_len >= self.max_seq:\n            #HDKIM\n#             if random.random()>0.02:\n#                 start = random.randint(0,(seq_len-self.max_seq))\n#                 end = start + self.max_seq\n#                 q[:] = q_[start:end]\n#                 qa[:] = qa_[start:end]\n#             else:\n#                 #HDKIMHDKIM\n            cum_id[:] = cum_id_[-self.max_seq:]\n            q[:] = q_[-self.max_seq:]\n            qa[:] = qa_[-self.max_seq:]\n            \n            \n        else:\n            \n            #HDKIM\n#             if random.random()>0.02:\n#                 #HDKIMHDKIM\n#                 start = 0\n#                 end = random.randint(2,seq_len)\n#                 seq_len = end - start\n#                 q[-seq_len:] = q_[0:seq_len]\n#                 qa[-seq_len:] = qa_[0:seq_len]\n#             else:\n                #HDKIMHDKIM\n            cum_id[:seq_len] = cum_id_\n            q[:seq_len] = q_\n            qa[:seq_len] = qa_\n            \n\n        \n        \n        x = np.ones(self.max_seq, dtype=int)* -1\n        \n        x[1:seq_len] = qa[:seq_len-1].copy()\n        \n      #  if seq_len < self.max_seq:\n            \n         #   x[-seq_len] = self.dummy_correctness\n       # else:\n    \n        x[0] = self.dummy_correctness\n        \n        \n#         x = np.zeros(self.max_seq, dtype=int)\n#         x[1:] = q[:-1].copy()\n#         x[0] = self.dummy_position\n        \n        \n        \n#         x = np.zeros(self.max_seq, dtype=int)\n#         x[1:] = q[:-1].copy()\n#         x[0] = self.dummy_elapt\n        \n        \n        \n        \n#         x = np.zeros(self.max_seq, dtype=int)\n#         x[1:] = q[:-1].copy()\n#         x[0] = self.dummy_lagt\n        \n        \n        target_id = torch.from_numpy(q)\n\n        label = torch.from_numpy(qa)\n\n        correctness = torch.from_numpy(x)\n        \n\n        #x += (qa[:-1] == 1) * self.n_skill #this is special to sakt\n        \n        \n        src_pad_mask =  target_id ==-1\n        \n        tgt_pad_mask = correctness ==-1\n        \n           \n        \n        src_mask =  self._add_container_info_to_mask(self.forward_peak_mask.clone(),cum_id)\n        \n        \n        tgt_mask = self._get_tgt_msk_from_src_msk(src_mask)\n        \n        \n        ###\n        \n        pos_id_e = torch.arange(target_id.size(0)) - torch.tensor(cum_id) \n        \n       \n              \n        cum_id2= cum_id.copy()\n        \n        cum_id2[1:] = cum_id[:-1]\n        \n        cum_id2[0] = 0\n        \n\n       \n        pos_id_d = torch.arange(target_id.size(0)) - torch.tensor(cum_id2 ) \n\n        \n        \n     #   lag_embedding\n        \n      #  interaction_embedding\n        \n        \n    \n        \n        \n     #   lag_embedding\n        \n      #  interaction_embedding\n        \n        inp = dict(excercise = target_id +1,correctness = correctness+1 ,\n                   \n                   src_mask = src_mask, tgt_mask=tgt_mask,\n                   \n                   src_pad_mask = src_pad_mask, tgt_pad_mask=tgt_pad_mask,\n                   \n                   pos_id_e = pos_id_e ,pos_id_d=pos_id_d )\n      \n    \n        \n        return  inp, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df=pd.read_csv('../input/riiid-test-answer-prediction/questions.csv',index_col = 'question_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, samples, test_df, n_skill, max_seq=MAX_SEQ): #HDKIM 100\n        super(TestDataset, self).__init__()\n        self.samples = samples\n        self.users = [x for x in test_df.index.unique()]\n        self.test_df = test_df\n        \n        self.n_skill = n_skill\n        self.max_seq = max_seq\n        \n        self.dummy_correctness = 2\n        self.dummy_elapsed_t = 0\n        self.dummy_pos = 100\n        \n        self.forward_peak_mask = self._forward_peak_mask()\n        self.questions_df = questions_df\n        \n    \n    def _forward_peak_mask(self):\n        \n        future_mask = np.triu(np.ones((self.max_seq, self.max_seq)), k=1).astype('bool')\n        return torch.from_numpy(future_mask)\n\n        \n        \n        \n        \n    def _get_tgt_msk_from_src_msk(self,tgt_msk):\n       \n        \n        d_future_mask = torch.cat( (torch.ones((1,self.max_seq),dtype=torch.bool),torch.cat((torch.zeros((self.max_seq-1,1),dtype=torch.bool),tgt_msk[0:-1,0:-1]),dim=1)),dim=0)\n\n        d_future_mask[0,0] = False\n        \n        return d_future_mask\n   \n    def _add_container_info_to_mask(self,res_ar,rep_list):\n      \n        for i,num in enumerate(rep_list):\n    \n            res_ar[i,i-num:i] = True\n\n        return res_ar\n\n    def __len__(self):\n        return len(self.users)\n\n    def __getitem__(self, index):\n        \n        user_id = self.users[index]\n        \n        test_info = self.test_df.loc[user_id]\n       \n        q = np.ones(self.max_seq, dtype=int) *-1\n        qp = np.ones(self.max_seq, dtype=int)*-1\n      #  qet = np.ones(self.max_seq, dtype=int)*-1\n        qa = np.ones(self.max_seq, dtype=int)*-1\n        cum_id = np.zeros(self.max_seq, dtype=int)\n        \n        x = np.ones(self.max_seq, dtype=int)* -1\n        \n        q_t_ = np.array(test_info['content_id'],ndmin=1)\n        \n        \n        \n        qp_t_= np.array(test_info['part'],ndmin=1)\n        \n        \n        \n        \n       # print(q_t_)\n        test_seq_len = len(q_t_)\n        \n        qa_t_ = np.ones(test_seq_len, dtype=int)* -1\n        #qet_t_ = np.ones(test_seq_len, dtype=int)* -1\n        \n        \n        cum_id_t_ = np.arange(test_seq_len)\n        \n        \n        \n        if user_id in self.samples.index:\n            \n            cur_samp = self.samples[user_id]\n            \n            cum_id_,q_, qa_,qp_= cur_samp[:4]\n            \n            \n            q_ = np.concatenate((q_,q_t_))\n            qp_ = np.concatenate((qp_, qp_t_))\n            \n            cum_id_ = np.concatenate((cum_id_,cum_id_t_))\n                        \n            qa_ = np.concatenate((qa_,qa_t_))\n         #   qet_ = np.concatenate((qet_,qet_t_))\n            \n            \n        \n        else:\n            \n            q_ = q_t_\n            qp_ =qp_t_\n            cum_id_ = cum_id_t_\n            qa_ = qa_t_ #np.array(-1,ndmin=1)\n        #    qet_ = qet_t_\n            \n        seq_len = len(q_)\n        \n        if seq_len >= self.max_seq:\n\n            cum_id[:] = cum_id_[-self.max_seq:]\n            q[:] = q_[-self.max_seq:]\n            qp[:] = qp_[-self.max_seq:]\n            qa[:] = qa_[-self.max_seq:]\n            #qet[:] = qet_[-self.max_seq:]\n            \n            \n          #  qet[1:]=qet[:-1]\n            x[1:]=qa[:-1].copy()\n            \n            \n            \n            pred_rng =(self.max_seq-test_seq_len,self.max_seq)\n            \n        else:\n\n            cum_id[:seq_len] = cum_id_\n            q[:seq_len] = q_\n            qp[:seq_len] = qp_\n          #  qet[:seq_len] = qet_\n            \n            qa[:seq_len] = qa_           \n            \n           # qet[1:seq_len+1] = qet[:seq_len]\n            x[1:seq_len+1] = qa[:seq_len].copy()\n        \n            pred_rng =(seq_len-test_seq_len,seq_len)\n       \n        \n        \n        #x[1:seq_len] = qa[:seq_len].copy()\n        \n      #  if seq_len < self.max_seq:\n            \n         #   x[-seq_len] = self.dummy_correctness\n       # else:\n    \n        x[0] = self.dummy_correctness\n        \n        \n        \n       # qet = np.roll(qet,1)\n       # qet[0] = self.dummy_elapsed_t\n        \n        \n        \n#         x = np.zeros(self.max_seq, dtype=int)\n#         x[1:] = q[:-1].copy()\n#         x[0] = self.dummy_position\n        \n        \n        \n#         x = np.zeros(self.max_seq, dtype=int)\n#         x[1:] = q[:-1].copy()\n#         x[0] = self.dummy_elapt\n        \n        \n        \n        \n#         x = np.zeros(self.max_seq, dtype=int)\n#         x[1:] = q[:-1].copy()\n#         x[0] = self.dummy_lagt\n        \n        \n        target_id = torch.from_numpy(q)\n\n        label = torch.from_numpy(qa)\n        \n        parts = torch.from_numpy(qp)\n\n        correctness = torch.from_numpy(x)\n       # elap_t = torch.from_numpy(np.where(qet<-1,-1,qet))\n\n        #x += (qa[:-1] == 1) * self.n_skill #this is special to sakt\n        \n        \n        src_pad_mask =  target_id ==-1\n        \n        tgt_pad_mask = correctness ==-1\n        \n           \n        \n        src_mask =  self._add_container_info_to_mask(self.forward_peak_mask.clone(),cum_id)\n        \n        \n        tgt_mask = self._get_tgt_msk_from_src_msk(src_mask)\n        \n        \n        ###\n        if cum_id[0]>0:\n            \n            cur_cum_ap = cum_id[0] \n            ctr=0\n            \n            while cur_cum_ap >0 :\n                \n                cum_id[ctr] = ctr\n                \n                ctr +=1\n                \n                cur_cum_ap = cum_id[ctr]\n                \n        \n        pos_id_e = torch.arange(target_id.size(0)) - torch.tensor(cum_id) \n        \n       \n              \n        cum_id2= cum_id.copy()\n        \n        cum_id2[1:] = cum_id[:-1]\n        \n        cum_id2[0] = - 100\n        \n\n        \n\n       \n        pos_id_d = torch.arange(target_id.size(0)) - torch.tensor(cum_id2 ) \n\n        \n        \n     #   lag_embedding\n        \n      #  interaction_embedding\n        \n        \n    \n        \n        \n     #   lag_embedding\n        \n      #  interaction_embedding\n        \n         \n        inp = dict(excercise = target_id +1,correctness = correctness+1 ,parts =parts +1 ,# elap_t =elap_t+1 ,\n                   \n                   src_mask = src_mask, tgt_mask=tgt_mask,\n                   \n                   src_pad_mask = src_pad_mask, tgt_pad_mask=tgt_pad_mask,\n                   \n                   pos_id_e = pos_id_e  ,pos_id_d=pos_id_d )\n      \n    \n        \n        return  inp, pred_rng","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"pred_rng"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"np.array(prev_test_df.iloc[0]['content_id'])"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"test_df, sample_prediction_df =next(iter_test)"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"\n\n\nfor (test_df, sample_prediction_df) in tqdm(iter_test):\n    #HDKIM\n\n    if (prev_test_df is not None) & (psutil.virtual_memory().percent<90):\n\n        print(psutil.virtual_memory().percent)\n\n        prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])\n        prev_test_df = prev_test_df[prev_test_df.content_type_id == False]\n\n        #supposing we recieve task container_id alltogether\n        prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id').apply(lambda r: ( np.arange(len(r)),\n            r['content_id'].values,\n            r['answered_correctly'].values))\n\n        for prev_user_id in prev_group.index:\n\n            prev_group_cum_ap = prev_group[prev_user_id][0]\n            prev_group_content = prev_group[prev_user_id][1]\n            prev_group_ac = prev_group[prev_user_id][2]\n            if prev_user_id in group.index:\n                group[prev_user_id] = (np.append(group[prev_user_id][0],prev_group_cum_ap),np.append(group[prev_user_id][1],prev_group_content), \n                                       np.append(group[prev_user_id][2],prev_group_ac))\n\n            else:\n\n                group[prev_user_id] = (prev_group_ac,prev_group_content,prev_group_ac)\n\n            if len(group[prev_user_id][0])>MAX_SEQ:\n\n                new_group_cum_ap = group[prev_user_id][0][-MAX_SEQ:]\n                new_group_content = group[prev_user_id][1][-MAX_SEQ:]\n                new_group_ac = group[prev_user_id][2][-MAX_SEQ:]\n\n                group[prev_user_id] = (new_group_cum_ap,new_group_content,new_group_ac)\n\n    prev_test_df = test_df.copy()\n\n    #HDKIMHDKIM\n\n    test_df = test_df[test_df.content_type_id == False]\n\n    test_dataset = TestDataset(group, test_df.loc[test_df['content_type_id'] == 0].set_index('user_id'), n_skill=13524)\n    test_dataloader = DataLoader(test_dataset, batch_size=12000, shuffle=False)\n\n    # outs = []\n\n    #for item in tqdm(test_dataloader):\n    model.eval()\n    final_loss = 0\n    valid_preds = []\n    # labels = []\n\n    tbar = tqdm(test_dataloader)\n\n    for item in tbar:\n\n        inp = item[0]\n        pred_rng = item[1]#.to(device)\n\n        for k, v in inp.items():\n\n            inp[k] = v.to(device)\n\n        with torch.no_grad():\n            output, msk = model( **inp)\n\n        output = torch.sigmoid(output)\n    # output = output[:, -1]\n\n    # pred = (output >= 0.5).long()\n    # loss = criterion(output, label)\n\n    # val_loss.append(loss.item())\n    # num_corrects += (pred == label).sum().item()\n    # num_total += len(label)\n\n    # labels.extend(label.squeeze(-1).data.cpu().numpy())\n        for i in range(len(pred_rng[0])):\n\n            valid_preds.extend(output[i,pred_rng[0][i]:pred_rng[1][i]].data.cpu().numpy())\n\n    test_df.loc[test_df['content_type_id'] == 0,'answered_correctly'] =  valid_preds\n\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndef valid_last_fn(model, loss_fn, dataloader, device):\n    model.eval()\n    final_loss = 0\n    valid_preds = []\n    labels = []\n    \n    tbar = tqdm(dataloader)\n    \n    for item in tbar:\n     \n        inp = item[0]\n        label = item[1].to(device)\n        \n        for k, v in inp.items():\n          \n            inp[k] = v.to(device)\n\n        \n      \n        output, msk = model( **inp)\n        \n        loss = loss_fn(output , (label * (~msk).int()).float() , weight = (~msk).int())\n        \n        output = torch.sigmoid(output)\n        \n        amin = (label==-1).int().argmax(1)-1\n\n        lasts_idx = torch.where(amin != -1 , amin, MAX_SEQ-1)\n       # \n        #[label[i,idx] for i,idx in enumerate(lasts_idx)]        \n        \n         \n        output = torch.tensor([output[i,idx] for i,idx in enumerate(lasts_idx)])   #.flatten()[(~msk).flatten()]\n        \n        #print(output)\n        \n        label = torch.tensor([label[i,idx] for i,idx in enumerate(lasts_idx)])    #.flatten()[(~msk).flatten()] \n        #print(label)\n        \n        #break\n        #labels.extend(label.view(-1).data.cpu().numpy())\n        #valid_preds.extend(output.view(-1).data.cpu().numpy())\n       \n        labels.extend(label.data.cpu().numpy())\n        valid_preds.extend(output.data.cpu().numpy())\n        \n        #valid_preds.append(output.sigmoid().detach().cpu().numpy())\n       # labels.append(label.detach().cpu().numpy()[0])\n        \n#         if scheduler.__class__ ==  torch.optim.lr_scheduler.ReduceLROnPlateau:\n#             scheduler.step(final_loss)\n    \n        \n        final_loss += loss.item()\n        \n    final_loss /= len(dataloader)\n    auc = roc_auc_score(labels, valid_preds)\n    \n    return final_loss,valid_preds,auc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndef inference_fn(model, loss_fn, dataloader, device):\n    model.eval()\n    final_loss = 0\n    valid_preds = []\n   # labels = []\n    \n    tbar = tqdm(dataloader)\n    \n    for item in tbar:\n     \n        inp = item[0]\n        pred_rng = item[1]#.to(device)\n        \n        for k, v in inp.items():\n          \n            inp[k] = v.to(device)\n\n        \n      \n        output, msk = model( **inp)\n        \n        #loss = loss_fn(output , (label * (~msk).int()).float() , weight = (~msk).int())\n        \n        output = torch.sigmoid(output)\n        \n       # amin = (label==-1).int().argmax(1)-1\n\n       # lasts_idx = torch.where(amin != -1 , amin, MAX_SEQ-1)\n       # \n        #[label[i,idx] for i,idx in enumerate(lasts_idx)]        \n        \n         \n       # output = torch.tensor([output[i,idx[0]:idx[1]] for i,idx in enumerate(pred_rng)])   #.flatten()[(~msk).flatten()]\n        #print(output)\n        \n        #label = torch.tensor([label[i,idx] for i,idx in enumerate(lasts_idx)])    #.flatten()[(~msk).flatten()] \n        #print(label)\n        \n        #break\n        #labels.extend(label.view(-1).data.cpu().numpy())\n        #valid_preds.extend(output.view(-1).data.cpu().numpy())\n       \n      #  labels.extend(label.data.cpu().numpy())\n        for i,idx in enumerate(pred_rng):\n            \n            valid_preds.extend(output[i,idx[0]:idx[1]].data.cpu().numpy())\n        \n        #valid_preds.append(output.sigmoid().detach().cpu().numpy())\n       # labels.append(label.detach().cpu().numpy()[0])\n        \n#         if scheduler.__class__ ==  torch.optim.lr_scheduler.ReduceLROnPlateau:\n#             scheduler.step(final_loss)\n    \n        \n        final_loss += loss.item()\n        \n    final_loss /= len(dataloader)\n    auc = roc_auc_score(labels, valid_preds)\n    \n    return final_loss,valid_preds,auc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = SAINTPLUSModel(n_skill=13524,n_correctness=3, embed_dim=256)\n\nmodel.load_state_dict(torch.load(f\"../input/saint-4l-8h-emb256-extra-part-virtual/SAint_4l_8h_emb256_extra_part_virtual2.pt\",map_location=device))\nmodel.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import psutil\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"test_df, sample_prediction_df =next(iter_test)"},{"metadata":{"trusted":true},"cell_type":"code","source":"prev_test_df = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"target_df = pd.read_pickle('../input/riiid-cross-validation-files/cv1_train.pickle')\n\n# I use the middle 2.5M range here.\ntarget_df = target_df[50_000_000:52_500_000]"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"class Iter_Valid(object):\n    def __init__(self, df, max_user=1000):\n        df = df.reset_index(drop=True)\n        self.df = df\n        self.user_answer = df['user_answer'].astype(str).values\n        self.answered_correctly = df['answered_correctly'].astype(str).values\n        df['prior_group_responses'] = \"[]\"\n        df['prior_group_answers_correct'] = \"[]\"\n        self.sample_df = df[df['content_type_id'] == 0][['row_id']]\n        self.sample_df['answered_correctly'] = 0\n        self.len = len(df)\n        self.user_id = df.user_id.values\n        self.task_container_id = df.task_container_id.values\n        self.content_type_id = df.content_type_id.values\n        self.max_user = max_user\n        self.current = 0\n        self.pre_user_answer_list = []\n        self.pre_answered_correctly_list = []\n\n    def __iter__(self):\n        return self\n    \n    def fix_df(self, user_answer_list, answered_correctly_list, pre_start):\n        df= self.df[pre_start:self.current].copy()\n        sample_df = self.sample_df[pre_start:self.current].copy()\n        df.loc[pre_start,'prior_group_responses'] = '[' + \",\".join(self.pre_user_answer_list) + ']'\n        df.loc[pre_start,'prior_group_answers_correct'] = '[' + \",\".join(self.pre_answered_correctly_list) + ']'\n        self.pre_user_answer_list = user_answer_list\n        self.pre_answered_correctly_list = answered_correctly_list\n        return df, sample_df\n\n    def __next__(self):\n        added_user = set()\n        pre_start = self.current\n        pre_added_user = -1\n        pre_task_container_id = -1\n\n        user_answer_list = []\n        answered_correctly_list = []\n        while self.current < self.len:\n            crr_user_id = self.user_id[self.current]\n            crr_task_container_id = self.task_container_id[self.current]\n            crr_content_type_id = self.content_type_id[self.current]\n            if crr_content_type_id == 1:\n                # no more than one task_container_id of \"questions\" from any single user\n                # so we only care for content_type_id == 0 to break loop\n                user_answer_list.append(self.user_answer[self.current])\n                answered_correctly_list.append(self.answered_correctly[self.current])\n                self.current += 1\n                continue\n            if crr_user_id in added_user and ((crr_user_id != pre_added_user) or (crr_task_container_id != pre_task_container_id)):\n                # known user(not prev user or differnt task container)\n                return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n            if len(added_user) == self.max_user:\n                if  crr_user_id == pre_added_user and crr_task_container_id == pre_task_container_id:\n                    user_answer_list.append(self.user_answer[self.current])\n                    answered_correctly_list.append(self.answered_correctly[self.current])\n                    self.current += 1\n                    continue\n                else:\n                    return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n            added_user.add(crr_user_id)\n            pre_added_user = crr_user_id\n            pre_task_container_id = crr_task_container_id\n            user_answer_list.append(self.user_answer[self.current])\n            answered_correctly_list.append(self.answered_correctly[self.current])\n            self.current += 1\n        if pre_start < self.current:\n            return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n        else:\n            raise StopIteration()"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"validaten_flg = True\nif validaten_flg:\n    iter_test = Iter_Valid(target_df,max_user=1000)\n    predicted = []\n    def set_predict(df):\n        predicted.append(df)\nelse:\n    import riiideducation\n    env = riiideducation.make_env()\n    iter_test = env.iter_test()\n    set_predict = env.predict"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"\nfor (test_df, sample_prediction_df) in tqdm(iter_test):\n    #HDKIM\n\n    if (prev_test_df is not None) & (psutil.virtual_memory().percent<90):\n\n       # print(psutil.virtual_memory().percent)\n        ans_cor = eval(test_df['prior_group_answers_correct'].iloc[0])\n        \n        prev_test_df['answered_correctly'] = [it for it in ans_cor if it>-1]\n        #prev_test_df['elapsed_time'] = test_df.iloc[1:['prior_question_elapsed_time']\n                                                    \n        #prev_test_df = prev_test_df[prev_test_df.content_type_id == False]\n\n        #supposing we recieve task container_id alltogether\n        prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly','part']].groupby('user_id').apply(lambda r: ( np.arange(len(r)),\n            r['content_id'].values,\n            r['answered_correctly'].values,r['part'].values))\n\n        for prev_user_id in prev_group.index:\n\n            prev_group_cum_ap = prev_group[prev_user_id][0]\n            prev_group_content = prev_group[prev_user_id][1]\n            prev_group_ac = prev_group[prev_user_id][2]\n            prev_group_part = prev_group[prev_user_id][3]\n           # prev_group_elapt = np.array([np.nan]*len(prev_group_ac),ndmin=1)\n            \n            if prev_user_id in group.index:\n                group[prev_user_id] = (np.append(group[prev_user_id][0],prev_group_cum_ap),np.append(group[prev_user_id][1],prev_group_content), \n                                       np.append(group[prev_user_id][2],prev_group_ac),np.append(group[prev_user_id][3],prev_group_part),group[prev_user_id][4])\n                                       #,np.append(group[prev_user_id][4])#,prev_group_elapt))\n\n            else:\n\n                group[prev_user_id] = (prev_group_ac,prev_group_content,prev_group_ac,prev_group_part,prev_group_elapt)\n\n            if len(group[prev_user_id][0])>MAX_SEQ:\n\n                new_group_cum_ap = group[prev_user_id][0][-MAX_SEQ:]\n                new_group_content = group[prev_user_id][1][-MAX_SEQ:]\n                new_group_ac = group[prev_user_id][2][-MAX_SEQ:]\n                new_group_part = group[prev_user_id][3][-MAX_SEQ:]\n                new_group_elapt = group[prev_user_id][4][-MAX_SEQ:]\n\n                group[prev_user_id] = (new_group_cum_ap,new_group_content,new_group_ac,new_group_part,new_group_elapt)\n                \n                \n#     for cur_user_id,elap_t in test_df.iloc[1:].drop_duplicates('user_id')[['user_id','prior_question_elapsed_time']].values:\n        \n#         ar = group[cur_user_id][4].copy()\n        \n#         indx = -1\n        \n#         while (ar[indx]<0) or np.isnan(ar[indx]): \n                \n#             indx-=1\n        \n        \n#         ar = ar[:len(ar)+indx+1]\n        \n        \n#         cur_group_cum_ap = group[cur_user_id][0][-MAX_SEQ:]\n#         cur_group_content = group[cur_user_id][1][-MAX_SEQ:]\n#         cur_group_ac = group[cur_user_id][2][-MAX_SEQ:]\n#         cur_group_part = group[cur_user_id][3][-MAX_SEQ:]\n                \n        \n#         group[cur_user_id] = (cur_group_cum_ap,cur_group_content,cur_group_ac, cur_group_part,np.append(ar,np.array([elap_t//1000]*(len(cur_group_cum_ap)-len(ar)) ))[-MAX_SEQ:])\n        \n    \n\n    #HDKIMHDKIM\n\n    test_df = test_df.loc[test_df.content_type_id == False]\n    \n    \n    test_df['part'] = test_df['content_id'].map(questions_df['part'])\n    \n    prev_test_df = test_df.copy()\n    \n   \n    \n    test_dataset =   TestDataset(group, test_df.set_index('user_id'), n_skill=13524)                #TestDataset(group, test_df.loc[test_df['content_type_id'] == 0].set_index('user_id'), n_skill=13524)\n    \n    test_dataloader = DataLoader(test_dataset, batch_size=1200, shuffle=False)\n\n    # outs = []\n\n    #for item in tqdm(test_dataloader):\n    model.eval()\n    final_loss = 0\n    valid_preds = []\n    # labels = []\n\n    tbar = tqdm(test_dataloader)\n\n    for item in tbar:\n\n        inp = item[0]\n        pred_rng = item[1]#.to(device)\n\n        for k, v in inp.items():\n\n            inp[k] = v.to(device)\n\n        with torch.no_grad():\n            output, msk = model( **inp)\n\n        output = torch.sigmoid(output)\n    # output = output[:, -1]\n\n    # pred = (output >= 0.5).long()\n    # loss = criterion(output, label)\n\n    # val_loss.append(loss.item())\n    # num_corrects += (pred == label).sum().item()\n    # num_total += len(label)\n\n    # labels.extend(label.squeeze(-1).data.cpu().numpy())\n        for i in range(len(pred_rng[0])):\n\n            valid_preds.extend(output[i,pred_rng[0][i]:pred_rng[1][i]].data.cpu().numpy())\n\n    test_df['answered_correctly'] =  valid_preds               #.loc[test_df['content_type_id'] == 0,'answered_correctly'] =  valid_preds\n    \n   # print(test_df[['row_id', 'answered_correctly']])\n    print(test_df.loc[:,['row_id', 'answered_correctly']])\n   # env.predict(test_df[['row_id', 'answered_correctly']])                 # .loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n    set_predict(test_df.loc[:,['row_id', 'answered_correctly']])\n    #pbar.update(len(current_test))\n    #env.predict(test_df[['row_id', 'answered_correctly']])                 # .loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])"},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nfor (test_df, sample_prediction_df) in tqdm(iter_test):\n    #HDKIM\n\n    if (prev_test_df is not None) & (psutil.virtual_memory().percent<90):\n\n       # print(psutil.virtual_memory().percent)\n        ans_cor = eval(test_df['prior_group_answers_correct'].iloc[0])\n        \n        prev_test_df['answered_correctly'] = [it for it in ans_cor if it>-1]\n        #prev_test_df['elapsed_time'] = test_df.iloc[1:['prior_question_elapsed_time']\n                                                    \n        #prev_test_df = prev_test_df[prev_test_df.content_type_id == False]\n\n        #supposing we recieve task container_id alltogether\n        prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly','part']].groupby('user_id').apply(lambda r: ( np.arange(len(r)),\n            r['content_id'].values,\n            r['answered_correctly'].values,r['part'].values))\n\n        for prev_user_id in prev_group.index:\n\n            prev_group_cum_ap = prev_group[prev_user_id][0]\n            prev_group_content = prev_group[prev_user_id][1]\n            prev_group_ac = prev_group[prev_user_id][2]\n            prev_group_part = prev_group[prev_user_id][3]\n           # prev_group_elapt = np.array([np.nan]*len(prev_group_ac),ndmin=1)\n            \n            if prev_user_id in group.index:\n                group[prev_user_id] = (np.append(group[prev_user_id][0],prev_group_cum_ap),np.append(group[prev_user_id][1],prev_group_content), \n                                       np.append(group[prev_user_id][2],prev_group_ac),np.append(group[prev_user_id][3],prev_group_part),group[prev_user_id][4])\n                                       #,np.append(group[prev_user_id][4])#,prev_group_elapt))\n\n            else:\n\n                group[prev_user_id] = (prev_group_ac,prev_group_content,prev_group_ac,prev_group_part,np.array([np.nan],ndmin=1))\n\n            if len(group[prev_user_id][0])>MAX_SEQ:\n\n                new_group_cum_ap = group[prev_user_id][0][-MAX_SEQ:]\n                new_group_content = group[prev_user_id][1][-MAX_SEQ:]\n                new_group_ac = group[prev_user_id][2][-MAX_SEQ:]\n                new_group_part = group[prev_user_id][3][-MAX_SEQ:]\n                new_group_elapt = group[prev_user_id][4][-MAX_SEQ:]\n\n                group[prev_user_id] = (new_group_cum_ap,new_group_content,new_group_ac,new_group_part,new_group_elapt)\n                \n                \n#     for cur_user_id,elap_t in test_df.iloc[1:].drop_duplicates('user_id')[['user_id','prior_question_elapsed_time']].values:\n        \n#         ar = group[cur_user_id][4].copy()\n        \n#         indx = -1\n        \n#         while (ar[indx]<0) or np.isnan(ar[indx]): \n                \n#             indx-=1\n        \n        \n#         ar = ar[:len(ar)+indx+1]\n        \n        \n#         cur_group_cum_ap = group[cur_user_id][0][-MAX_SEQ:]\n#         cur_group_content = group[cur_user_id][1][-MAX_SEQ:]\n#         cur_group_ac = group[cur_user_id][2][-MAX_SEQ:]\n#         cur_group_part = group[cur_user_id][3][-MAX_SEQ:]\n                \n        \n#         group[cur_user_id] = (cur_group_cum_ap,cur_group_content,cur_group_ac, cur_group_part,np.append(ar,np.array([elap_t//1000]*(len(cur_group_cum_ap)-len(ar)) ))[-MAX_SEQ:])\n        \n    \n\n    #HDKIMHDKIM\n\n    test_df = test_df.loc[test_df.content_type_id == False]\n    \n    \n    test_df['part'] = test_df['content_id'].map(questions_df['part'])\n    \n    prev_test_df = test_df.copy()\n    \n   \n    \n    test_dataset =   TestDataset(group, test_df.set_index('user_id'), n_skill=13524)                #TestDataset(group, test_df.loc[test_df['content_type_id'] == 0].set_index('user_id'), n_skill=13524)\n    \n    test_dataloader = DataLoader(test_dataset, batch_size=1200, shuffle=False)\n\n    # outs = []\n\n    #for item in tqdm(test_dataloader):\n    model.eval()\n    final_loss = 0\n    valid_preds = []\n    # labels = []\n\n    tbar = tqdm(test_dataloader)\n\n    for item in tbar:\n\n        inp = item[0]\n        pred_rng = item[1]#.to(device)\n\n        for k, v in inp.items():\n\n            inp[k] = v.to(device)\n\n        with torch.no_grad():\n            output, msk = model( **inp)\n\n        output = torch.sigmoid(output)\n    # output = output[:, -1]\n\n    # pred = (output >= 0.5).long()\n    # loss = criterion(output, label)\n\n    # val_loss.append(loss.item())\n    # num_corrects += (pred == label).sum().item()\n    # num_total += len(label)\n\n    # labels.extend(label.squeeze(-1).data.cpu().numpy())\n        for i in range(len(pred_rng[0])):\n\n            valid_preds.extend(output[i,pred_rng[0][i]:pred_rng[1][i]].data.cpu().numpy())\n\n    test_df['answered_correctly'] =  valid_preds               #.loc[test_df['content_type_id'] == 0,'answered_correctly'] =  valid_preds\n               #.loc[test_df['content_type_id'] == 0,'answered_correctly'] =  valid_preds\n    \n   # print(test_df[['row_id', 'answered_correctly']])\n    \n   # env.predict(test_df[['row_id', 'answered_correctly']])\n    \n    env.predict(test_df.loc[:,['row_id', 'answered_correctly']])                 # .loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"sub = pd.read_csv('./submission.csv')\n\nsub.info()"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"\n\n\n#for (test_df, sample_prediction_df) in tqdm(iter_test):\n    #HDKIM\n\n#HDKIMHDKIM\n\ntest_df = test_df.loc[test_df.content_type_id == False]\n\ntest_df['part']=questions_df.loc[test_df['content_id'],'part']\n\ntest_dataset =   TestDataset(group, test_df.set_index('user_id'), n_skill=13524)                #TestDataset(group, test_df.loc[test_df['content_type_id'] == 0].set_index('user_id'), n_skill=13524)\n\ntest_dataloader = DataLoader(test_dataset, batch_size=1200, shuffle=False)\n\n# outs = []\n\n#for item in tqdm(test_dataloader):\nmodel.eval()\nfinal_loss = 0\nvalid_preds = []\n# labels = []\n\ntbar = tqdm(test_dataloader)\n\nfor item in tbar:\n\n    inp = item[0]\n    pred_rng = item[1]#.to(device)\n\n    for k, v in inp.items():\n\n        inp[k] = v.to(device)\n\n    with torch.no_grad():\n        output, msk = model( **inp)\n\n    output = torch.sigmoid(output)\n# output = output[:, -1]\n\n# pred = (output >= 0.5).long()\n# loss = criterion(output, label)\n\n# val_loss.append(loss.item())\n# num_corrects += (pred == label).sum().item()\n# num_total += len(label)\n\n# labels.extend(label.squeeze(-1).data.cpu().numpy())\n    for i in range(len(pred_rng[0])):\n\n        valid_preds.extend(output[i,pred_rng[0][i]:pred_rng[1][i]].data.cpu().numpy())\n\ntest_df['answered_correctly'] =  valid_preds               #.loc[test_df['content_type_id'] == 0,'answered_correctly'] =  valid_preds\n\nenv.predict(test_df[['row_id', 'answered_correctly']])                 # .loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}