{"cells":[{"metadata":{"papermill":{"duration":0.018715,"end_time":"2020-12-12T19:28:26.061856","exception":false,"start_time":"2020-12-12T19:28:26.043141","status":"completed"},"tags":[]},"cell_type":"markdown","source":"###### * Base Source: \nSAKT with Randomization & State Updates LB0.771 https://www.kaggle.com/leadbest/sakt-with-randomization-state-updates\n\n1. Version 1: ++bundle_id-Embedding, ++tags-Embedding,  ++CustomTrainer\n2. this is a repeat of version 8 of the notebook since at submission time I got this message:\n\"Cannot submit\nYour Notebook cannot use internet access in this competition. Please disable internet in the Notebook editor and save a new version.\"\n3. Hence, I am reusing the model trained at version 8. You better see the version 8 of the notebook for the actual code. Otherwise from this verson remove the code: \"model.load_state_dict...\" and set the \"num_train_epochs=13\" in the first trainin loop. Second training loop still be \"num_train_epochs=1\" only."},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.execute_input":"2020-12-12T19:28:26.156557Z","iopub.status.busy":"2020-12-12T19:28:26.155823Z","iopub.status.idle":"2020-12-12T19:28:28.442588Z","shell.execute_reply":"2020-12-12T19:28:28.441397Z"},"papermill":{"duration":2.310574,"end_time":"2020-12-12T19:28:28.442722","exception":false,"start_time":"2020-12-12T19:28:26.132148","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import gc\nimport random\nfrom   tqdm import tqdm\nfrom   sklearn.model_selection import train_test_split\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport random\nimport psutil\nimport datatable as dt\nfrom collections import namedtuple  \n\nimport os\nimport seaborn as sns\nfrom   sklearn.utils import shuffle\nimport datetime\nimport time\nfrom   sklearn.metrics import accuracy_score, precision_recall_fscore_support, matthews_corrcoef, roc_auc_score\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.utils.rnn as rnn_utils\nfrom   torch.autograd import Variable\nfrom   torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler\nfrom   transformers import TrainingArguments\nfrom   transformers import AdamW, BertConfig\nfrom   transformers import get_linear_schedule_with_warmup","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-12T19:28:28.485609Z","iopub.status.busy":"2020-12-12T19:28:28.484566Z","iopub.status.idle":"2020-12-12T19:28:28.487832Z","shell.execute_reply":"2020-12-12T19:28:28.487253Z"},"papermill":{"duration":0.025858,"end_time":"2020-12-12T19:28:28.487934","exception":false,"start_time":"2020-12-12T19:28:28.462076","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"MAX_SEQ = 256","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.029429,"end_time":"2020-12-12T19:28:28.53531","exception":false,"start_time":"2020-12-12T19:28:28.505881","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Load train df and join the tags column from the questions df"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-12T19:28:28.578Z","iopub.status.busy":"2020-12-12T19:28:28.577392Z","iopub.status.idle":"2020-12-12T19:30:41.792026Z","shell.execute_reply":"2020-12-12T19:30:41.791419Z"},"papermill":{"duration":133.238959,"end_time":"2020-12-12T19:30:41.792147","exception":false,"start_time":"2020-12-12T19:28:28.553188","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"%%time\ndtype = {'timestamp'         :'int64', \n         'user_id'           :'int32' ,\n         'content_id'        :'int16',\n         'content_type_id'   :'int8',\n         'answered_correctly':'int8'}\n\ntrain_df = pd.read_csv ('../input/riiid-test-answer-prediction/train.csv', usecols=[1, 2, 3, 4, 7], dtype=dtype)\n# train_df = train_df.sample (frac=0.0001)  # TODO: comment this\ntrain_df.head ()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dtype = {\n    'question_id' :'int16',\n    'bundle_id'   :'int16', \n    'tags'        :'str',\n}\nquestions_df         = pd.read_csv ('../input/riiid-test-answer-prediction/questions.csv', usecols=[0, 1, 4], dtype=dtype)\n# convert tag from string and pad the tags with 0 and make it as fixed length (=5) array\nquestions_df['tags'] = questions_df['tags'].map (lambda s: [int(v) for v in s.split()], 'ignore')\nquestions_df['tags'] = questions_df['tags'].fillna (0)\nquestions_df['tags'] = questions_df['tags'].map (lambda s: [0]*5 if s==0 else s*(5//len(s))+s[:5%len(s)] if len(s)<5 else s[:5] )\nquestions_df.head ()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-12T19:30:41.838448Z","iopub.status.busy":"2020-12-12T19:30:41.837612Z","iopub.status.idle":"2020-12-12T19:31:10.265544Z","shell.execute_reply":"2020-12-12T19:31:10.264912Z"},"papermill":{"duration":28.452576,"end_time":"2020-12-12T19:31:10.265685","exception":false,"start_time":"2020-12-12T19:30:41.813109","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train_df = train_df[train_df.content_type_id == False]\n#arrange by timestamp\ntrain_df = train_df.sort_values (['timestamp'], ascending=True).reset_index (drop = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.018413,"end_time":"2020-12-12T19:31:10.303263","exception":false,"start_time":"2020-12-12T19:31:10.28485","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Preprocess"},{"metadata":{"trusted":true},"cell_type":"code","source":"known_bundle_ids = set (questions_df[\"bundle_id\"].unique())\nprint(\"number bundles =\", len (known_bundle_ids))\nn_bundles = max (known_bundle_ids) + 1\nn_bundles","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skills  = train_df[\"content_id\"].unique ()\n# n_skill = max (skills) + 1\n# max (skills), len(skills)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"known_qtn_ids = set (questions_df[\"question_id\"].unique())\nprint (\"number questions =\", len (known_qtn_ids))\nn_skill = max (known_qtn_ids) + 1\nn_skill","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"known_tags = questions_df[\"tags\"].apply (pd.Series).unstack ().reset_index ().dropna ()\nknown_tags = set (known_tags[0].astype ('int16').unique ())\nn_tags     = max (known_tags) + 1\nn_tags, len (known_tags)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_df = train_df.merge (questions_df, left_on='content_id', right_on='question_id', how='left').drop (columns=['question_id'])\ngc.collect ()\ntrain_df.head ()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# To save from going out of memory, split the df into 2, then create the groups below and join them back\ntrain_df_1 = train_df[:train_df.shape[0]//2]\ntrain_df_2 = train_df[train_df.shape[0]//2:]\ndel train_df; gc.collect ()                  # TODO - uncomment this\n\ngroup = train_df_1[['user_id', 'content_id', 'answered_correctly']].groupby('user_id').apply (lambda r: (\n            r['content_id'].values,\n            r['answered_correctly'].values,\n))\ndel train_df_1\ngc.collect ()\ngroup","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group_2 = train_df_2[['user_id', 'content_id', 'answered_correctly']].groupby('user_id').apply (lambda r: (\n            r['content_id'].values,\n            r['answered_correctly'].values,\n))\ndel train_df_2\ngc.collect ()\ngroup_2","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-12T19:31:11.827447Z","iopub.status.busy":"2020-12-12T19:31:11.491869Z","iopub.status.idle":"2020-12-12T19:31:54.832057Z","shell.execute_reply":"2020-12-12T19:31:54.832552Z"},"papermill":{"duration":43.72132,"end_time":"2020-12-12T19:31:54.832683","exception":false,"start_time":"2020-12-12T19:31:11.111363","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# merge the groups\ngroup_2_set = set (group_2.index)\nfor k in group.index:\n    if k in group_2_set:\n        \n        group[k] = (np.append (group[k][0],  group_2[k][0]), np.append (group[k][1],  group_2[k][1]) ) #, np.append (group[k][2],  group_2[k][2]), np.vstack ((group[k][3], group_2[k][3])) )\n        group_2.pop (k)\ngc.collect ()\ngroup = group.append (group_2)\ndel group_2; gc.collect ()\nprint ('len (group) =', len (group))\ngroup","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-12T19:31:54.873876Z","iopub.status.busy":"2020-12-12T19:31:54.873106Z","iopub.status.idle":"2020-12-12T19:31:54.876112Z","shell.execute_reply":"2020-12-12T19:31:54.875641Z"},"papermill":{"duration":0.024931,"end_time":"2020-12-12T19:31:54.876214","exception":false,"start_time":"2020-12-12T19:31:54.851283","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"random.seed (1)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-12T19:31:54.932684Z","iopub.status.busy":"2020-12-12T19:31:54.931803Z","iopub.status.idle":"2020-12-12T19:31:54.934544Z","shell.execute_reply":"2020-12-12T19:31:54.934028Z"},"papermill":{"duration":0.039956,"end_time":"2020-12-12T19:31:54.934637","exception":false,"start_time":"2020-12-12T19:31:54.894681","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class SAKTDataset (Dataset):\n    \n    def __init__(self, group, n_skill, max_seq=MAX_SEQ):\n        \n        super (SAKTDataset, self).__init__()\n        self.max_seq   = max_seq\n        self.n_skill   = n_skill\n        self.n_bundles = n_bundles\n        self.n_tags    = n_tags\n        self.samples   = group\n        \n        # self.user_ids = [x for x in group.index]\n        self.user_ids  = []\n        for user_id in group.index:\n            \n            # q, qa, b, t = group[user_id]\n            q, qa = group[user_id]\n            if len(q) < 2: #  10\n                continue\n            self.user_ids.append (user_id)\n            \n            # Memory reduction\n            # if len(q)>self.max_seq:\n            #     group[user_id] = (q[-self.max_seq:],qa[-self.max_seq:])\n        return\n\n    def __len__(self):\n        return len(self.user_ids)\n\n    def __getitem__(self, index):\n        \n        user_id  = self.user_ids[index]\n        # q_, qa_, b_, t_ = self.samples[user_id]\n        q_, qa_  = self.samples[user_id]\n        q_b_t_df = pd.DataFrame.from_dict ({'question_id': q_})\n        q_b_t_df = q_b_t_df.merge (questions_df, on='question_id', how='left')\n        b_       = q_b_t_df.bundle_id.values\n        t_       = np.vstack (q_b_t_df.tags.values)\n        del q_b_t_df\n        seq_len  = len (q_)\n        \n        q  = np.zeros (self.max_seq, dtype=int)\n        qa = np.zeros (self.max_seq, dtype=int)\n        b  = np.zeros (self.max_seq, dtype=int)\n        t  = np.array ([[0]*5]*self.max_seq, dtype=int)  # a question can have upto say 5 tags\n        \n        if seq_len >= self.max_seq:\n            if random.random ()>0.1:\n                \n                start = random.randint (0,(seq_len-self.max_seq))\n                end   = start + self.max_seq\n                q[:]  = q_[start:end]\n                qa[:] = qa_[start:end]\n                b[:]  = b_[start:end]\n                t[:]  = t_[start:end]\n            else:\n                \n                q[:]  = q_[-self.max_seq:]\n                qa[:] = qa_[-self.max_seq:]\n                b[:]  = b_[-self.max_seq:]\n                t[:]  = t_[-self.max_seq:]\n        else:\n            if random.random ()>0.1:\n                \n                start = 0\n                end = random.randint (2,seq_len)\n                seq_len = end - start\n                q[-seq_len:]  = q_[0:seq_len]\n                qa[-seq_len:] = qa_[0:seq_len]\n                b[-seq_len:]  = b_[0:seq_len]\n                t[-seq_len:]  = t_[0:seq_len]\n            else:\n                \n                q[-seq_len:]  = q_\n                qa[-seq_len:] = qa_\n                b[-seq_len:]  = b_\n                t[-seq_len:]  = t_\n        target_id = q[1:]\n        label = qa[1:]\n\n        x = np.zeros (self.max_seq-1, dtype=int)\n        x = q[:-1].copy ()\n        x += (qa[:-1] == 1) * self.n_skill\n        \n        bundle_x = b[:-1].copy ()\n        tags_x   = t[:-1].copy ()\n        b_target = b[1:]\n        t_target = t[1:]\n        \n        return x, target_id, label, bundle_x, tags_x, b_target, t_target","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.02598,"end_time":"2020-12-12T19:31:58.194592","exception":false,"start_time":"2020-12-12T19:31:58.168612","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Define model"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-12T19:31:58.256024Z","iopub.status.busy":"2020-12-12T19:31:58.254948Z","iopub.status.idle":"2020-12-12T19:31:58.288736Z","shell.execute_reply":"2020-12-12T19:31:58.289545Z"},"papermill":{"duration":0.068656,"end_time":"2020-12-12T19:31:58.28971","exception":false,"start_time":"2020-12-12T19:31:58.221054","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class FFN (nn.Module):\n    \n    def __init__(self, state_size=200):\n        super (FFN, self).__init__()\n        self.state_size = state_size\n\n        self.lr1 = nn.Linear (state_size, state_size)\n        self.relu = nn.ReLU ()\n        self.lr2 = nn.Linear (state_size, state_size)\n        self.dropout = nn.Dropout (0.2)\n    \n    def forward (self, x):\n        x = self.lr1 (x)\n        x = self.relu (x)\n        x = self.lr2 (x)\n        return self.dropout (x)\n\ndef future_mask (seq_length):\n    future_mask = np.triu (np.ones ((seq_length, seq_length)), k=1).astype ('bool')\n    return torch.from_numpy (future_mask)\n\n\nclass SAKTModel (nn.Module):\n    \n    def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128): #  100->MAX_SEQ\n        super(SAKTModel, self).__init__()\n        self.n_skill   = n_skill\n        self.embed_dim = embed_dim\n\n        self.embedding        = nn.Embedding (2*n_skill+1, embed_dim)\n        self.pos_embedding    = nn.Embedding (max_seq-1,   embed_dim)\n        self.e_embedding      = nn.Embedding (n_skill+1,   embed_dim)\n        self.bundle_embedding = nn.Embedding (n_bundles,   embed_dim)\n        self.tag_embedding    = nn.Embedding (n_tags,      embed_dim)\n\n        self.multi_att    = nn.MultiheadAttention (embed_dim=embed_dim, num_heads=8, dropout=0.2)\n\n        self.dropout      = nn.Dropout (0.2)\n        self.layer_normal = nn.LayerNorm (embed_dim) \n        self.ffn          = FFN (embed_dim)\n        self.outDense     = nn.Linear (embed_dim, 2)\n        self.outActivtn   = nn.LogSoftmax (dim=1)\n        self.NLLLoss      = nn.NLLLoss ()             # criterion\n        return\n    \n    def getTagsEmb (self, tags):\n        \"\"\"\n        list_of_tagList 3D np.array of size (1, seqLen, 5) \n        \"\"\"\n        \n        t_emb = self.tag_embedding (tags)\n        t_emb = torch.mean (t_emb, dim=-2)\n        return t_emb\n        \n    def forward (self, x, question_ids, bundle_x, tags_x, b_target, t_target, labels=None):\n        \"\"\"\n        when you call `model (x ,y, z, ...)` then thid method is invoked\n        \"\"\"\n        \n        device   = x.device                                            #;print ('x.shape        =', x.shape)\n        x        = self.embedding (x)                                  #;print ('x.shape        =', x.shape)\n        pos_id   = torch.arange (x.size (1)).unsqueeze (0).to (device) #;print ('pos_id.shape   =', pos_id.shape)\n        pos_x    = self.pos_embedding (pos_id)                         #;print ('pos_x.shape    =', pos_x.shape)\n        bundle_x = self.bundle_embedding (bundle_x)                    #;print ('bundle_x.shape =', bundle_x.shape)\n        b_target = self.bundle_embedding (b_target)                    #;print ('b_target.shape =', b_target.shape)\n        tags_x   = self.getTagsEmb (tags_x)                            #;print ('tags_x.shape   =', tags_x.shape)\n        t_target = self.getTagsEmb (t_target)                          #;print ('t_target.shape =', t_target.shape)\n        \n        # x    = x + pos_x\n        x      = x + pos_x + bundle_x + tags_x                         #;print ('x.shape        =', x.shape) \n        # e    = self.e_embedding (question_ids)\n        e      = self.e_embedding (question_ids) + b_target + t_target #;print ('e.shape        =', e.shape)\n        \n        x = x.permute (1, 0, 2) ;\"\"\"x: [bs, s_len, embed] => [s_len, bs, embed]\"\"\"   #;print ('x.shape        =', x.shape)\n        e = e.permute (1, 0, 2)                                        #;print ('e.shape        =', e.shape)\n        att_mask   = future_mask (x.size (0)).to (device)              #;print ('att_mask.shape = ', att_mask.shape) \n        att_output, att_weight = self.multi_att (e, x, x, attn_mask=att_mask)   #;print ('att_output.shape = ', att_output.shape) \n        att_output = self.layer_normal (att_output + e)                #;print ('att_output.shape = ', att_output.shape) \n        att_output = att_output.permute (1, 0, 2) #;\"\"\"att_output: [s_len, bs, embed] => [bs, s_len, embed]\"\"\" ;print ('att_output.shape = ', att_output.shape) \n\n        x          = self.ffn (att_output)                             #;print ('x.shape = ', x.shape) \n        x          = self.layer_normal (x + att_output)                #;print ('x.shape = ', x.shape) \n        out_logits = self.outDense (x)                                 #;print ('out_logits.shape = ', out_logits.shape)        \n        \n        if labels is None:\n            \n            # target question id is the last in the sequence of target question ids, hence return the last col\n            out_logits = out_logits[:, -1]\n            # return a named tuple\n            Logits     = namedtuple ('Logits',['logits'])\n            out_logits = Logits (out_logits)\n            return out_logits\n        \n        out_logits = out_logits.view (out_logits.shape[0] * out_logits.shape[1], -1)           #;print ('out_logits.shape = ', out_logits.shape)\n        log_ps     = self.outActivtn (out_logits)                                              # ;print('labels.size=', labels.size(), 'log_ps.size=', log_ps.size())\n        labels     = torch.squeeze (labels.view (labels.shape[0] * labels.shape[1], -1), -1)   # ;print ('labels.shape = ', labels.shape)\n        batchLoss  = self.NLLLoss (log_ps, labels)\n        \n        # return a named tuple\n        Loss_Logits = namedtuple ('Loss_Logits',['loss','logits'])\n        loss_logits = Loss_Logits (batchLoss, out_logits)\n        return loss_logits","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create Train and Eval Datasets and train"},{"metadata":{"trusted":true},"cell_type":"code","source":"frac  = 0.05\ntemp  = list (group.index)\ntemp  = shuffle (temp)\nfrac  = int (len (temp) * frac)\ntemp2 = temp[:frac]\ntemp  = temp[frac:]\ntrain_group = group[temp]\neval_group  = group[temp2]\ndel group\ngc.collect ()\nprint ('len (train_group) =', len (train_group))\nprint ('len (eval_group) =',  len (eval_group))\ntrain_group","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = SAKTDataset (train_group, n_skill)\neval_dataset  = SAKTDataset (eval_group,  n_skill)\nprint (train_dataset[1][0].shape, train_dataset[1][1].shape, train_dataset[1][2].shape, train_dataset[1][3].shape, train_dataset[1][4].shape, train_dataset[1][5].shape, train_dataset[1][6].shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Utilities for my custom trainer\n\ndef format_time (elapsed):\n    '''\n    Takes a time in seconds and returns a string hh:mm:ss\n    '''\n    \n    # Round to the nearest second.\n    elapsed_rounded = int(round((elapsed)))\n    \n    # Format as hh:mm:ss\n    return str (datetime.timedelta (seconds=elapsed_rounded))\n\ndef compute_metrics (labels, pred_logits):\n    \n    preds   = pred_logits.argmax (-1)             #;print ('labels.shape=', labels.shape, 'preds.shape=', preds.shape, 'pred_logits.shape=', pred_logits.shape)\n    precision, recall, f1, _ = precision_recall_fscore_support (labels, preds, average='macro')\n    acc     = accuracy_score (labels, preds)\n    mcc     = matthews_corrcoef (labels, preds)   # matthews correlation coefficient\n    softmax = nn.Softmax (dim=1)\n    pred_pr = softmax (torch.tensor (pred_logits))\n    auc     = roc_auc_score (labels, pred_pr[:, 1])\n    metrics = {\n        'mcc'      : mcc,\n        'accuracy' : acc,\n        'f1'       : f1,\n        'precision': precision,\n        'recall'   : recall,\n        'auc'      : auc\n    }\n    return metrics","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MyTrainer:\n    \n    def __init__(self, model, args, train_dataset, eval_dataset, compute_metrics=compute_metrics):\n        \n        self.model           = model\n        self.args            = args\n        self.train_dataset   = train_dataset\n        self.eval_dataset    = eval_dataset\n        self.compute_metrics = compute_metrics\n        self.isTrained       = False\n        self.device          = self.get_device_type ()\n        \n        # Note: AdamW is a class from the huggingface library (as opposed to pytorch) \n        # I believe the 'W' stands for 'Weight Decay fix\"\n        self.optimizer = AdamW (model.parameters (),\n                           lr  = args.learning_rate,\n                           eps = args.adam_epsilon # args.adam_epsilon  - default is 1e-8 is “a very small number to prevent any division by zero\"\n        )\n\n        # Number of training epochs. The BERT authors recommend between 2 and 4. \n        # We chose to run for 4, but we'll see later that this may be over-fitting the\n        # training data.\n        self.epochs = self.args.num_train_epochs\n        self.train_dataloader, self.validation_dataloader, self.lr_scheduler, self.num_training_steps = self.get_dataLoaders ()        \n        return\n    \n    def get_device_type (self):\n        \n        # If there's a GPU available...\n        if torch.cuda.is_available ():    \n\n            # Tell PyTorch to use the GPU.    \n            device = torch.device (\"cuda\")\n            print('There are %d GPU(s) available.' % torch.cuda.device_count ())\n            print('We will use the GPU:', torch.cuda.get_device_name (0))\n        # If not...\n        else:\n            print('No GPU available, using the CPU instead.')\n            device = torch.device (\"cpu\")\n        return device\n    \n    def get_dataLoaders (self):        \n        \n        # Create the DataLoaders for our training and validation sets.\n        if isinstance (self.train_dataset, torch.utils.data.IterableDataset):\n            train_sampler = None\n        else:\n            train_sampler = SequentialSampler (self.train_dataset)       # Better use RandomSampler\n        train_dataloader  = DataLoader (\n                    self.train_dataset,                                  # The training samples.\n                    sampler     = train_sampler,                           \n                    batch_size  = self.args.per_device_train_batch_size,\n                    num_workers = 8    # TODO: uncomment this\n        )\n        validation_dataloader = None\n        if self.eval_dataset:\n            \n            # For validation the order doesn't matter, so we'll just read them sequentially.\n            validation_dataloader = DataLoader (\n                        self.eval_dataset,             # The validation/dev samples.\n                        sampler     = SequentialSampler (self.eval_dataset),\n                        batch_size  = self.args.per_device_eval_batch_size,\n                        num_workers = 8    # TODO: uncomment this\n            )\n        \n        # Total number of training steps is [number of batches] x [number of epochs]. \n        # (Note that this is not the same as the number of training samples).\n        num_training_steps = len (train_dataloader) * self.epochs\n\n        # Create the learning rate scheduler.\n        lr_scheduler = get_linear_schedule_with_warmup (self.optimizer, \n                                                        num_warmup_steps   = self.args.warmup_steps, # Default value in run_glue.py\n                                                        num_training_steps = num_training_steps)\n        return train_dataloader, validation_dataloader, lr_scheduler, num_training_steps\n    \n    \n    def test_iterate_dataloader ():\n        \n        for step, batch in enumerate (self.train_dataloader):\n            print (step)\n            print (batch)\n            break\n        return\n    \n    \n    def train (self):\n        \n        # This training code is based on the `run_glue.py` script here:\n        # https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n        \n        # Set the seed value all over the place to make this reproducible.\n        seed_val = 42\n        random.seed (seed_val)\n        np.random.seed (seed_val)\n        torch.manual_seed (seed_val)\n        torch.cuda.manual_seed_all (seed_val)\n\n        # We'll store a number of quantities such as training and validation loss, \n        # validation accuracy, and timings.\n        training_stats = []\n        # Measure the total training time for the whole run.\n        total_t0 = time.time ()\n        # inint min_val_loss to a large val, if after each epoch eval-loss < min_val_loss, then save the model\n        min_val_loss   = 9999\n        min_train_loss = 9999\n        step = 0\n        \n        # For each epoch...\n        for epoch_i in range (0, self.epochs):\n\n            # ========================================\n            #               Training\n            # ========================================\n\n            # Perform one full pass over the training set.\n\n            print(\"\")\n            print('======== Epoch {:} / {:} ========'.format (epoch_i + 1, self.epochs))\n            print('Training...')\n\n            # Measure how long the training epoch takes.\n            t0 = time.time ()\n\n            # Reset the total loss for this epoch.\n            total_train_loss = 0\n\n            # Put the model into training mode. Don't be mislead--the call to \n            # `train` just changes the *mode*, it doesn't *perform* the training.\n            # `dropout` and `batchnorm` layers behave differently during training\n            # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)\n            self.model.train ()\n\n            # For each batch of training data...\n            for stp, batch in enumerate (self.train_dataloader):\n\n                step += 1\n                # Progress update every 40 batches.\n                # print ('batch =', batch)\n                if step % 50 == 0 and not step == 0:\n                    \n                    # Calculate elapsed time in minutes.\n                    elapsed = format_time (time.time() - t0)\n                    # Report progress.\n                    print ('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len (self.train_dataloader), elapsed))\n                if (self.args.max_steps > 0 and self.args.max_steps < step) or  \\\n                   (self.args.eval_steps> 0 and step % self.args.eval_steps==0 and step>0):\n                    \n                    avg_train_loss = total_train_loss / step\n                    training_time = format_time (time.time () - t0)\n                    if self.validation_dataloader:\n                        \n                        print (\"Running Validation...\")\n                        avg_val_loss, avg_val_f1, avg_val_mcc, avg_val_auc, avg_val_precision, avg_val_recall, avg_val_accuracy, validation_time = self.evaluate ()\n                        training_stats.append ({\n                                'epoch'         : epoch_i + 1,\n                                'training_loss' : avg_train_loss,\n                                'eval_loss'     : avg_val_loss,\n                                'eval_f1'       : avg_val_f1,\n                                'eval_mcc'      : avg_val_mcc, \n                                'eval_precision': avg_val_precision,\n                                'eval_recall'   : avg_val_recall,\n                                'eval_auc'      : avg_val_auc, \n                                'eval_accuracy' : avg_val_accuracy,\n                                'training_time' : training_time,\n                                'eval_time'     : validation_time                   \n                        })\n                        # save this model if the eval loss decreases from the minimum so far\n                        if avg_val_loss < min_val_loss: \n\n                            min_val_loss = avg_val_loss\n                            torch.save (model.state_dict (), \"SAKT_Rishi.pt\")     # TODO: uncomment this                    \n                self.model.zero_grad ()        \n\n                # Perform a forward pass (evaluate the model on this training batch).\n                # The documentation for this `model` function is here: \n                # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n                # It returns different numbers of parameters depending on what arguments\n                # arge given and what flags are set. For our useage here, it returns\n                # the loss (because we provided labels) and the \"logits\"--the model\n                # outputs prior to activation.\n                # loss, logits = self.model (b_inputs, .., labels=b_labels)\n                # print ('batch: \\n', batch)\n                x         = batch[0].to (device).long ()\n                target_id = batch[1].to (device).long ()\n                labels    = batch[2].to (device).long ()\n                bundle_x  = batch[3].to (device).long ()\n                tags_x    = batch[4].to (device).long ()\n                b_target  = batch[5].to (device).long ()\n                t_target  = batch[6].to (device).long ()\n                output    = self.model (x, target_id, bundle_x, tags_x, b_target, t_target, labels)\n                loss      = output.loss\n                logits    = output.logits\n                \n                # Accumulate the training loss over all of the batches so that we can\n                # calculate the average loss at the end. `loss` is a Tensor containing a\n                # single value; the `.item()` function just returns the Python value \n                # from the tensor.\n                total_train_loss += loss.item ()\n                # Perform a backward pass to calculate the gradients.\n                loss.backward ()\n\n                # Clip the norm of the gradients to 1.0.\n                # This is to help prevent the \"exploding gradients\" problem.\n                torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1.0)\n                # Update parameters and take a step using the computed gradient.\n                # The optimizer dictates the \"update rule\"--how the parameters are\n                # modified based on their gradients, the learning rate, etc.\n                self.optimizer.step ()\n                # Update the learning rate.\n                self.lr_scheduler.step ()\n            # At the end of each epoch measure stats and eval:\n            # Calculate the average loss over all of the batches.\n            avg_train_loss = total_train_loss / len (self.train_dataloader)\n            # Measure how long this epoch took.\n            training_time = format_time (time.time () - t0)            \n            print (\"  Average training loss: {0:.2f}\".format (avg_train_loss))\n            print (\"  Training epcoh took: {:}\".format (training_time))\n            \n            if self.validation_dataloader:\n                \n                print (\"\\n  Running Validation...\")\n                avg_val_loss, avg_val_f1, avg_val_mcc, avg_val_auc, avg_val_precision, avg_val_recall, avg_val_accuracy, validation_time = self.evaluate ()\n                # Record all statistics from this epoch.\n                training_stats.append ({\n                        'epoch'         : epoch_i + 1,\n                        'training_loss' : avg_train_loss,\n                        'eval_loss'     : avg_val_loss,\n                        'eval_f1'       : avg_val_f1,\n                        'eval_mcc'      : avg_val_mcc, \n                        'eval_precision': avg_val_precision,\n                        'eval_recall'   : avg_val_recall,\n                        'eval_auc'      : avg_val_auc, \n                        'eval_accuracy' : avg_val_accuracy,\n                        'training_time' : training_time,\n                        'eval_time'     : validation_time                   \n                })\n                # save this epoch's model if the eval loss decreases from the minimum so far\n                if avg_val_loss < min_val_loss:\n                    \n                    min_val_loss = avg_val_loss\n                    torch.save (model.state_dict (), \"SAKT_Rishi.pt\")     # TODO: uncomment this\n            else:\n                \n                training_stats.append ({\n                    'epoch'         : epoch_i + 1,\n                    'training_loss' : avg_train_loss,\n                    'training_time' : training_time,\n                })\n                if avg_train_loss < min_train_loss: \n                    \n                    min_train_loss = avg_train_loss\n                    torch.save (model.state_dict (), \"SAKT_Rishi.pt\")     # TODO: uncomment this\n        print (\"\")\n        print (\"Training complete!\")\n        print (\"Total training took {:} (h:mm:ss)\".format (format_time (time.time ()-total_t0)))\n        self.isTrained = True\n        self.plot_train_stats (training_stats)\n        return training_stats\n    \n    def evaluate (self):\n        \n        t0 = time.time ()\n        # Put the model in evaluation mode--the dropout layers behave differently\n        # during evaluation.\n        self.model.eval ()\n\n        # Tracking variables \n        total_eval_mcc       = 0\n        total_eval_f1        = 0\n        total_eval_precision = 0\n        total_eval_recall    = 0\n        total_eval_auc       = 0\n        total_eval_accuracy  = 0\n        total_eval_loss      = 0\n        nb_eval_steps        = 0\n\n        # Evaluate data for one epoch\n        for batch in self.validation_dataloader:\n            with torch.no_grad ():\n                \n                # Forward pass, calculate logit predictions.\n                x         = batch[0].to (device).long ()\n                target_id = batch[1].to (device).long ()\n                labels    = batch[2].to (device).long ()\n                bundle_x  = batch[3].to (device).long ()\n                tags_x    = batch[4].to (device).long ()\n                b_target  = batch[5].to (device).long ()\n                t_target  = batch[6].to (device).long ()\n                output    = self.model (x, target_id, bundle_x, tags_x, b_target, t_target, labels)\n                loss      = output.loss\n                logits    = output.logits\n\n            # Accumulate the validation loss.\n            total_eval_loss += loss.item ()\n            # Move logits and labels to CPU\n            logits    = logits.detach ().cpu ().numpy ()\n            label_ids = labels.to ('cpu').numpy ()\n            label_ids = np.squeeze (label_ids.reshape ((label_ids.shape[0] * label_ids.shape[1], -1)), -1)\n\n            # Calculate the accuracy for this batch of test sentences, and\n            # accumulate it over all batches.\n            metrics = self.compute_metrics (label_ids, logits)\n            total_eval_mcc       += metrics['mcc']\n            total_eval_f1        += metrics['f1']\n            total_eval_precision += metrics['precision']\n            total_eval_recall    += metrics['recall']\n            total_eval_auc       += metrics['auc']\n            total_eval_accuracy  += metrics['accuracy']\n\n        # Report the final accuracy for this validation run.\n        avg_val_f1 = total_eval_f1 / len (self.validation_dataloader)\n        print (\"  F1: {0:.3f}\".format (avg_val_f1))\n        avg_val_mcc = total_eval_mcc / len (self.validation_dataloader)\n        print (\"  MCC: {0:.3f}\".format (avg_val_mcc))\n        avg_val_precision = total_eval_precision / len (self.validation_dataloader)\n        print (\"  Precision: {0:.3f}\".format (avg_val_precision))\n        avg_val_recall = total_eval_recall / len (self.validation_dataloader)\n        print (\"  Recall: {0:.3f}\".format (avg_val_recall))\n        avg_val_auc = total_eval_auc / len (self.validation_dataloader)\n        print (\"  AUC: {0:.3f}\".format (avg_val_auc))\n        avg_val_accuracy = total_eval_accuracy / len (self.validation_dataloader)\n        print (\"  Accuracy: {0:.3f}\".format (avg_val_accuracy))\n        # Calculate the average loss over all of the batches.\n        avg_val_loss = total_eval_loss / len (self.validation_dataloader)\n        # Measure how long the validation run took.\n        validation_time = format_time (time.time () - t0)\n        print (\"  Validation Loss: {0:.2f}\".format (avg_val_loss))\n        print (\"  Validation took: {:}\".format (validation_time))            \n        return avg_val_loss, avg_val_f1, avg_val_mcc, avg_val_auc, avg_val_precision, avg_val_recall, avg_val_accuracy, validation_time\n    \n    def plot_train_stats (self, training_stats):\n        \"\"\"\n        Draw Classification Report curve\n        \"\"\"\n        \n        mccs   = accuracies = f1_scores = precisions = recalls = auc = losses = epochs = -1\n        epochs = training_stats[-1]['epoch']\n        if 'eval_mcc' in training_stats[0]:\n            mccs       = [e['eval_mcc'] for e in training_stats]\n            sns.lineplot (x=np.arange(1, epochs + 1), y=mccs,       label='val_mcc')\n        if 'eval_accuracy' in training_stats[0]:\n            accuracies = [e['eval_accuracy'] for e in training_stats]\n            sns.lineplot (x=np.arange(1, epochs + 1), y=accuracies, label='val_accuracy')\n        if 'eval_f1' in training_stats[0]:\n            f1_scores  = [e['eval_f1'] for e in training_stats]\n            sns.lineplot (x=np.arange(1, epochs + 1), y=f1_scores,  label='val_f1') \n        if 'eval_precision' in training_stats[0]:\n            precisions = [e['eval_precision'] for e in training_stats]\n            sns.lineplot (x=np.arange(1, epochs + 1), y=precisions, label='val_precision')\n        if 'eval_recall' in training_stats[0]:\n            recalls    = [e['eval_recall'] for e in training_stats]\n            sns.lineplot (x=np.arange(1, epochs + 1), y=recalls,    label='val_recall')\n        if 'eval_auc' in training_stats[0]:\n            auc        = [e['eval_auc'] for e in training_stats]\n            sns.lineplot (x=np.arange(1, epochs + 1), y=mccs,       label='val_auc')\n        if 'eval_loss' in training_stats[0]:\n            losses     = [e['eval_loss'] for e in training_stats]\n        if 'training_loss'  in training_stats[0]:\n            tr_losses  = [e['training_loss'] for e in training_stats]\n            sns.lineplot (x=np.arange(1, epochs + 1), y=tr_losses,  label='tr_losses')\n            \n        plt.show ()\n        print ('mccs       :', mccs)\n        print ('accuracies :', accuracies)\n        print ('precisions :', precisions)\n        print ('recalls    :', recalls)\n        print ('f1_scores  :', f1_scores)\n        print ('auc        :', auc)\n        print ('losses     :', losses)\n        print ('tr_losses  :', tr_losses)\n        return\n    \n    \n    def getTrainedModel (self):\n        \n        if self.isTrained:\n            return self.model\n        return None\n    \n    \n    def predict (self, prediction_dataset, isRemoveLabels=True):\n        \"\"\"\n        return: pred_logits, true_labels, metrics (if true 'labels' are input in the prediction_dataset)\n        \"\"\"     \n        \n        prediction_sampler    = SequentialSampler (prediction_dataset)\n        prediction_dataloader = DataLoader (prediction_dataset, sampler=prediction_sampler, batch_size=self.args.per_device_eval_batch_size)\n        print ('Predicting labels for {:,} test sentences...'.format (len (prediction_dataset)))\n        \n        # Put model in evaluation mode\n        self.model.eval ()\n\n        # Tracking variables \n        predictions = []\n        # true_labels = []\n        \n        # Predict \n        for batch in prediction_dataloader:\n            \n            # Add batch to GPU\n            batch = {t:batch[t].to (self.device) for t in batch}\n\n            # Unpack the inputs from our dataloader\n            # b_input_ids, b_input_mask, b_segment_ids = batch\n\n            # Telling the model not to compute or store gradients, saving memory and \n            # speeding up prediction\n            with torch.no_grad ():\n                # Forward pass, calculate logit predictions\n                if isRemoveLabels:\n                    batch.pop ('labels')\n                for k in batch:\n                    batch[k] = batch[k].to (self.device)\n                outputs = model (**batch)\n            logits = outputs[0]\n\n            # Move logits and labels to CPU\n            logits = logits.detach ().cpu ().numpy ()\n            # label_ids = b_labels.to ('cpu').numpy ()\n\n            # Store predictions and true labels\n            predictions.append (logits)\n            # true_labels.append (label_ids)\n            print ('Done predictions for ', len(predictions), '/', len(prediction_dataloader), 'batches')\n        print ('Done prediction')\n        \n        # Combine the results across all batches to get the predicted logits\n        pred_logits = np.concatenate (predictions, axis=0)\n        # For each sample, pick the label (0,1,2) with the highest score.\n        # pred_labels = np.argmax (pred_logits, axis=1).flatten()\n        # returns the predicted logits\n        return pred_logits, None, None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device (\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nmodel = SAKTModel (n_skill, embed_dim=128)\ntry:\n    model.load_state_dict (torch.load (\"../input/rishi-sakt-featureembeddings-riidchallenge/SAKT_Rishi.pt\"))\nexcept:\n    model.load_state_dict (torch.load (\"../input/rishi-sakt-featureembeddings-riidchallenge/SAKT_Rishi.pt\", map_location='cpu'))\n\nmodel.to (device)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-12T19:31:58.352377Z","iopub.status.busy":"2020-12-12T19:31:58.351577Z","iopub.status.idle":"2020-12-12T19:32:02.875364Z","shell.execute_reply":"2020-12-12T19:32:02.874265Z"},"papermill":{"duration":4.558313,"end_time":"2020-12-12T19:32:02.875486","exception":false,"start_time":"2020-12-12T19:31:58.317173","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"training_args = TrainingArguments (\n\n    output_dir      = './results',     # output directory\n    num_train_epochs= 1,               # Actually = 13 for training from scratch, without using the SAKT_Rishi.pt\n    warmup_steps    = 1000,            # for lr scheduling\n    eval_steps      = 500,             # Number of update steps between two evaluations, if <=0 then eval at end of each epoch\n    max_steps       = 0,               # If set to a positive number, the total number of training steps to perform. Overrides num_train_epochs\n    learning_rate   = 1e-6,            # Actually = 1e-2 for training from scratch, without using the SAKT_Rishi.pt\n    # adam_epsilon  = 1e-8             # - default is 1e-8 is “a very small number to prevent any division by zero\"\n    per_device_train_batch_size= 1024, # batch size per device during training\n    per_device_eval_batch_size = 1024, # batch size for evaluation\n)\n\ntrainer = MyTrainer (\n    \n    model         = model,           # the instantiated 🤗 Transformers model to be trained\n    args          = training_args,   # training arguments, defined above\n    train_dataset = train_dataset,   # training dataset\n    eval_dataset  = eval_dataset,    # evaluation dataset\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer.train ()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"training_args = TrainingArguments (\n\n    output_dir       = './results',     # output directory\n    num_train_epochs = 1,               # total # of training epochs\n    warmup_steps     = 100,             # for lr scheduling\n    eval_steps       = 300,             # Number of update steps between two evaluations, if <=0 then eval at end of each epoch\n    max_steps        = 0,               # If set to a positive number, the total number of training steps to perform. Overrides num_train_epochs\n    learning_rate    = 1e-7,            # Actually = 1e-4 for training from scratch, without using the SAKT_Rishi.pt\n    # adam_epsilon=1e-8                 # - default is 1e-8 is “a very small number to prevent any division by zero\"\n    per_device_train_batch_size = 1024, # batch size per device during training\n    per_device_eval_batch_size  = 1024, # batch size for evaluation\n)\n\ntrainer = MyTrainer (\n    \n    model         = model,          # the instantiated 🤗 Transformers model to be trained\n    args          = training_args,  # training arguments, defined above\n    train_dataset = eval_dataset,  # training dataset\n    eval_dataset  = None,           # evaluation dataset\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer.train ()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group = train_group.append (eval_group)\nprint ('len (group) =', len (group))\ndel train_group, eval_group, train_dataset, eval_dataset, training_args, trainer\ngc.collect ()\ngroup","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# get test SAKT Features"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-12T20:06:13.367642Z","iopub.status.busy":"2020-12-12T20:06:13.36558Z","iopub.status.idle":"2020-12-12T20:06:13.368626Z","shell.execute_reply":"2020-12-12T20:06:13.369182Z"},"papermill":{"duration":4.86332,"end_time":"2020-12-12T20:06:13.369339","exception":false,"start_time":"2020-12-12T20:06:08.506019","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class TestDataset (Dataset):\n    \n    def __init__(self, samples, test_df, skills, max_seq=MAX_SEQ):\n        \n        super (TestDataset, self).__init__()\n        self.samples  = samples\n        self.user_ids = [x for x in test_df[\"user_id\"].unique()]\n        self.test_df  = test_df\n        self.skills   = skills\n        self.n_skill  = max (skills) + 1\n        self.max_seq  = max_seq\n        return\n    \n    def __len__(self):\n        return self.test_df.shape[0]\n    \n    def __getitem__(self, index):\n        \n        test_info     = self.test_df.iloc[index]\n        user_id       = test_info[\"user_id\"]\n        target_id     = test_info[\"content_id\"]\n        target_tags   = test_info[\"tags\"]\n        target_bundle = test_info[\"bundle_id\"]\n        if target_id not in known_qtn_ids:\n            target_id = 0\n        if target_bundle not in known_bundle_ids:            \n            target_bundle = 0\n        for i in range (len (target_tags)):\n            if target_tags[i] not in known_tags:                \n                target_tags[i] = 0\n        q  = np.zeros (self.max_seq, dtype=int)\n        qa = np.zeros (self.max_seq, dtype=int)\n        b  = np.zeros (self.max_seq, dtype=int)\n        t  = np.array ([[0]*5]*self.max_seq, dtype=int)  # a question can have upto say 5 tags\n        if user_id in self.samples.index:\n            \n            # q_, qa_, b_, t_ = self.samples[user_id]\n            q_, qa_ = self.samples[user_id]\n            q_b_t_df = pd.DataFrame.from_dict ({'question_id': q_})\n            q_b_t_df = q_b_t_df.merge (questions_df, on='question_id', how='left')\n            b_       = q_b_t_df.bundle_id.values\n            t_       = np.vstack (q_b_t_df.tags.values)\n            del q_b_t_df\n            seq_len = len (q_)            \n            if seq_len >= self.max_seq:\n                \n                q  = q_[-self.max_seq:]\n                qa = qa_[-self.max_seq:]\n                b  = b_[-self.max_seq:]\n                t  = t_[-self.max_seq:]\n            else:\n                \n                q[-seq_len:]  = q_\n                qa[-seq_len:] = qa_\n                b[-seq_len:]  = b_\n                t[-seq_len:]  = t_\n        \n        x = np.zeros (self.max_seq-1, dtype=int)\n        x = q[1:].copy ()\n        x += (qa[1:] == 1) * self.n_skill\n        \n        questions = np.append (q[2:], [target_id]) # just [target_id] should do, no need of a sequence of max_seq_len for query Qn_id\n        bundle_x  = b[1:].copy ()\n        tags_x    = t[1:].copy ()\n        b_target  = np.append (b[2:], [target_bundle])\n        t_target  = np.vstack ((t[2:], target_tags))\n        \n        return x, questions, bundle_x, tags_x, b_target, t_target","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"test_dataset = TestDataset (group, train_df, train_df[\"content_id\"])\nitem = test_dataset.__getitem__(5)\nprint (item[0].shape, item[1].shape, item[2].shape, item[3].shape, item[4].shape, item[5].shape)"},{"metadata":{},"cell_type":"markdown","source":"for item in DataLoader (test_dataset):\n    print (item[0].shape, item[1].shape, item[2].shape, item[3].shape, item[4].shape, item[5].shape)\n    break"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-12T20:06:23.419984Z","iopub.status.busy":"2020-12-12T20:06:23.419378Z","iopub.status.idle":"2020-12-12T20:06:23.445684Z","shell.execute_reply":"2020-12-12T20:06:23.445039Z"},"papermill":{"duration":4.932867,"end_time":"2020-12-12T20:06:23.4458","exception":false,"start_time":"2020-12-12T20:06:18.512933","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv       = riiideducation.make_env ()\niter_test = env.iter_test ()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-12T20:06:33.601468Z","iopub.status.busy":"2020-12-12T20:06:33.600816Z","iopub.status.idle":"2020-12-12T20:06:34.273653Z","shell.execute_reply":"2020-12-12T20:06:34.274755Z"},"papermill":{"duration":5.602003,"end_time":"2020-12-12T20:06:34.274921","exception":false,"start_time":"2020-12-12T20:06:28.672918","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"model.eval ()\nprev_test_df = None\nfor (test_df, sample_prediction_df) in tqdm (iter_test):\n    \n    # test_df = test_df[test_df.content_type_id == False]\n    test_df = test_df.merge (questions_df, left_on='content_id', right_on='question_id', how='left').drop (columns=['question_id'])\n    if (prev_test_df is not None) & (psutil.virtual_memory ().percent<90):\n        \n        print (psutil.virtual_memory ().percent)\n        prev_test_df['answered_correctly'] = eval (test_df['prior_group_answers_correct'].iloc[0])\n        prev_test_df = prev_test_df[prev_test_df.content_type_id == False]\n        prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly', 'bundle_id', 'tags']].groupby ('user_id').apply (lambda r: (\n            r['content_id'].values,\n            r['answered_correctly'].values,\n            r['bundle_id'].values,\n            np.vstack (list(r['tags'].values))))\n        for prev_user_id in prev_group.index:\n            \n            prev_group_content = prev_group[prev_user_id][0]\n            prev_group_ac      = prev_group[prev_user_id][1]\n            prev_group_bund_id = prev_group[prev_user_id][2]\n            prev_group_tags    = prev_group[prev_user_id][3]\n            if prev_user_id in group.index:\n                \n                group[prev_user_id] = (np.append(group[prev_user_id][0],  prev_group_content), \n                                       np.append(group[prev_user_id][1],  prev_group_ac))\n            else:\n                group[prev_user_id] = (prev_group_content, prev_group_ac)\n            if len (group[prev_user_id][0]) > MAX_SEQ:\n                \n                new_group_content   = group[prev_user_id][0][-MAX_SEQ:]\n                new_group_ac        = group[prev_user_id][1][-MAX_SEQ:]\n                group[prev_user_id] = (new_group_content, new_group_ac)\n    prev_test_df    = test_df.copy ()    \n    test_df         = test_df[test_df.content_type_id == False]                \n    test_dataset    = TestDataset (group, test_df, skills)\n    test_dataloader = DataLoader (test_dataset, batch_size=1024, shuffle=False)  # TODO: uncomment this\n    outs            = []\n    softmax         = nn.Softmax (dim=1)\n    for item in tqdm (test_dataloader):\n        \n        # item = (x, questions, bundle_x, tags_x, b_target, t_target)\n        x         = item[0].to (device).long ()\n        target_id = item[1].to (device).long ()\n        bundle_x  = item[2].to (device).long ()\n        tags_x    = item[3].to (device).long ()\n        b_target  = item[4].to (device).long ()\n        t_target  = item[5].to (device).long ()\n        \n        with torch.no_grad ():\n            pred_logits = model (x, target_id, bundle_x, tags_x, b_target, t_target)  # returns a named tuple\n        pred_logits = pred_logits.logits.cpu ()\n        pred_pr     = softmax (pred_logits).numpy ()\n        pred_pr     = pred_pr[:, 1]   # return the prob of binary class=1\n        \n        # pred = (output >= 0.5).long()\n        # loss = criterion(output, label)\n        # val_loss.append(loss.item())\n        # num_corrects += (pred == label).sum().item()\n        # num_total += len(label)\n        # labels.extend(label.squeeze(-1).data.cpu().numpy())\n        outs.extend (pred_pr)\n        \n    test_df['answered_correctly'] =  outs\n    env.predict (test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":4.992076,"end_time":"2020-12-12T20:06:45.516878","exception":false,"start_time":"2020-12-12T20:06:40.524802","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"print (\"Done !\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}