{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#utils.py\n\nimport torch\nimport numpy as np\nimport pandas as pd #HDKIM\nfrom torch.autograd import Variable\n\ndef subsequent_mask(size):\n    \"Mask out subsequent positions.\"\n    attn_shape = (1, size, size)\n    subsequent_mask = np.triu(np.ones(attn_shape), k=1).astype('uint8')\n    return torch.from_numpy(subsequent_mask) == 0\n\ndef make_std_mask(x, pad):\n    \"Create a mask to hide padding and future words.\"\n    mask = torch.unsqueeze((x!=pad), -1)\n\n    tgt_mask = mask & Variable(\n        subsequent_mask(x.size(-1)).type_as(mask.data))\n    #         print('tgt_mask size after: ', tgt_mask.size())\n    return tgt_mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!nvcc --version\n!rm -rf ~/.nv/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# multihead_attn.py\n\nimport torch\nimport torch.nn as nn\nimport numpy as np\nimport math\nimport torch.nn.functional as F\nimport copy\nfrom torch.nn import LayerNorm\n\n\ndef clones(module, N):\n    \"Produce N identical layers.\"\n    return nn.ModuleList([copy.deepcopy(module) for _ in range(N)])\n\n\ndef attention(query, key, value, key_masks=None, query_masks=None, future_masks=None, dropout=None, infer=False):\n    \"Compute 'Scaled Dot Product Attention'\"\n    d_k = query.size(-1)\n    layernorm = LayerNorm(d_k).cuda()\n    # query shape = [nbatches, h, T_q, d_k]       key shape = [nbatches, h, T_k, d_k] == value shape\n    # scores shape = [nbatches, h, T_q, T_k]  == p_attn shape\n    scores = torch.matmul(query, key.transpose(-2, -1)) \\\n             / math.sqrt(d_k)\n    # if key_masks is not None:\n    #     scores = scores.masked_fill(key_masks.unsqueeze(1).cuda() == 0, -1e9)\n    if future_masks is not None:\n        scores = scores.masked_fill(future_masks.unsqueeze(0).cuda() == 0, -1e9)\n\n\n    p_attn = F.softmax(scores, dim=-1)\n    outputs = p_attn\n    # if query_masks is not None:\n    #     outputs = outputs * query_masks.unsqueeze(1)\n    if dropout is not None:\n        outputs = dropout(outputs)\n    outputs = torch.matmul(outputs, value)\n\n    outputs += query\n    return layernorm(outputs), p_attn\n\n\nclass MultiHeadedAttention(nn.Module):\n    def __init__(self, h, d_model, dropout=0.2, infer=False):\n        \"Take in model size and number of heads.\"\n        super(MultiHeadedAttention, self).__init__()\n        assert d_model % h == 0\n        # We assume d_v always equals d_k\n        self.d_k = d_model // h\n        self.h = h\n        self.linears = clones(nn.Linear(d_model, d_model), 4)\n        self.attn = None\n        self.dropout = nn.Dropout(p=dropout)\n        self.layernorm = LayerNorm(d_model).cuda()\n        self.infer = infer\n\n    def forward(self, query, key, value, key_masks=None, query_masks=None, future_masks=None):\n        nbatches = query.size(0)\n        \n        # 1) Do all the linear projections in batch from d_model => h x d_k\n        query, key, value = \\\n            [F.relu(l(x).view(nbatches, -1, self.h, self.d_k).transpose(1, 2), inplace=True)\n             for l, x in zip(self.linears, (query, key, value))]\n        # k v shape = [nbatches, h, T_k, d_k],  d_k * h = d_model\n        # q shape = [nbatches, h, T_q, d_k]\n        # 2) Apply attention on all the projected vectors in batch.\n        x, self.attn = attention(query, key, value, query_masks=query_masks,\n                                 key_masks=key_masks, future_masks=future_masks, dropout=self.dropout, infer=self.infer)\n\n        # 3) \"Concat\" using a view and apply a final linear.\n        x = x.transpose(1, 2).contiguous() \\\n            .view(nbatches, -1, self.h * self.d_k)\n        return self.layernorm(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/humanfriendly82/humanfriendly-8.2-py2.py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip  install ../input/coloredlogs140/coloredlogs-14.0-py2.py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#wordtest.py\n\nimport logging\nimport coloredlogs\nimport pickle\n\nlogger = logging.getLogger('__file__')\ncoloredlogs.install(level='INFO', logger=logger)\n\ndef pickle_io(path, mode='r', obj=None):\n    \"\"\"\n    Convinient pickle load and dump.\n    \"\"\"\n    if mode in ['rb', 'r']:\n        logger.info(\"Loading obj from {}...\".format(path))\n        with open(path, 'rb') as f:\n            obj = pickle.load(f)\n        logger.info(\"Load obj successfully!\")\n        return obj\n    elif mode in ['wb', 'w']:\n        logger.info(\"Dumping obj to {}...\".format(path))\n        with open(path, 'wb') as f:\n            pickle.dump(obj, f)\n        logger.info(\"Dump obj successfully!\")\n\nclass WordTestResource(object):\n\n    def __init__(self, resource_path, verbose=False):\n\n        resource = pickle_io(resource_path, mode='r')\n\n        self.id2index = resource['id2index']\n        self.index2id = resource['index2id']\n        self.num_skills = len(self.id2index)\n\n        if verbose:\n            self.word2id = resource['word2id']\n            self.id2all = resource['id2all']\n            # rank0 already be set to a large number\n            self.words_by_rank = resource['words_by_rank']\n            self.pos2id = resource['pos2id']\n            self.words_by_rank.sort(key=lambda x: x[u'rank'])\n            self.id_by_rank = [x[u'word_id'] for x in self.words_by_rank]\n\ndef str2bool(s):\n    if s not in {'False', 'True'}:\n        raise ValueError('Not a valid boolean string')\n    return s == 'True'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#config.py\n\nclass DefaultConfig(object):\n    model = 'SAKT'\n    #train_data = \"../input/assist2015files/assist2015_train.csv\"  # train_data_path\n    #test_data = \"../input/assist2015files/assist2015_test.csv\"\n    batch_size = 4 #HDKIM 256\n    state_size = 200\n    num_heads = 5\n    max_len = 50\n    dropout = 0.1\n    max_epoch = 5 #10\n    lr = 3e-3\n    lr_decay = 0.9\n    max_grad_norm = 1.0\n    weight_decay = 0  # l2正则化因子\n\nopt = DefaultConfig()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/prefetchgenerator101/prefetch_generator-1.0.1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# dataset.py\n\nimport csv\nimport torch\nimport time\nimport itertools\nimport numpy as np\n#from config import DefaultConfig\n#from wordtest import WordTestResource\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\nfrom prefetch_generator import BackgroundGenerator\n\nimport joblib #HDKIM\n\nclass Data(Dataset):\n    #HDKIM def __init__(self, train=True):\n    def __init__(self, df, train=True):\n        start_time = time.time()\n        #HDKIM if train:\n        #HDKIM    fileName = opt.train_data\n        #HDKIM else:\n        #HDKIM     fileName = opt.test_data\n        self.students = []\n        self.max_skill_num = 0\n        begin_index = 1e9\n        \n        #HDKIM with open(fileName, \"r\") as csvfile:\n            #HDKIM for num_ques, ques, ans in itertools.zip_longest(*[csvfile] * 3):\n                #HDKIM num_ques = int(num_ques.strip().strip(','))\n                #HDKIM ques = [int(q) for q in ques.strip().strip(',').split(',')]\n                #HDKIM ans = [int(a) for a in ans.strip().strip(',').split(',')]\n        for index, row in df.iterrows():\n                num_ques = int(row['num_ques'])\n                #print(row['num_ques'])\n                #print(row['ques'])\n                #print(row['ans'])\n                ques = [int(q) for q in row['ques']]\n                ans = [int(a) for a in row['ans']]\n                \n                tmp_max_skill = max(ques)\n                tmp_min_skill = min(ques)\n                begin_index = min(tmp_min_skill, begin_index)\n                self.max_skill_num = max(tmp_max_skill, self.max_skill_num)\n                \n                #HDKIM if (num_ques <= 2):\n                #HDKIM     continue\n                #HDKIM elif num_ques <= opt.max_len:\n                #HDKIM if num_ques <= opt.max_len:\n                '''\n                if num_ques <= opt.max_len:\n                    problems = np.zeros(opt.max_len, dtype=np.int64)\n                    correct = np.ones(opt.max_len, dtype=np.int64)\n                    problems[-num_ques:] = ques[-num_ques:]\n                    correct[-num_ques:] = ans[-num_ques:]\n                    self.students.append((num_ques, problems, correct))\n                else:\n                    start_idx = 0\n                    while opt.max_len + start_idx <= num_ques:\n                        problems = np.array(ques[start_idx:opt.max_len + start_idx])\n                        correct = np.array(ans[start_idx:opt.max_len + start_idx])\n                        tup = (opt.max_len, problems, correct)\n                        start_idx += opt.max_len\n                        self.students.append(tup)\n                    left_num_ques = num_ques - start_idx\n                ''' \n                #HDKIM\n                # first part of the student\n                copy_len = opt.max_len - 1\n                if copy_len > num_ques:\n                    copy_len = num_ques\n                problems = np.zeros(opt.max_len, dtype=np.int64)\n                correct = np.ones(opt.max_len, dtype=np.int64)\n                problems[-copy_len:] = ques[-copy_len:]\n                correct[-copy_len:] = ans[-copy_len:]\n                tup = (copy_len, problems, correct)\n                self.students.append(tup)\n                \n                if num_ques > opt.max_len - 1:\n                    start_idx = opt.max_len - 1\n                    while opt.max_len - 1 + start_idx <= num_ques:\n                        problems = np.array(ques[(start_idx-1):(start_idx + opt.max_len -1 )])\n                        correct = np.array(ans[(start_idx-1):(start_idx + opt.max_len -1)])\n                        tup = (opt.max_len, problems, correct)\n                        self.students.append(tup)\n                        start_idx += (opt.max_len-1)\n                    left_num_ques = num_ques - start_idx\n                    \n                    #HDKIM if left_num_ques>2: \n                    if left_num_ques>0:\n                        problems = np.zeros(opt.max_len, dtype=np.int64)\n                        correct = np.ones(opt.max_len, dtype=np.int64)\n                        problems[-left_num_ques:] = ques[-left_num_ques:]\n                        correct[-left_num_ques:] = ans[-left_num_ques:]\n                        tup = (left_num_ques, problems, correct)\n                        self.students.append(tup)\n                        \n        if train==False:\n            if len(self.students) % opt.batch_size > 0:\n                for i in range(opt.batch_size - (len(self.students) % opt.batch_size)):\n                    self.students.append(tup)\n                    \n        print(len(self.students))\n\n\n    def __getitem__(self, index):\n        student = self.students[index]\n        problems = student[1]\n        #print(\"before\",problems)\n        correct = student[2]\n        #HDKIM x = np.zeros(opt.max_len - 1)\n        x = problems[:-1].copy()\n        # we assume max_skill_num + 1 = num_skills because skill index starts from 0 to max_skill_num\n        x += (correct[:-1] == 1) * (self.max_skill_num + 1)\n        problems = problems[1:]\n        correct = correct[1:]\n        \n        #print(\"after\",problems)\n        \n        return x, problems, correct\n\n    def __len__(self):\n        return len(self.students)\n\n\n    \nclass DataLoaderX(DataLoader):\n\n    def __iter__(self):\n        return BackgroundGenerator(super().__iter__())\n\n\nclass DataPrefetcher():\n    def __init__(self, loader, device):\n        self.loader = iter(loader)\n        self.device = device\n        self.stream = torch.cuda.Stream()\n        # With Amp, it isn't necessary to manually convert data to half.\n        # if args.fp16:\n        #     self.mean = self.mean.half()\n        #     self.std = self.std.half()\n        self.preload()\n\n    def preload(self):\n        try:\n            self.batch = next(self.loader)\n        except StopIteration:\n            self.batch = None\n            return\n        with torch.cuda.stream(self.stream):\n            for k in range(len(self.batch)):\n                self.batch[k] = self.batch[k].to(device=self.device, non_blocking=True)\n\n            # With Amp, it isn't necessary to manually convert data to half.\n            # if args.fp16:\n            #     self.next_input = self.next_input.half()\n            # else:\n            #     self.next_input = self.next_input.float()\n\n    def next(self):\n        torch.cuda.current_stream().wait_stream(self.stream)\n        batch = self.batch\n        self.preload()\n        return batch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# student_model.py\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport math\n#from config import DefaultConfig\n#from utils import subsequent_mask\nfrom torch.autograd import Variable\n#from multihead_attn import MultiHeadedAttention\nfrom torch.nn import LayerNorm\n\nclass PositionalEncoding(nn.Module):\n    \"Implement the PE function.\"\n\n    def __init__(self, state_size, dropout=0.1, max_len=50):\n        super(PositionalEncoding, self).__init__()\n        self.dropout = nn.Dropout(p=dropout)\n\n        # Compute the positional encodings once in log space.\n        self.pe = torch.zeros(max_len, state_size)\n        position = torch.arange(0.0, max_len).unsqueeze(1)\n        div_term = torch.exp(torch.arange(0.0, state_size, 2) *\n                             -(math.log(10000.0) / state_size))\n        self.pe[:, 0::2] = torch.sin(position * div_term)\n        self.pe[:, 1::2] = torch.cos(position * div_term)\n        self.pe = self.pe.unsqueeze(0)\n    def forward(self, x):\n        x = x + Variable(self.pe[:, :x.size(1)],\n                         requires_grad=False)\n        return self.dropout(x)\n\n\nclass student_model(nn.Module):\n\n    def __init__(self, num_skills, state_size, num_heads=2, dropout=0.2, infer=False):\n        super(student_model, self).__init__()\n        self.infer = infer\n        self.num_skills = num_skills\n        self.state_size = state_size\n        # we use the (num_skills * 2 + 1) as key padding_index\n        self.embedding = nn.Embedding(num_embeddings=num_skills*2+1,\n                                      embedding_dim=state_size)\n                                      # padding_idx=num_skills*2\n        # self.position_embedding = PositionalEncoding(state_size)\n        self.position_embedding = nn.Embedding(num_embeddings=opt.max_len-1,\n                                               embedding_dim=state_size)\n        # we use the (num_skills + 1) as query padding_index\n        self.problem_embedding = nn.Embedding(num_embeddings=num_skills+1,\n                                      embedding_dim=state_size)\n                                      # padding_idx=num_skills)\n        self.multi_attn = MultiHeadedAttention(h=num_heads, d_model=state_size, dropout=dropout, infer=self.infer)\n        self.feedforward1 = nn.Linear(in_features=state_size, out_features=state_size)\n        self.feedforward2 = nn.Linear(in_features=state_size, out_features=state_size)\n        self.pred_layer = nn.Linear(in_features=state_size, out_features=num_skills)\n        self.dropout = nn.Dropout(dropout)\n        self.layernorm = LayerNorm(state_size)\n\n    def forward(self, x, problems, target_index):\n        # self.key_masks = torch.unsqueeze( (x!=self.num_skills*2).int(), -1)\n        # self.problem_masks = torch.unsqueeze( (problems!=self.num_skills).int(), -1)\n        x = self.embedding(x)\n        pe = self.position_embedding(torch.arange(x.size(1)).unsqueeze(0).cuda())\n        x += pe\n        # x = self.position_embedding(x)\n        problems = self.problem_embedding(problems)\n        # self.key_masks = self.key_masks.type_as(x)\n        # self.problem_masks = self.problem_masks.type_as(problems)\n        # x *= self.key_masks\n        # problems *= self.problem_masks\n        x = self.dropout(x)\n        res = self.multi_attn(query=self.layernorm(problems), key=x, value=x,\n                              key_masks=None, query_masks=None, future_masks=None)\n        outputs = F.relu(self.feedforward1(res))\n        outputs = self.dropout(outputs)\n        outputs = self.dropout(self.feedforward2(outputs))\n        # Residual connection\n        outputs += self.layernorm(res)\n        outputs = self.layernorm(outputs)\n        logits = self.pred_layer(outputs)\n        \n        #HDKIM logits = logits.contiguous().view(logits.size(0) * opt.max_len - 1, -1)\n        logits = logits.contiguous().view(logits.size(0) * (opt.max_len - 1), -1)\n        logits = logits.contiguous().view(-1)\n        selected_logits = torch.gather(logits, 0, torch.LongTensor(target_index).cuda())\n        return selected_logits","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# run.py\n\nimport time\nimport torch\nimport numpy as np\nimport torch.nn as nn\n#from dataset import DataPrefetcher\n#from config import DefaultConfig\nfrom sklearn.metrics import mean_squared_error\nfrom math import sqrt\nfrom sklearn import metrics\nfrom sklearn.metrics import r2_score\n\ndef run_epoch(m, dataloader, optimizer, scheduler, criterion, num_skills,\n                  epoch_id=None, writer=None, is_training=True):\n    epoch_start_time = time.time()\n    if is_training:\n        m.train()\n    else:\n        m.eval()\n    m.cuda()\n    actual_labels = []\n    pred_labels = []\n    num_batch = len(dataloader)\n    prefetcher = DataPrefetcher(dataloader, device='cuda')\n    batch = prefetcher.next()\n    k = 0\n\n    if is_training:\n        while batch is not None:\n            target_index = []\n            x, problems, correctness = batch\n            x = x.long()\n            problems = problems.long()\n            correctness = correctness.view(-1).float()\n\n            #HDKIM actual_labels += list(np.array(correctness))\n            actual_labels += list(np.array(correctness.cpu()))\n            offset = 0\n            helper = np.array(problems.cpu()).reshape(-1)\n            for i in range(problems.size(0)):\n                for j in range(problems.size(1)):\n                    target_index.append((offset + helper[i * problems.size(1) + j])) #HDKIM j+1 -> j?\n                    offset += num_skills\n\n            logits = m(x, problems, target_index) #HDKIM , correctness)\n            pred = torch.sigmoid(logits)\n            loss = criterion(pred, correctness.cuda())\n            optimizer.zero_grad()\n            loss.backward()\n            nn.utils.clip_grad_norm_(m.parameters(), opt.max_grad_norm)\n            \n            #HDKIM\n            # GPU\n            # optimizer.step()\n            # TPU\n            xm.optimizer_step(optimizer, barrier=True) \n            \n            scheduler.step()\n            pred_labels += list(np.array(pred.data.cpu()))\n            k += 1\n            if k % 500 == 0:\n                print('\\r batch{}/{}'.format(k, num_batch), end='')\n            #HDKIM if k >= num_batch - 1:\n            if k >= num_batch:\n                break\n            batch = prefetcher.next()\n    else:\n        with torch.no_grad():\n            while batch is not None:\n                target_index = []\n                x, problems, correctness = batch\n                x = x.long()\n                actual_num_problems = torch.sum(problems != num_skills, dim=1)\n                num_problems = problems.size(1)\n                problems = problems.long()\n                correctness = correctness.view(-1).float()\n                offset = 0\n                helper = np.array(problems.cpu()).reshape(-1)\n                for i in range(problems.size(0)):\n                    for j in range(problems.size(1)):\n                        target_index.append((offset + helper[i * problems.size(1) + j]))\n                        offset += num_skills\n\n                logits = m(x, problems, target_index) #HDKIM , correctness)\n                pred = torch.sigmoid(logits)\n                for J in range(x.size(0)):\n                    actual_num_problem = actual_num_problems[J]\n                    num_to_throw = num_problems - actual_num_problem\n\n                    pred[J * num_problems:J * num_problems + num_to_throw] = correctness[\n                                                                             J * num_problems:J * num_problems + num_to_throw]\n                #HDKIM actual_labels += list(np.array(correctness))\n                actual_labels += list(np.array(correctness.cpu()))\n\n                pred_labels += list(np.array(pred.data.cpu()))\n                \n                k += 1\n                if k % 500 == 0:\n                    print('\\r batch{}/{}'.format(k, num_batch), end='')\n   \n                #HDKIM last batch removing\n                #HDKIM if k >= num_batch - 1:\n                if k>=num_batch:\n                    break\n                batch = prefetcher.next()\n\n\n    rmse = sqrt(mean_squared_error(actual_labels, pred_labels))\n    fpr, tpr, thresholds = metrics.roc_curve(actual_labels, pred_labels, pos_label=1)\n    auc = metrics.auc(fpr, tpr)\n    r2 = r2_score(actual_labels, pred_labels)\n    acc = metrics.accuracy_score(actual_labels, np.array(pred_labels) >= 0.5)\n    epoch_end_time = time.time()\n    print('Epoch costs %.2f s' % (epoch_end_time - epoch_start_time))\n    #HDKIM return rmse, auc, r2, acc\n    return rmse, auc, r2, acc, pred_labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# main.py\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\n#from dataset import Data\n#from dataset import DataLoaderX\n#from config import DefaultConfig\n#from student_model import student_model\n#from run import run_epoch\n\nif __name__ == '__main__':\n\n    #ques = pd.read_csv(\"../input/riiid-test-answer-prediction/questions.csv\")\n    #print(ques.head())\n    #print(ques.question_id.max()) #13522 -> num_skills = 13522+1\n    \n    num_skills = 13523\n    m = student_model(num_skills=num_skills, state_size=opt.state_size,\n                      num_heads=opt.num_heads, dropout=opt.dropout, infer=False)\n   \n    PATH = '../input/sakt-self-attentive-knowledge-tracing/sakt_model_auc_920.pkl'\n    m.load_state_dict(torch.load(PATH))\n    m.eval()\n    \n    torch.backends.cudnn.benchmark = True\n    best_auc = 0\n    optimizer = optim.Adam(m.parameters(), lr=opt.lr, weight_decay=opt.weight_decay)\n    scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=1000, gamma=opt.lr_decay)\n    criterion = nn.BCELoss()\n\n    import riiideducation\n    \n    env = riiideducation.make_env()\n    iter_test = env.iter_test()\n    \n    for (test_df, sample_prediction_df) in iter_test:\n        test_df = test_df[test_df.content_type_id == False].reset_index(drop=True)\n        test_df = test_df.sort_values(['user_id','timestamp'], ascending=True).reset_index(drop = True)\n        \n        num_ques = test_df[['user_id','content_id']].groupby('user_id')['content_id'].count().to_frame()\n        num_ques.columns = [\"num_ques\"]\n        ques = test_df[['user_id','content_id']].groupby('user_id')['content_id'].apply(list).to_frame()\n        ques.columns = ['ques']\n        num_ques = num_ques.merge(ques,on=\"user_id\",how=\"left\")\n        # creating dummy answers\n        anses = []\n        for que in num_ques.ques:\n            anses.append([1]*len(que)) \n        num_ques[\"ans\"] = anses\n\n        test_dataset = Data(num_ques,train=False)\n        test_loader = DataLoaderX(test_dataset, batch_size=opt.batch_size, num_workers=4, pin_memory=True)\n        \n        epoch = opt.max_epoch\n        \n        rmse, auc, r2, acc, preds = run_epoch(m, test_loader, optimizer, scheduler, criterion,\n                                       num_skills=num_skills, epoch_id=epoch, is_training=False)\n        \n        final_preds = []\n        start_idx = 0        \n        for num in num_ques.num_ques:\n            copy_len = opt.max_len - 1\n            if copy_len > num:\n                copy_len = num\n            next_idx = start_idx + opt.max_len - 1\n            final_preds.append(preds[(next_idx - copy_len):next_idx])\n            start_idx = next_idx\n            if num > opt.max_len - 1:\n                while opt.max_len - 1 + (next_idx-start_idx) <= num:\n                    final_preds.append(preds[next_idx:(next_idx + opt.max_len - 1)])            \n                    next_idx += (opt.max_len - 1)\n                left_num = num - (next_idx-start_idx)\n                if left_num>0:\n                    next_idx += (opt.max_len - 1)\n                    final_preds.append(preds[(next_idx-left_num):next_idx])\n                start_idx = next_idx  \n        final_preds = np.concatenate(final_preds)\n        \n        if len(final_preds) == test_df.shape[0]:\n            test_df['answered_correctly'] =  final_preds\n        else:\n            test_df['answered_correctly'] = 0.65\n            \n        env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n    print(\"mission completed!\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}