{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"# add part emb\n# add elapsed time\n# saint model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nfrom sklearn.metrics import roc_auc_score\nfrom collections import defaultdict\nfrom tqdm.notebook import tqdm\nimport lightgbm as lgb\nimport os\nimport pickle\n\n\ndebug = True\nvalidaten_flg = False\nEXP_ID = 'exp004'\nCV_NUM = 'cv1'\n\n\ndef unpickle(filename):\n    with open(filename, mode='rb') as fo:\n        p = pickle.load(fo)\n    return p\n\n\nclass CFG:\n    INPUT_DIR = '../input/riiid-test-answer-prediction'\n    OUT_DIR = '../input/riiid-trained-models-exp004'\n    DATA_DIR = '../input/riiid-cv-datasets'\n    TRAIN_PATH = os.path.join(INPUT_DIR, \"train.csv\")\n    TEST_PATH = os.path.join(INPUT_DIR, \"example_test.csv\")\n    LECTURE_PATH = os.path.join(INPUT_DIR, \"lectures.csv\")\n    QUESTION_PATH = os.path.join(INPUT_DIR, \"questions.csv\")\n    SUBMISSION_PATH = os.path.join(INPUT_DIR, \"example_sample_submission.csv\")\n    SEED = 6718\n    \n    \ntrain_pickle = '../input/riiid-cross-validation-files/cv1_train.pickle'\nvalid_pickle = '../input/riiid-cross-validation-files/cv1_valid.pickle'\nquestion_file = '../input/riiid-test-answer-prediction/questions.csv'\ndebug = True\nvalidaten_flg = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\nimport torch\nimport torch.nn as nn\nimport torch.nn.utils.rnn as rnn_utils\nfrom torch.autograd import Variable\nfrom torch.utils.data import Dataset, DataLoader\n\nimport psutil\nimport os\nimport time\nimport sys\nimport math\nfrom contextlib import contextmanager\n\n@contextmanager\ndef trace(title):\n    t0 = time.time()\n    p = psutil.Process(os.getpid())\n    m0 = p.memory_info()[0] / 2. ** 30\n    yield\n    m1 = p.memory_info()[0] / 2. ** 30\n    delta = m1 - m0\n    sign = '+' if delta >= 0 else '-'\n    delta = math.fabs(delta)\n    print(f\"[{m1:.1f}GB({sign}{delta:.1f}GB):{time.time() - t0:.1f}sec] {title} \", file=sys.stderr)\n    \n\ndef seed_everything(seed=1129):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \n    \nseed_everything(CFG.SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with trace(''):\n    # read data\n    #feld_needed = ['timestamp', 'user_id', 'content_id', 'content_type_id', 'answered_correctly', 'prior_question_elapsed_time']\n    #train = pd.read_pickle(train_pickle)[feld_needed] # [-10000000:]\n    #valid = pd.read_pickle(valid_pickle)[feld_needed] # [-2500000:]\n\n\n    #train = train.loc[train.content_type_id == False].reset_index(drop=True)\n    #valid = valid.loc[valid.content_type_id == False].reset_index(drop=True)\n    \n    \n    # part\n    questions_df = pd.read_csv(question_file)[['question_id', 'part', 'tags', 'bundle_id']]\n\n    #train = train.merge(questions_df[questions_df.question_id.isin(train.content_id)][['part']],\n    #            how='left', left_on='content_id', right_index=True)\n\n    #valid = valid.merge(questions_df[questions_df.question_id.isin(valid.content_id)][['part']],\n    #            how='left', left_on='content_id', right_index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pickle\n\n\ndef to_pickle(filename, obj):\n    with open(filename, mode='wb') as f:\n        pickle.dump(obj, f)\n\ndef unpickle(filename):\n    with open(filename, mode='rb') as fo:\n        p = pickle.load(fo)\n    return p  \n\n\nwith trace(''):\n    group = unpickle('../input/riiid-transformer-all-v7-add-elapsed-time-pre/group.pkl')\n    val_group = unpickle('../input/riiid-transformer-all-v7-add-elapsed-time-pre/val_group.pkl')\n    skills = unpickle('../input/riiid-transformer-all-v7-add-elapsed-time-pre/skills.pkl')\n    # lag_time_n_unq = unpickle('../input/riiid-transformer-all-v5-add-elapsed-time-pre/lag_time_n_unq.pkl')\n    elapsed_time_n_unq = unpickle('../input/riiid-transformer-all-v7-add-elapsed-time-pre/elapsed_time_n_unq.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_SEQ = 160\n\n\nwith trace(''):\n    # skills = train[\"content_id\"].unique()\n    # n_skill = len(skills)\n    n_skill = 13523\n    print(\"number skills\", len(skills))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SAKTDataset(Dataset):\n    def __init__(self, group, n_skill, max_seq=MAX_SEQ): #HDKIM 100\n        super(SAKTDataset, self).__init__()\n        self.max_seq = max_seq\n        self.n_skill = n_skill\n        self.samples = {}\n        \n        self.user_ids = []\n        for user_id in group.index:\n            q, qa, pt, elt = group[user_id]\n            if len(q) < 5:\n                continue\n            \n            # Main Contribution\n            if len(q) > self.max_seq:\n                total_questions = len(q)\n                initial = total_questions % self.max_seq\n                if initial >= 5:\n                    self.user_ids.append(f\"{user_id}_0\")\n                    self.samples[f\"{user_id}_0\"] = (q[:initial], qa[:initial], pt[:initial], elt[:initial])\n                for seq in range(total_questions // self.max_seq):\n                    self.user_ids.append(f\"{user_id}_{seq+1}\")\n                    start = initial + seq * self.max_seq\n                    end = start + self.max_seq\n                    self.samples[f\"{user_id}_{seq+1}\"] = (q[start:end], qa[start:end], pt[start:end], elt[start:end])\n            else:\n                user_id = str(user_id)\n                self.user_ids.append(user_id)\n                self.samples[user_id] = (q, qa, pt, elt)\n\n    def __len__(self):\n        return len(self.user_ids)\n\n    def __getitem__(self, index):\n        user_id = self.user_ids[index]\n        q_, qa_, pt_, elt_ = self.samples[user_id]\n        seq_len = len(q_)\n\n        q = np.zeros(self.max_seq, dtype=int)\n        qa = np.zeros(self.max_seq, dtype=int)\n        pt = np.zeros(self.max_seq, dtype=int)\n        elt = np.zeros(self.max_seq, dtype=int)\n        \n        if seq_len == self.max_seq:\n            q[:] = q_\n            qa[:] = qa_\n            pt[:] = pt_\n            elt[:] = elt_\n\n        else:\n            q[-seq_len:] = q_\n            qa[-seq_len:] = qa_\n            pt[-seq_len:] = pt_\n            elt[-seq_len:] = elt_\n\n        \n        target_id = q[1:]\n        label = qa[1:]\n        part_id = pt[1:]\n        \n        elapsed_time = elt[:-1]\n\n        x = np.zeros(self.max_seq-1, dtype=int)\n        x = q[:-1].copy()\n        x += (qa[:-1] == 1) * self.n_skill\n\n        # x = qa[:-1].copy()\n        return x, target_id, part_id, elapsed_time, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch \nimport torch.nn as nn\nimport torch.nn.functional as F \nimport numpy as np\nimport copy\n\n\ndef get_clones(module, N):\n    return nn.ModuleList([copy.deepcopy(module) for i in range(N)])\n\n\ndef get_mask(seq_len):\n    ##todo add this to device\n    return torch.from_numpy( np.triu(np.ones((seq_len ,seq_len)), k=1).astype('bool')).to(device)\n\ndef get_pos(seq_len):\n    # use sine positional embeddinds\n    return torch.arange( seq_len ).unsqueeze(0).to(device) \n\n\nclass Feed_Forward_block(nn.Module):\n    \"\"\"\n    out =  Relu( M_out*w1 + b1) *w2 + b2\n    \"\"\"\n    def __init__(self, dim_ff):\n        super().__init__()\n        self.layer1 = nn.Linear(in_features=dim_ff , out_features=dim_ff)\n        self.layer2 = nn.Linear(in_features=dim_ff , out_features=dim_ff)\n\n    def forward(self,ffn_in):\n        return  self.layer2(   F.relu( self.layer1(ffn_in) )   )\n        \n\nclass Encoder_block(nn.Module):\n    \"\"\"\n    M = SkipConct(Multihead(LayerNorm(Qin;Kin;Vin)))\n    O = SkipConct(FFN(LayerNorm(M)))\n    \"\"\"\n\n    def __init__(self , dim_model, heads_en, total_ex ,total_pt, seq_len):\n        super().__init__()\n        self.seq_len = seq_len\n        self.embd_ex =   nn.Embedding( total_ex , embedding_dim = dim_model )                   # embedings  q,k,v = E = exercise ID embedding, category embedding, and positionembedding.\n        self.embd_pt =  nn.Embedding( total_pt, embedding_dim = dim_model )\n        self.embd_pos   = nn.Embedding(  seq_len , embedding_dim = dim_model )                  #positional embedding\n\n        self.multi_en = nn.MultiheadAttention( embed_dim= dim_model, num_heads= heads_en,  )     # multihead attention    ## todo add dropout, LayerNORM\n        self.ffn_en = Feed_Forward_block( dim_model )                                            # feedforward block     ## todo dropout, LayerNorm\n        self.layer_norm1 = nn.LayerNorm( dim_model )\n        self.layer_norm2 = nn.LayerNorm( dim_model )\n\n\n    def forward(self, in_ex, in_pt, first_block=True):\n\n        ## todo create a positional encoding ( two options numeric, sine)\n        if first_block:\n            in_ex = self.embd_ex( in_ex )\n            in_pt = self.embd_pt( in_pt )\n            #in_pos = self.embd_pos( in_pos )\n            #combining the embedings\n            out = in_ex + in_pt #+ in_pos                      # (b,n,d)\n        else:\n            out = in_ex\n        \n        in_pos = get_pos(self.seq_len)\n        in_pos = self.embd_pos( in_pos )\n        out = out + in_pos                                      # Applying positional embedding\n\n        out = out.permute(1,0,2)                                # (n,b,d)  # print('pre multi', out.shape )\n        \n        #Multihead attention                            \n        n,_,_ = out.shape\n        out = self.layer_norm1( out )                           # Layer norm\n        skip_out = out \n        out, attn_wt = self.multi_en( out , out , out ,\n                                attn_mask=get_mask(seq_len=n))  # attention mask upper triangular\n        out = out + skip_out                                    # skip connection\n\n        #feed forward\n        out = out.permute(1,0,2)                                # (b,n,d)\n        out = self.layer_norm2( out )                           # Layer norm \n        skip_out = out\n        out = self.ffn_en( out )\n        out = out + skip_out                                    # skip connection\n\n        return out\n\n\nclass Decoder_block(nn.Module):\n    \"\"\"\n    M1 = SkipConct(Multihead(LayerNorm(Qin;Kin;Vin)))\n    M2 = SkipConct(Multihead(LayerNorm(M1;O;O)))\n    L = SkipConct(FFN(LayerNorm(M2)))\n    \"\"\"\n\n    def __init__(self, dim_model, total_in, total_et, heads_de, seq_len  ):\n        super().__init__()\n        self.seq_len    = seq_len\n        self.embd_in    = nn.Embedding(total_in, embedding_dim = dim_model)                  #interaction embedding\n        self.embd_et    = nn.Embedding(total_et, embedding_dim = dim_model)                  #elapsed time embedding\n        self.embd_pos   = nn.Embedding(seq_len, embedding_dim = dim_model)                  #positional embedding\n        self.multi_de1  = nn.MultiheadAttention(embed_dim=dim_model, num_heads=heads_de)  # M1 multihead for interaction embedding as q k v\n        self.multi_de2  = nn.MultiheadAttention(embed_dim=dim_model, num_heads=heads_de)  # M2 multihead for M1 out, encoder out, encoder out as q k v\n        self.ffn_en     = Feed_Forward_block(dim_model)                                         # feed forward layer\n\n        self.layer_norm1 = nn.LayerNorm(dim_model)\n        self.layer_norm2 = nn.LayerNorm(dim_model)\n        self.layer_norm3 = nn.LayerNorm(dim_model)\n\n\n    def forward(self, in_in, in_et, en_out, first_block=True):\n\n         ## todo create a positional encoding ( two options numeric, sine)\n        if first_block:\n            in_in = self.embd_in(in_in)\n            in_et = self.embd_et(in_et)\n\n            #combining the embedings\n            out = in_in + in_et #+ in_cat #+ in_pos                         # (b,n,d)\n        else:\n            out = in_in\n\n        in_pos = get_pos(self.seq_len)\n        in_pos = self.embd_pos( in_pos )\n        out = out + in_pos                                          # Applying positional embedding\n\n        out = out.permute(1,0,2)                                    # (n,b,d)# print('pre multi', out.shape )\n        n,_,_ = out.shape\n\n        #Multihead attention M1                                     ## todo verify if E to passed as q,k,v\n        out = self.layer_norm1( out )\n        skip_out = out\n        out, attn_wt = self.multi_de1( out , out , out, \n                                     attn_mask=get_mask(seq_len=n)) # attention mask upper triangular\n        out = skip_out + out                                        # skip connection\n\n        #Multihead attention M2                                     ## todo verify if E to passed as q,k,v\n        en_out = en_out.permute(1,0,2)                              # (b,n,d)-->(n,b,d)\n        en_out = self.layer_norm2( en_out )\n        skip_out = out\n        out, attn_wt = self.multi_de2( out , en_out , en_out,\n                                    attn_mask=get_mask(seq_len=n))  # attention mask upper triangular\n        out = out + skip_out\n\n        #feed forward\n        out = out.permute(1,0,2)                                    # (b,n,d)\n        out = self.layer_norm3( out )                               # Layer norm \n        skip_out = out\n        out = self.ffn_en( out )                                    \n        out = out + skip_out                                        # skip connection\n\n        return out\n    \n    \nclass SAINT(nn.Module):\n    def __init__(self, dim_model, num_en, num_de, heads_en, total_ex, total_pt, total_in, total_et, heads_de, seq_len):\n        super().__init__( )\n\n        self.num_en = num_en\n        self.num_de = num_de\n\n        self.encoder = get_clones(Encoder_block(dim_model, heads_en, total_ex, total_pt, seq_len), num_en)\n        self.decoder = get_clones(Decoder_block(dim_model ,total_in, total_et, heads_de, seq_len), num_de)\n\n        self.out = nn.Linear(in_features= dim_model, out_features=1)\n    \n    def forward(self, in_in, in_ex, in_pt, in_et):\n        \n        ## pass through each of the encoder blocks in sequence\n        first_block = True\n        for x in range(self.num_en):\n            if x>=1:\n                first_block = False\n            in_ex = self.encoder[x](in_ex, in_pt, first_block=first_block)\n            in_pt = in_ex                                  # passing same output as q,k,v to next encoder block\n\n        \n        ## pass through each decoder blocks in sequence\n        first_block = True\n        for x in range(self.num_de):\n            if x>=1:\n                first_block = False\n            in_in = self.decoder[x](in_in, in_et, en_out=in_ex, first_block=first_block)\n\n        ## Output layer\n        # in_in = torch.sigmoid( self.out( in_in ) )\n        in_in = self.out( in_in )\n        return in_in.squeeze(-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with trace(''):\n    n_part = 7\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    \n    model = SAINT(dim_model=128, \n                  num_en=3, \n                  num_de=3, \n                  heads_en=8, \n                  total_ex=n_skill+1, \n                  total_pt=n_part+1, \n                  total_in=2*n_skill+1, \n                  total_et=elapsed_time_n_unq+1, \n                  heads_de=8, \n                  seq_len=MAX_SEQ-1)\n    \n    model.load_state_dict(torch.load('../input/riiid-transformer-add-part-v11/SAKT-HDKIM.pt'))\n\n    model.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ADD = [\n       'answered_correctly_true_weight_sum_u',\n       'count_u_c',\n       'count_c',\n    \n       'task_container_id', \n       'task_container_count',\n\n       'answered_correctly_u_avg', 'elapsed_time_u_avg', 'explanation_u_avg',\n       'answered_correctly_q_avg', 'elapsed_time_q_avg', 'explanation_q_avg', \n       'answered_correctly_uq_count', 'timestamp_u_recency_1', 'timestamp_u_recency_2', 'timestamp_u_recency_3', \n       'timestamp_u_incorrect_recency',\n    \n        'part_bundle_id', \n        'tags1', 'tags2', 'tags3', \n        'tags4', 'tags5', 'tags6', \n        'part_bundle_count',\n        'count_u_tag1',\n        'count_u_tag2',\n        'count_u_tag3',\n        'count_u_part_bundle_id',\n        'task_container_id_u',\n      ]\n\nTARGET = 'answered_correctly'\nFEATS = ['answered_correctly_avg_u', 'answered_correctly_sum_u', 'count_u', 'answered_correctly_avg_c', 'part', 'prior_question_had_explanation', 'prior_question_elapsed_time']\nFEATS = FEATS + ADD","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\ndro_cols = list(set(train.columns) - set(FEATS))\ny_tr = train[TARGET]\ny_va = valid[TARGET]\ntrain.drop(dro_cols, axis=1, inplace=True)\nvalid.drop(dro_cols, axis=1, inplace=True)\n_=gc.collect()\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nlgb_train = lgb.Dataset(train[FEATS], y_tr)\nlgb_valid = lgb.Dataset(valid[FEATS], y_va)\ndel train, y_tr\n_=gc.collect()\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nparams = {'objective': 'binary', \n          'seed': CFG.SEED,\n          # 'metric': 'auc',\n          'num_leaves': 200,\n          'feature_fraction': 0.75,\n          'bagging_freq': 10,\n          'bagging_fraction': 0.80\n         }\n    \n\nmodel = lgb.train(\n                    params=params,\n                    train_set=lgb_train,\n                    valid_sets=[lgb_train, lgb_valid],\n                    verbose_eval=100,\n                    num_boost_round=10000,\n                    early_stopping_rounds=10\n                )\nprint('auc:', roc_auc_score(y_va, model.predict(valid[FEATS])))\n_ = lgb.plot_importance(model)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# del lgb_train, lgb_valid; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, samples, test_df, skills, max_seq=MAX_SEQ): #HDKIM 100\n        super(TestDataset, self).__init__()\n        self.samples = samples\n        self.user_ids = [x for x in test_df[\"user_id\"].unique()]\n        self.test_df = test_df\n        self.skills = skills\n        self.n_skill = len(skills)\n        self.max_seq = max_seq\n\n    def __len__(self):\n        return self.test_df.shape[0]\n\n    def __getitem__(self, index):\n        test_info = self.test_df.iloc[index]\n\n        user_id = test_info[\"user_id\"]\n        target_id = test_info[\"content_id\"]\n        part_id = test_info[\"part\"]\n        # elapsed_time = test_info[\"prior_question_elapsed_time\"]\n\n        q = np.zeros(self.max_seq, dtype=int)\n        qa = np.zeros(self.max_seq, dtype=int)\n        pt = np.zeros(self.max_seq, dtype=int)\n        elt = np.zeros(self.max_seq, dtype=int)\n        \n        if user_id in self.samples.index:\n            q_, qa_, pt_, elt_ = self.samples[user_id]\n            \n            seq_len = len(q_)\n\n            if seq_len >= self.max_seq:\n                q = q_[-self.max_seq:]\n                qa = qa_[-self.max_seq:]\n                pt = pt_[-self.max_seq:]\n                elt = elt_[-self.max_seq:]\n            else:\n                q[-seq_len:] = q_\n                qa[-seq_len:] = qa_  \n                pt[-seq_len:] = pt_    \n                elt[-seq_len:] = elt_      \n        \n        x = np.zeros(self.max_seq-1, dtype=int)\n        x = q[1:].copy()\n        x += (qa[1:] == 1) * self.n_skill\n        \n        # x = qa[:-1].copy()\n        \n        questions = np.append(q[2:], [target_id])\n        parts = np.append(pt[2:], [part_id])\n\n        elapsed_times = elt[:-1]\n        # elapsed_times = np.append(elt[2:], [elapsed_time])\n        \n        return x, questions, parts, elapsed_times","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-12T18:42:48.174757Z","iopub.status.busy":"2020-10-12T18:42:48.173901Z","iopub.status.idle":"2020-10-12T18:42:49.112276Z","shell.execute_reply":"2020-10-12T18:42:49.111472Z"},"papermill":{"duration":1.016814,"end_time":"2020-10-12T18:42:49.112404","exception":false,"start_time":"2020-10-12T18:42:48.09559","status":"completed"},"scrolled":true,"tags":[],"trusted":true},"cell_type":"code","source":"import riiideducation\nimport psutil\nmodel.eval()\n\n\nenv = riiideducation.make_env()\niter_test = env.iter_test()\nset_predict = env.predict\n\n\nprevious_test_df = None\nfor (test_df, sample_prediction_df) in iter_test:\n    if (previous_test_df is not None) & (psutil.virtual_memory().percent<90):\n        print(psutil.virtual_memory().percent)\n        previous_test_df[TARGET] = eval(test_df[\"prior_group_answers_correct\"].iloc[0])\n        previous_test_df = previous_test_df[previous_test_df.content_type_id == False]\n        \n        previous_test_df = previous_test_df.merge(questions_df[questions_df.question_id.isin(previous_test_df.content_id)][['part']],\n                how='left', left_on='content_id', right_index=True)\n    \n        prev_group = previous_test_df[['user_id', 'content_id', 'answered_correctly', 'part', 'timestamp', 'prior_question_elapsed_time']].groupby('user_id').apply(lambda r: (\n            r['content_id'].values,\n            r['answered_correctly'].values,\n            r['part'].values,\n            (r['prior_question_elapsed_time'].fillna(0).values/1000).astype(np.int64),\n        ))\n        \n        for prev_user_id in prev_group.index:\n            if prev_user_id in group.index:\n                group[prev_user_id] = (\n                    np.append(group[prev_user_id][0], prev_group[prev_user_id][0])[-MAX_SEQ:], \n                    np.append(group[prev_user_id][1], prev_group[prev_user_id][1])[-MAX_SEQ:],\n                    np.append(group[prev_user_id][2], prev_group[prev_user_id][2])[-MAX_SEQ:], \n                    np.append(group[prev_user_id][3], prev_group[prev_user_id][3])[-MAX_SEQ:]\n                )\n \n            else:\n                group[prev_user_id] = (\n                    prev_group[prev_user_id][0], \n                    prev_group[prev_user_id][1],\n                    prev_group[prev_user_id][2],\n                    prev_group[prev_user_id][3]\n                )\n        \n        # update_features_v4(previous_test_df, answered_correctly_sum_u_dict, count_u_dict,\n        #                    pqhe_true_answered_correctly_sum_u_dict, count_c_dict,\n        #                    answered_correctly_u_sum, answered_correctly_q_sum, timestamp_u_incorrect)\n    \n    \n    previous_test_df = test_df.copy()\n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n    test_df = test_df.merge(questions_df[questions_df.question_id.isin(test_df.content_id)][['part']],\n                how='left', left_on='content_id', right_index=True)\n    \n    test_dataset = TestDataset(group, test_df, skills)\n    test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)\n    \n    outs = []\n\n    for item in tqdm(test_dataloader):\n        x = item[0].to(device).long()\n        target_id = item[1].to(device).long()\n        part_id = item[2].to(device).long()\n        elapsed_time = item[3].to(device).long()\n\n        with torch.no_grad():\n            output = model(x, target_id, part_id, elapsed_time)\n        \n        \n        output = torch.sigmoid(output)\n        output = output[:, -1]\n        \n        outs.extend(output.view(-1).data.cpu().numpy())\n        \n    \n    \"\"\"\n    test_df = test_df.merge(questions_df[questions_df.question_id.isin(test_df.content_id)],\n                how='left', left_on='content_id', right_index=True)\n    test_df = test_df.merge(part_bundle_df[part_bundle_df.part_bundle_id.isin(test_df.part_bundle_id)],\n                        how='left', on='part_bundle_id')\n    test_df = test_df.merge(task_container_df[task_container_df.task_container_id.isin(test_df.task_container_id)],\n                        how='left', on='task_container_id')\n    test_df = test_df.merge(content_df[content_df.content_id.isin(test_df.content_id)],\n                        how='left', on='content_id')\n    \n    test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False).astype('int8')\n    test_df['prior_question_elapsed_time_mean'] = test_df.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)\n    \n    test_df[TARGET] = 0\n    test_df = add_user_feats_v4(test_df, count_u_tag1_dict, count_u_tag2_dict, count_u_tag3_dict,\n                                count_u_part_bundle_id_dict, task_container_id_u_dict,\n                                answered_correctly_sum_u_dict, count_u_dict,\n                                pqhe_true_answered_correctly_sum_u_dict,\n                                count_c_dict, count_u_c_dict,\n                                answered_correctly_u_count, answered_correctly_u_sum, \n                                elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, \n                                answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, \n                                answered_correctly_uq, update = False)\n    \n    test_df[TARGET] =  model.predict(test_df[FEATS])   \n    \"\"\"\n    \n    test_df[TARGET] = outs\n    \n    set_predict(test_df[['row_id', TARGET]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test_df[TARGET]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Have a fun with loops! :)"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}