{"cells":[{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.execute_input":"2021-01-04T06:11:54.00696Z","iopub.status.busy":"2021-01-04T06:11:54.006119Z","iopub.status.idle":"2021-01-04T06:11:56.533367Z","shell.execute_reply":"2021-01-04T06:11:56.532706Z"},"papermill":{"duration":2.54482,"end_time":"2021-01-04T06:11:56.53349","exception":false,"start_time":"2021-01-04T06:11:53.98867","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import numpy as np\nimport random\nimport pandas as pd\n\nimport json\nimport joblib\nimport pickle\n\nfrom collections import defaultdict\nimport lightgbm as lgb\nfrom catboost import CatBoostClassifier\nimport riiideducation\nfrom sklearn.metrics import roc_auc_score\nimport gc\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\n\n_ = np.seterr(divide='ignore', invalid='ignore')","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-04T06:11:56.580911Z","iopub.status.busy":"2021-01-04T06:11:56.572599Z","iopub.status.idle":"2021-01-04T06:12:18.867046Z","shell.execute_reply":"2021-01-04T06:12:18.866255Z"},"papermill":{"duration":22.322649,"end_time":"2021-01-04T06:12:18.867186","exception":false,"start_time":"2021-01-04T06:11:56.544537","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"with open('../input/file-collected-for-inference/baggingfeat/attempt_no_sum_dict.pickle', 'rb') as handle:\n    attempt_no_sum_dict = pickle.load(handle)\n\nwith open('../input/file-collected-for-inference/baggingfeat/user_prior_question_elapsed_time_dict.pickle', 'rb') as handle:\n    user_prior_question_elapsed_time_dict = pickle.load(handle)\n\nwith open('../input/file-collected-for-inference/baggingfeat/max_timestamp_u_dict3.pickle', 'rb') as handle:\n    max_timestamp_u_dict3 = pickle.load(handle)\n\nwith open('../input/file-collected-for-inference/baggingfeat/max_timestamp_u_dict2.pickle', 'rb') as handle:\n    max_timestamp_u_dict2 = pickle.load(handle)\n\nwith open('../input/file-collected-for-inference/baggingfeat/max_timestamp_u_dict.pickle', 'rb') as handle:\n    max_timestamp_u_dict = pickle.load(handle)\n    \nwith open('../input/file-collected-for-inference/baggingfeat/user_lecture_count_dict.pickle', 'rb') as handle:\n    user_lecture_count_dict = pickle.load(handle)\n    \nwith open('../input/file-collected-for-inference/baggingfeat/user_lecture_sum_dict.pickle', 'rb') as handle:\n    user_lecture_sum_dict = pickle.load(handle)\n    \nwith open('../input/file-collected-for-inference/baggingfeat/explanation_count_dict.pickle', 'rb') as handle:\n    explanation_count_dict = pickle.load(handle)\n    \nwith open('../input/file-collected-for-inference/baggingfeat/explanation_sum_dict.pickle', 'rb') as handle:\n    explanation_sum_dict = pickle.load(handle)\n    \nwith open('../input/file-collected-for-inference/baggingfeat/task_container_std_dict.pickle', 'rb') as handle:\n    task_container_std_dict = pickle.load(handle)\n\nwith open('../input/file-collected-for-inference/baggingfeat/task_container_count_dict.pickle', 'rb') as handle:\n    task_container_count_dict = pickle.load(handle)\n\nwith open('../input/file-collected-for-inference/baggingfeat/task_container_sum_dict.pickle', 'rb') as handle:\n    task_container_sum_dict = pickle.load(handle)\n    \nwith open('../input/file-collected-for-inference/baggingfeat/user_sum_dict.pickle', 'rb') as handle:\n    user_sum_dict = pickle.load(handle)\n    \nwith open('../input/file-collected-for-inference/baggingfeat/user_count_dict.pickle', 'rb') as handle:\n    user_count_dict = pickle.load(handle)\n\nlgb_model = joblib.load('../input/file-collected-for-inference/lgbmmodel/lgb.pkl')\n\nprior_question_elapsed_time_mean = 25423.810042960275\nlagtime_mean2,lagtime_mean3,lagtime_mean = (39610585.65514838, 58879590.7244385, 20061832.085456338)\ndelta_prior_question_elapsed_time_mean = -4.699396307374765\n\nquestions_df = pd.read_csv('../input/file-collected-for-inference/baggingfeat/questions_df.csv')\n\nN=[0.4,0.6]\n\nfeatures_dict = {\n    'timestamp':'float16',#\n    'user_interaction_count':'int16',\n    'user_interaction_timestamp_mean':'float32',\n    'lagtime':'float32',#\n    'lagtime2':'float32',\n    'lagtime3':'float32',\n    'content_id':'int16',\n    'task_container_id':'int16',\n    'user_lecture_sum':'int16',#\n    'user_lecture_lv':'float16',##\n    'prior_question_elapsed_time':'float32',#\n    'delta_prior_question_elapsed_time':'int32',#\n    'user_correctness':'float16',#\n    'user_uncorrect_count':'int16',#\n    'user_correct_count':'int16',#\n    'content_correctness_std':'float16',\n    'content_correct_count':'int32',\n    'content_uncorrect_count':'int32',#\n    'content_elapsed_time_mean':'float16',\n    'content_had_explanation_mean':'float16',\n    'content_explation_false_mean':'float16',\n    'content_explation_true_mean':'float16',\n    'task_container_correctness':'float16',\n    'task_container_std':'float16',\n    'task_container_cor_count':'int32',#\n    'task_container_uncor_count':'int32',#\n    'attempt_no':'int8',#\n    'part':'int8',\n    'part_correctness_mean':'float16',\n    'part_correctness_std':'float16',\n    'part_uncor_count':'int32',\n    'part_cor_count':'int32',\n    'tags0': 'int8',\n    'tags1': 'int8',\n    'tags2': 'int8',\n    'tags3': 'int8',\n    'tags4': 'int8',\n    'tags5': 'int8',\n    'part_bundle_id':'int32',\n    'content_sub_bundle':'int8',\n    'prior_question_had_explanation':'int8',\n    'explanation_mean':'float16', #\n    'explanation_false_count':'int16',#\n    'explanation_true_count':'int16',#\n}\nfeatures=list(features_dict.keys())\ntarget = 'answered_correctly'\n\ndef get_max_attempt(user_id,content_id):\n    k = (user_id,content_id)\n\n    if k in attempt_no_sum_dict.keys():\n        attempt_no_sum_dict[k]+=1\n        return attempt_no_sum_dict[k]\n\n    attempt_no_sum_dict[k] = 1\n    return attempt_no_sum_dict[k]","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-04T06:12:18.898437Z","iopub.status.busy":"2021-01-04T06:12:18.89754Z","iopub.status.idle":"2021-01-04T06:12:37.542138Z","shell.execute_reply":"2021-01-04T06:12:37.541361Z"},"papermill":{"duration":18.666582,"end_time":"2021-01-04T06:12:37.542272","exception":false,"start_time":"2021-01-04T06:12:18.87569","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# extra_features = [  'part_elapsed_time_mean',\n#                     'part_had_explanation_mean',\n#                     'part_correctly_q_mean',\n#                     'question_elapsed_time_mean',\n#                     'question_had_explanation_mean',\n#                     'question_correctly_q_count',\n#                     'question_correctly_q_mean',\n#                     'tags_lsi',\n#                     'tag_acc_count',\n#                     'tag_acc_max',\n#                     'tag_acc_min'\n#                  ]\n\n# part_fea = pd.read_csv('../input/file-collected-for-inference/extrafeat/part_fea_merge.csv')\n# que_fea = pd.read_csv('../input/file-collected-for-inference/extrafeat/que_fea_merge.csv')\n# que_fea.drop(['part'],inplace = True,axis = 1)\n# que_fea.set_index('content_id',inplace = True)\n# part_fea.set_index('part',inplace = True)\n\n# extra_model = joblib.load(open('../input/file-collected-for-inference/extralgbm/lgb.pkl','rb'))","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-04T06:12:37.630315Z","iopub.status.busy":"2021-01-04T06:12:37.579748Z","iopub.status.idle":"2021-01-04T06:12:56.820671Z","shell.execute_reply":"2021-01-04T06:12:56.821553Z"},"papermill":{"duration":19.270792,"end_time":"2021-01-04T06:12:56.821721","exception":false,"start_time":"2021-01-04T06:12:37.550929","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"MAX_SEQ = 240\nDROPOUT=0.1\n\nclass FFN(nn.Module):\n    def __init__(self, state_size = 200, forward_expansion = 1, bn_size = MAX_SEQ - 1, dropout=0.2):\n        super(FFN, self).__init__()\n        self.state_size = state_size\n        \n        self.lr1 = nn.Linear(state_size, forward_expansion * state_size)\n        self.relu = nn.ReLU()\n        self.bn = nn.BatchNorm1d(bn_size)\n        self.lr2 = nn.Linear(forward_expansion * state_size, state_size)\n        self.dropout = nn.Dropout(dropout)\n        \n    def forward(self, x):\n        x = self.relu(self.lr1(x))\n        x = self.bn(x)\n        x = self.lr2(x)\n        return self.dropout(x)\n    \nclass FFN0(nn.Module):\n    def __init__(self, state_size = 200, forward_expansion = 1, bn_size = MAX_SEQ - 1, dropout=0.2):\n        super(FFN0, self).__init__()\n        self.state_size = state_size\n\n        self.lr1 = nn.Linear(state_size, forward_expansion * state_size)\n        self.relu = nn.ReLU()\n        self.lr2 = nn.Linear(forward_expansion * state_size, state_size)\n        self.layer_normal = nn.LayerNorm(state_size) \n        self.dropout = nn.Dropout(0.2)\n    \n    def forward(self, x):\n        x = self.lr1(x)\n        x = self.relu(x)\n        x = self.lr2(x)\n        x=self.layer_normal(x)\n        return self.dropout(x)\n    \ndef future_mask(seq_length):\n    future_mask = np.triu(np.ones((seq_length, seq_length)), k=1).astype('bool')\n    return torch.from_numpy(future_mask)\n\n\nclass TransformerBlock(nn.Module):\n    def __init__(self, embed_dim, heads = 8, dropout = DROPOUT, forward_expansion = 1):\n        super(TransformerBlock, self).__init__()\n        self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=heads, dropout=dropout)\n        self.dropout = nn.Dropout(dropout)\n        self.layer_normal = nn.LayerNorm(embed_dim)\n        self.ffn = FFN(embed_dim, forward_expansion = forward_expansion, dropout=dropout)\n        self.ffn0  = FFN0(embed_dim, forward_expansion = forward_expansion, dropout=dropout)\n        self.layer_normal_2 = nn.LayerNorm(embed_dim)\n\n    def forward(self, value, key, query, att_mask):\n        att_output, att_weight = self.multi_att(value, key, query, attn_mask=att_mask)\n        att_output = self.dropout(self.layer_normal(att_output + value))\n        att_output = att_output.permute(1, 0, 2) # att_output: [s_len, bs, embed] => [bs, s_len, embed]\n        x = self.ffn(att_output)\n        x1 = self.ffn0(att_output)\n        x = self.dropout(self.layer_normal_2(x + x1 + att_output))\n        return x.squeeze(-1), att_weight\n    \nclass Encoder(nn.Module):\n    def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, num_layers=1, heads = 8):\n        super(Encoder, self).__init__()\n        self.n_skill, self.embed_dim = n_skill, embed_dim\n        self.embedding = nn.Embedding(2 * n_skill + 1, embed_dim)\n        self.pos_embedding = nn.Embedding(max_seq - 1, embed_dim)\n        self.e_embedding = nn.Embedding(n_skill+1, embed_dim)\n        self.layers = nn.ModuleList([TransformerBlock(embed_dim, forward_expansion = forward_expansion) for _ in range(num_layers)])\n        self.dropout = nn.Dropout(dropout)\n        \n    def forward(self, x, question_ids):\n        device = x.device\n        x = self.embedding(x)\n        pos_id = torch.arange(x.size(1)).unsqueeze(0).to(device)\n        pos_x = self.pos_embedding(pos_id)\n        x = self.dropout(x + pos_x)\n        x = x.permute(1, 0, 2) # x: [bs, s_len, embed] => [s_len, bs, embed]\n        e = self.e_embedding(question_ids)\n        e = e.permute(1, 0, 2)\n        for layer in self.layers:\n            att_mask = future_mask(e.size(0)).to(device)\n            x, att_weight = layer(e, x, x, att_mask=att_mask)\n            x = x.permute(1, 0, 2)\n        x = x.permute(1, 0, 2)\n        return x, att_weight\n\nclass SAKTModel(nn.Module):\n    def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, enc_layers=1, heads = 8):\n        super(SAKTModel, self).__init__()\n        self.encoder = Encoder(n_skill, max_seq, embed_dim, dropout, forward_expansion, num_layers=enc_layers)\n        self.pred = nn.Linear(embed_dim, 1)\n        \n    def forward(self, x, question_ids):\n        x, att_weight = self.encoder(x, question_ids)\n        x = self.pred(x)\n        return x.squeeze(-1), att_weight\n    \n\nskills = joblib.load(\"../input/file-collected-for-inference/sakt100/skills.pkl.zip\")\nn_skill = len(skills)\ngroup = joblib.load(\"../input/file-collected-for-inference/sakt100/group.pkl.zip\")\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nsakt_model = SAKTModel(n_skill, max_seq=240, embed_dim=256, forward_expansion=1, enc_layers=1, heads=4, dropout=0.1)\nsakt_model.load_state_dict(torch.load(\"../input/v4-fork-of-riiid-sakt-model-full/sakt_model.pt\", map_location='cpu'))\nsakt_model.to(device)\nsakt_model.eval()\n\nclass TestDataset(Dataset):\n    def __init__(self, samples, test_df, skills, max_seq=MAX_SEQ): \n        super(TestDataset, self).__init__()\n        self.samples = samples\n        self.user_ids = [x for x in test_df[\"user_id\"].unique()]\n        self.test_df = test_df\n        self.skills = skills\n        self.n_skill = len(skills)\n        self.max_seq = max_seq\n\n    def __len__(self):\n        return self.test_df.shape[0]\n\n    def __getitem__(self, index):\n        test_info = self.test_df.iloc[index]\n\n        user_id = test_info[\"user_id\"]\n        target_id = test_info[\"content_id\"]\n\n        q = np.zeros(self.max_seq, dtype=int)\n        qa = np.zeros(self.max_seq, dtype=int)\n\n        if user_id in self.samples.index:\n            q_, qa_ = self.samples[user_id]\n            \n            seq_len = len(q_)\n\n            if seq_len >= self.max_seq:\n                q = q_[-self.max_seq:]\n                qa = qa_[-self.max_seq:]\n            else:\n                q[-seq_len:] = q_\n                qa[-seq_len:] = qa_          \n        \n        x = np.zeros(self.max_seq-1, dtype=int)\n        x = q[1:].copy()\n        x += (qa[1:] == 1) * self.n_skill\n        \n        questions = np.append(q[2:], [target_id])\n        \n        return x, questions","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-04T06:12:56.842516Z","iopub.status.busy":"2021-01-04T06:12:56.841818Z","iopub.status.idle":"2021-01-04T06:12:56.984282Z","shell.execute_reply":"2021-01-04T06:12:56.984834Z"},"papermill":{"duration":0.154609,"end_time":"2021-01-04T06:12:56.984979","exception":false,"start_time":"2021-01-04T06:12:56.83037","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"cat_model = CatBoostClassifier()\ncat_model.load_model('../input/file-collected-for-inference/catmodel/catboost_model')","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-04T06:12:57.00705Z","iopub.status.busy":"2021-01-04T06:12:57.00634Z","iopub.status.idle":"2021-01-04T06:12:57.416849Z","shell.execute_reply":"2021-01-04T06:12:57.416291Z"},"papermill":{"duration":0.422784,"end_time":"2021-01-04T06:12:57.41697","exception":false,"start_time":"2021-01-04T06:12:56.994186","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"DROPOUT=0.1\nMAX_SEQ2 = 170\nACCEPTED_USER_CONTENT_SIZE = 5\nEMBED_SIZE = 128\nclass FFN2(nn.Module):\n    def __init__(self, state_size = 200, forward_expansion = 1, bn_size=MAX_SEQ2 - 1, dropout=0.2):\n        super(FFN2, self).__init__()\n        self.state_size = state_size\n        \n        self.lr1 = nn.Linear(state_size, forward_expansion * state_size)\n        self.relu = nn.ReLU()\n        self.bn = nn.BatchNorm1d(bn_size)\n        self.lr2 = nn.Linear(forward_expansion * state_size, state_size)\n        self.dropout = nn.Dropout(dropout)\n        \n    def forward(self, x):\n        x = self.relu(self.lr1(x))\n        x = self.bn(x)\n        x = self.lr2(x)\n        return self.dropout(x)\n\n\nclass TransformerBlock(nn.Module):\n    def __init__(self, embed_dim, heads = 8, dropout = DROPOUT, forward_expansion = 1):\n        super(TransformerBlock, self).__init__()\n        self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=heads, dropout=dropout)\n        self.dropout = nn.Dropout(dropout)\n        self.layer_normal = nn.LayerNorm(embed_dim)\n        self.ffn = FFN2(embed_dim, forward_expansion = forward_expansion, dropout=dropout)\n        self.layer_normal_2 = nn.LayerNorm(embed_dim)\n        \n\n    def forward(self, value, key, query, att_mask):\n        att_output, att_weight = self.multi_att(value, key, query, attn_mask=att_mask)\n        att_output = self.dropout(self.layer_normal(att_output + value))\n        att_output = att_output.permute(1, 0, 2) # att_output: [s_len, bs, embed] => [bs, s_len, embed]\n        x = self.ffn(att_output)\n        x = self.dropout(self.layer_normal_2(x + att_output))\n        return x.squeeze(-1), att_weight\n    \nclass Encoder(nn.Module):\n    def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, num_layers=1, heads = 8):\n        super(Encoder, self).__init__()\n        self.n_skill, self.embed_dim = n_skill, embed_dim\n        self.embedding = nn.Embedding(2 * n_skill + 1, embed_dim)\n        self.pos_embedding = nn.Embedding(max_seq - 1, embed_dim)\n        self.e_embedding = nn.Embedding(n_skill+1, embed_dim)\n        self.layers = nn.ModuleList([TransformerBlock(embed_dim, forward_expansion = forward_expansion) for _ in range(num_layers)])\n        self.dropout = nn.Dropout(dropout)\n        \n    def forward(self, x, question_ids):\n        device = x.device\n        x = self.embedding(x)\n        pos_id = torch.arange(x.size(1)).unsqueeze(0).to(device)\n        pos_x = self.pos_embedding(pos_id)\n        x = self.dropout(x + pos_x)\n        x = x.permute(1, 0, 2) # x: [bs, s_len, embed] => [s_len, bs, embed]\n        e = self.e_embedding(question_ids)\n        e = e.permute(1, 0, 2)\n        for layer in self.layers:\n            att_mask = future_mask(e.size(0)).to(device)\n            x, att_weight = layer(e, x, x, att_mask=att_mask)\n            x = x.permute(1, 0, 2)\n        x = x.permute(1, 0, 2)\n        return x, att_weight\n\nclass SAKTModel2(nn.Module):\n    def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, enc_layers=1, heads = 8):\n        super(SAKTModel2, self).__init__()\n        self.encoder = Encoder(n_skill, max_seq, embed_dim, dropout, forward_expansion, num_layers=enc_layers)\n        self.pred = nn.Linear(embed_dim, 1)\n        \n    def forward(self, x, question_ids):\n        x, att_weight = self.encoder(x, question_ids)\n        x = self.pred(x)\n        return x.squeeze(-1), att_weight\n\ndef create_model():\n    return SAKTModel2(n_skill, max_seq=MAX_SEQ2, embed_dim=EMBED_SIZE, forward_expansion=3, enc_layers=1, heads=8, dropout=0.1)\nsakt_model2 = create_model()\nsakt_model2.load_state_dict(torch.load('../input/file-collected-for-inference/sakt170/sakt.pth',map_location = 'cpu'))\nsakt_model2.eval()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-04T06:12:57.44464Z","iopub.status.busy":"2021-01-04T06:12:57.442573Z","iopub.status.idle":"2021-01-04T06:12:57.445483Z","shell.execute_reply":"2021-01-04T06:12:57.446017Z"},"papermill":{"duration":0.01918,"end_time":"2021-01-04T06:12:57.446164","exception":false,"start_time":"2021-01-04T06:12:57.426984","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()\niter_test = env.iter_test()\nprior_test_df = None","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-04T06:12:57.537188Z","iopub.status.busy":"2021-01-04T06:12:57.495999Z","iopub.status.idle":"2021-01-04T06:12:59.016934Z","shell.execute_reply":"2021-01-04T06:12:59.015948Z"},"papermill":{"duration":1.561192,"end_time":"2021-01-04T06:12:59.017059","exception":false,"start_time":"2021-01-04T06:12:57.455867","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:    \n    if prior_test_df is not None:\n        prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])\n\n\n        prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)\n        prior_test_df['prior_question_had_explanation'].fillna(False, inplace=True)\n        prior_test_df.prior_question_had_explanation=prior_test_df.prior_question_had_explanation.astype('int8')\n        user_ids = prior_test_df['user_id'].values\n        targets = prior_test_df[target].values        \n        for user_id, answered_correctly in zip(user_ids,targets):\n            user_sum_dict[user_id] += answered_correctly\n            user_count_dict[user_id] += 1\n            \n        prev_group = prior_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id').apply(lambda r: (\n            r['content_id'].values,\n            r['answered_correctly'].values))\n        for prev_user_id in prev_group.index:\n            if prev_user_id in group.index:\n                group[prev_user_id] = (\n                    np.append(group[prev_user_id][0], prev_group[prev_user_id][0])[-MAX_SEQ:], \n                    np.append(group[prev_user_id][1], prev_group[prev_user_id][1])[-MAX_SEQ:]\n                )\n \n            else:\n                group[prev_user_id] = (\n                    prev_group[prev_user_id][0], \n                    prev_group[prev_user_id][1]\n                )\n        \n    prior_test_df = test_df.copy() \n    test_df = test_df[test_df.content_type_id == False]\n    test_dataset = TestDataset(group, test_df, skills)\n    test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)\n    sakt_preds = []\n    for item in test_dataloader:\n        x = item[0].to(device).long()\n        target_id = item[1].to(device).long()\n\n        with torch.no_grad():\n            output, att_weight = sakt_model(x, target_id)\n        sakt_preds.extend(torch.sigmoid(output)[:, -1].view(-1).data.cpu().numpy())\n        \n    sakt_preds2 = []\n    test_dataset = TestDataset(group, test_df, skills,max_seq = MAX_SEQ2)\n    test_dataloader = DataLoader(test_dataset, batch_size= len(test_df), shuffle=False)\n    for item in test_dataloader:\n        x = item[0].to(device).long()\n        target_id = item[1].to(device).long()\n\n        with torch.no_grad():\n            output, _ = sakt_model2(x, target_id)\n        sakt_preds2.extend(torch.sigmoid(output)[:, -1].view(-1).data.cpu().numpy())\n                    \n    question_len=len( test_df[test_df['content_type_id'] == 0])\n    test_df['prior_question_had_explanation'].fillna(False, inplace=True)\n    test_df.prior_question_had_explanation=test_df.prior_question_had_explanation.astype('int8')\n    test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)\n    user_lecture_sum = np.zeros(question_len, dtype=np.int16)\n    user_lecture_count = np.zeros(question_len, dtype=np.int16) \n    user_sum = np.zeros(question_len, dtype=np.int16)\n    user_count = np.zeros(question_len, dtype=np.int16)\n    task_container_sum = np.zeros(question_len, dtype=np.int32)\n    task_container_count = np.zeros(question_len, dtype=np.int32)\n    task_container_std = np.zeros(question_len, dtype=np.float16)\n    explanation_sum = np.zeros(question_len, dtype=np.int32)\n    explanation_count = np.zeros(question_len, dtype=np.int32)\n    delta_prior_question_elapsed_time = np.zeros(question_len, dtype=np.int32)\n    attempt_no_count = np.zeros(question_len, dtype=np.int16)\n    lagtime = np.zeros(question_len, dtype=np.float32)\n    lagtime2 = np.zeros(question_len, dtype=np.float32)\n    lagtime3 = np.zeros(question_len, dtype=np.float32)\n   \n    i=0\n    for j, (user_id,prior_question_had_explanation,content_type_id,prior_question_elapsed_time,timestamp, content_id,task_container_id) in enumerate(zip(test_df['user_id'].values,test_df['prior_question_had_explanation'].values,test_df['content_type_id'].values,test_df['prior_question_elapsed_time'].values,test_df['timestamp'].values, test_df['content_id'].values, test_df['task_container_id'].values)):\n        user_lecture_sum_dict[user_id] += content_type_id\n        user_lecture_count_dict[user_id] += 1\n        if(content_type_id==1):#\n            x=1\n        if(content_type_id==0):#   \n            user_lecture_sum[i] = user_lecture_sum_dict[user_id]\n            user_lecture_count[i] = user_lecture_count_dict[user_id]\n            user_sum[i] = user_sum_dict[user_id]\n            user_count[i] = user_count_dict[user_id]\n            task_container_sum[i] = task_container_sum_dict[task_container_id]\n            task_container_count[i] = task_container_count_dict[task_container_id]\n            task_container_std[i]=task_container_std_dict[task_container_id]\n            explanation_sum_dict[user_id] += prior_question_had_explanation\n            explanation_count_dict[user_id] += 1\n            explanation_sum[i] = explanation_sum_dict[user_id]\n            explanation_count[i] = explanation_count_dict[user_id]\n            if user_id in max_timestamp_u_dict['max_time_stamp'].keys():\n                lagtime[i]=timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]\n                if(max_timestamp_u_dict2['max_time_stamp2'][user_id]==lagtime_mean2):#\n                    lagtime2[i]=lagtime_mean2\n                    lagtime3[i]=lagtime_mean3\n                else:\n                    lagtime2[i]=timestamp-max_timestamp_u_dict2['max_time_stamp2'][user_id]\n                    if(max_timestamp_u_dict3['max_time_stamp3'][user_id]==lagtime_mean3):\n                        lagtime3[i]=lagtime_mean3 \n                    else:\n                        lagtime3[i]=timestamp-max_timestamp_u_dict3['max_time_stamp3'][user_id]\n                    \n                    max_timestamp_u_dict3['max_time_stamp3'][user_id]=max_timestamp_u_dict2['max_time_stamp2'][user_id]\n                        \n                max_timestamp_u_dict2['max_time_stamp2'][user_id]=max_timestamp_u_dict['max_time_stamp'][user_id]\n                max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp\n\n            else:\n                lagtime[i]=lagtime_mean\n                max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})\n                lagtime2[i]=lagtime_mean2#\n                max_timestamp_u_dict2['max_time_stamp2'].update({user_id:lagtime_mean2})\n                lagtime3[i]=lagtime_mean3#\n                max_timestamp_u_dict3['max_time_stamp3'].update({user_id:lagtime_mean3})\n\n\n            if user_id in user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].keys():            \n                delta_prior_question_elapsed_time[i]=prior_question_elapsed_time-user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]\n                user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]=prior_question_elapsed_time\n            else:           \n                delta_prior_question_elapsed_time[i]=delta_prior_question_elapsed_time_mean    \n                user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].update({user_id:prior_question_elapsed_time})\n            i=i+1 \n\n\n        \n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n   \n    test_df=test_df.merge(questions_df.loc[questions_df.index.isin(test_df['content_id'])],\n                  how='left', on='content_id', right_index=True)\n\n \n    test_df['user_lecture_lv'] = user_lecture_sum / user_lecture_count\n    test_df['user_lecture_sum'] = user_lecture_sum\n    \n    test_df['user_interaction_count'] = user_lecture_count\n    test_df['user_interaction_timestamp_mean'] = test_df['timestamp']/user_lecture_count\n    \n    test_df['user_correctness'] = user_sum / user_count\n    test_df['user_uncorrect_count'] =user_count-user_sum\n    test_df['user_correct_count'] =user_sum\n\n    test_df['task_container_correctness'] = task_container_sum / task_container_count\n    test_df['task_container_cor_count'] = task_container_sum \n    test_df['task_container_uncor_count'] =task_container_count-task_container_sum \n    test_df['task_container_std'] = task_container_std \n    \n    test_df['explanation_mean'] = explanation_sum / explanation_count\n    test_df['explanation_true_count'] = explanation_sum\n    test_df['explanation_false_count'] = explanation_count-explanation_sum \n    \n    test_df['delta_prior_question_elapsed_time'] = delta_prior_question_elapsed_time \n    \n  \n \n    test_df[\"attempt_no\"] = test_df[[\"user_id\", \"content_id\"]].apply(lambda row: get_max_attempt(row[\"user_id\"], row[\"content_id\"]), axis=1)\n    test_df[\"lagtime\"]=lagtime\n    test_df[\"lagtime2\"]=lagtime2\n    test_df[\"lagtime3\"]=lagtime3\n    \n\n    test_df['timestamp']=test_df['timestamp']/(1000*3600)\n    test_df.timestamp=test_df.timestamp.astype('float16')\n    test_df['lagtime']=test_df['lagtime']/(1000*3600)\n    test_df.lagtime=test_df.lagtime.astype('float32')\n    test_df['lagtime2']=test_df['lagtime2']/(1000*3600)\n    test_df.lagtime2=test_df.lagtime2.astype('float32')\n    test_df['lagtime3']=test_df['lagtime3']/(1000*3600)\n    test_df.lagtime3=test_df.lagtime3.astype('float32')\n    test_df['user_interaction_timestamp_mean']=test_df['user_interaction_timestamp_mean']/(1000*3600)\n    test_df.user_interaction_timestamp_mean=test_df.user_interaction_timestamp_mean.astype('float32')\n    \n    test_df['user_correctness'].fillna(0.67, inplace=True)\n    \n    lgb_preds  = lgb_model.predict(test_df[features])\n    cat_preds = cat_model.predict_proba(test_df[features])[:,1]\n    \n#     test_df = test_df.join(part_fea, on='part')\n#     test_df = test_df.join(que_fea, on='content_id')\n    \n#     extra_preds  = extra_model.predict(test_df[features + extra_features])\n    test_df[target] = 0.20*(lgb_preds) + 0.30*cat_preds+ 0.22*np.array(sakt_preds2) + 0.28*np.array(sakt_preds)\n                          \n    env.predict(test_df[['row_id', target]])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}