{"cells":[{"metadata":{},"cell_type":"markdown","source":"# **This is a simple Ensemble of LGBM and SAINT+ like model 0.804 public / 0.806 private**\n\n<font size=\"3\">Since this is my first competition, I was clueless on the ensembling techniques that I might use to maximize the improvemement of the model, so I've done a simple SAINT *1.15 + LGBM*0.85 ensemble. Tried stacking, it didn't work well.</font>\n\n\n# **TL;DR** \n\n<font size=\"3\">**SAINT**: 128 d_model, 2 att_heads, 100 seq_len no padding, other than timestamp lag, part and user answer, no other feature improved my score. It score 0.803 without LGBM. I will post a notebook that trains and submits the model from scratch.\n\n**LGBM** : Basically whole lotta of features that are listed below, 54 feature, LGBM 0.792 , I trained 0.796 LGBM yesterday but I couldn't manage to impelement it before deadline.</font>\n\n------------------------\n\nMy biggest complain in this competition is the obscure submission errors, it really did limit my manoeuvrability, and the ideas I experimented with, however this competition was fun and I learn a lot throughout it. I will post notebooks and discussions explaining everything I have done. Big thanks to the hosts."},{"metadata":{},"cell_type":"markdown","source":"## This notebook showcases the architecture of my network, the data preparation for LGBM and submission."},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\nimport ast\n\nenv = riiideducation.make_env()\n\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.utils.rnn as rnn_utils\nfrom torch.autograd import Variable\n\nimport torch.nn.functional as F\nfrom torch.nn import TransformerEncoder, TransformerEncoderLayer\nfrom torch.nn import TransformerDecoder, TransformerDecoderLayer\nimport math\n\nimport pandas as pd\n\nfrom sys import getsizeof\n\n#supress warnings\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport time\nimport gc\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Setting up SAINT** "},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nq_pad = 13523 #Last value is not used\n\na_pad = 3\nstart_token = 2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group = pd.read_pickle(\"../input/groups/group.pandas\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features_1_path = '../input/get-features-1/'\nque_data = pd.read_pickle(features_1_path + \"que_data.pickle\")\n\ndifficulty = (np.round(que_data.que_correct_per, 1)*10).astype(\"int8\").values #Array of the difficulty of questions 0 -> 11\ndifficulty = torch.Tensor(difficulty).long().to(device)\n\nunique_tags = pd.concat([que_data.tags1,que_data.tags2, que_data.tags3, que_data.tags4,que_data.tags5,que_data.tags6]).unique()\ntags_n = len(unique_tags)\n\nunk_tag = tags_n-1  #Unknown tag token\nque_data = que_data.replace(-1, unk_tag)\n\npart_valus = torch.from_numpy(que_data.part.values).long().to(device)\nque_data = que_data.to_dict(\"index\")\n\n\nque_arr = np.zeros((np.array(list(que_data.keys())).shape[0], 6))\n\nfor i in que_data:\n    a = que_data[i]\n    que_arr[i] = [a['tags1'],a['tags2'],a['tags3'],a['tags4'],a['tags5'],a['tags6']]\n    \n    \nimport pickle \n\nwith open('../input/hashtable/user_info', 'rb') as handle:\n    user_info = pickle.load(handle)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"st_user_info = {}\n\nfor i in user_info:\n    st_user_info[i] = {\"timestamp_ms\":user_info[i][\"first_timestamp\"]}\n    \ndel user_info","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PositionalEncoding(nn.Module):\n    def __init__(self, d_model, dropout=0.1, max_len=5000):\n        super(PositionalEncoding, self).__init__()\n        self.dropout = nn.Dropout(p=dropout)\n        self.scale = nn.Parameter(torch.ones(1))\n\n        pe = torch.zeros(max_len, d_model)\n        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n        div_term = torch.exp(torch.arange(\n            0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n        pe[:, 0::2] = torch.sin(position * div_term)\n        pe[:, 1::2] = torch.cos(position * div_term)\n        pe = pe.unsqueeze(0).transpose(0, 1)\n        self.register_buffer('pe', pe)\n\n    def forward(self, x):\n        x = x + self.scale * self.pe[:x.size(0), :]\n        return self.dropout(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class EmbedTag(nn.Module):\n    \n    def __init__(self, d_model, que_arr, tags_n):\n        super(EmbedTag, self).__init__()\n        self.que_arr = torch.LongTensor(que_arr).to(device)\n        \n        self.embedding = nn.Embedding(tags_n, d_model)\n        \n    def forward(self, x): #(seq_len, batch)\n        \n        x = self.que_arr[x, :]  #(seq_len, batch, 6)\n        x = self.embedding(x)  #(seq_len, batch, 6, hidden)\n        \n        return torch.sum(x, dim=-2)  #(seq_len, batch, hidden)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TransformerModel(nn.Module):\n    \n    def __init__(self, intoken, outtoken, hidden, que_arr, part_arr, difficulty, enc_layers=4, dec_layers=4, dropout=0.1, ts_unique=70, prior_unique=50):\n        super(TransformerModel, self).__init__()\n        nhead = hidden//64\n        \n        self.encoder = nn.Embedding(intoken, hidden)\n        self.pos_encoder = PositionalEncoding(hidden, dropout)\n\n        self.decoder = nn.Embedding(outtoken, hidden)\n        self.pos_decoder = PositionalEncoding(hidden, dropout)\n        \n        \n        self.tagsEmbedder = EmbedTag(hidden, que_arr, tags_n)\n\n        self.transformer = nn.Transformer(d_model=hidden, nhead=nhead, num_encoder_layers=enc_layers, num_decoder_layers=dec_layers, dim_feedforward=hidden*4, dropout=dropout, activation='relu')\n        self.fc_out = nn.Linear(hidden, 1)\n\n        self.src_mask = None\n        self.trg_mask = None\n        self.memory_mask = None\n      \n        self.part_embedding = nn.Embedding(7,hidden)\n        self.part_arr = part_arr\n        \n        self.ts_embedding = nn.Embedding(ts_unique, hidden)\n        self.prior_embedding = nn.Embedding(prior_unique, hidden)\n        \n        self.task_container_embedding = nn.Embedding(10000, hidden)\n        self.user_answer_embedding = nn.Embedding(5, hidden)\n        \n        self.difficulty = difficulty\n        self.difficulty_embedding = nn.Embedding(11, hidden)\n        self.dropout_7 = nn.Dropout(dropout)\n        \n        self.dropout_1 = nn.Dropout(dropout)\n        self.dropout_2 = nn.Dropout(dropout)\n        self.dropout_3 = nn.Dropout(dropout)\n        self.dropout_4 = nn.Dropout(dropout)\n        self.dropout_5 = nn.Dropout(dropout)\n        self.dropout_6 = nn.Dropout(dropout)\n        \n        \n        self.explan_embedding = nn.Embedding(3, hidden)\n        self.dropout_9 = nn.Dropout(dropout)\n    \n    def generate_square_subsequent_mask(self, sz, sz1=None):\n        \n        if sz1 == None:\n            mask = torch.triu(torch.ones(sz, sz), 1)\n        else:\n            mask = torch.triu(torch.ones(sz, sz1), 1)\n            \n        return mask.masked_fill(mask==1, float('-inf'))\n\n    def make_len_mask_a(self, inp):\n        return (inp == a_pad).transpose(0, 1)   #(batch_size, output_seq_len)\n    \n    def make_len_mask_q(self, inp):\n        return (inp == q_pad).transpose(0, 1) #(batch_size, input_seq_len)\n    \n    \n\n    \n    def forward(self, src, trg, ts, prior, task_container, user_answer, explan):\n\n        if self.trg_mask is None or self.trg_mask.size(0) != len(trg):\n            self.trg_mask = self.generate_square_subsequent_mask(len(trg)).to(trg.device)\n            \n        if self.src_mask is None or self.src_mask.size(0) != len(src):\n            self.src_mask = self.generate_square_subsequent_mask(len(src)).to(trg.device)\n            \n        if self.memory_mask is None or self.memory_mask.size(0) != len(trg) or self.memory_mask.size(1) != len(src):\n            self.memory_mask = self.generate_square_subsequent_mask(len(trg),len(src)).to(trg.device)\n            \n\n        #Adding padding mask\n        src_pad_mask = self.make_len_mask_q(src)\n        trg_pad_mask = self.make_len_mask_a(trg)\n\n        #Get part, prior, timestamp, task_container and user answer embedding\n        part_emb = self.dropout_1(self.part_embedding(self.part_arr[src]-1))\n        ts_emb = self.dropout_3(self.ts_embedding(ts))\n        user_answer_emb = self.dropout_5(self.user_answer_embedding(user_answer))        \n        \n        \n        #Add embeddings Encoder\n        src = self.encoder(src)  #Embedding\n        src = torch.add(src, part_emb)\n        src = torch.add(src, ts_emb)   #Last interaction days \n        src = self.pos_encoder(src)   #Pos embedding\n        \n\n        #Add embedding decoder\n        trg = self.decoder(trg)\n        trg = torch.add(trg, user_answer_emb)\n        trg = self.pos_decoder(trg)\n\n        output = self.transformer(src, trg, src_mask=self.src_mask, tgt_mask=self.trg_mask, memory_mask=self.memory_mask,\n                                  src_key_padding_mask=src_pad_mask, tgt_key_padding_mask=trg_pad_mask, memory_key_padding_mask=src_pad_mask)\n        \n\n        output = self.fc_out(output)\n\n        return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"d_model = 128\n\nINPUT_DIM = q_pad+1\nOUTPUT_DIM = 4\n\nmodel_saint = TransformerModel(INPUT_DIM, OUTPUT_DIM, hidden=d_model, que_arr=que_arr,part_arr=part_valus, difficulty=difficulty).to(device)\nweights = torch.load(\"../input/last-saint/last.torch\", map_location=torch.device(device))\nmodel_saint.load_state_dict(weights)\n\nmodel_saint.to(device)\n\nmodel_saint.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def pred_users(vals): #Input must be (eval_batch, 3): [\"user_id\", \"content_id\", \"content_type_id\"]\n\n    eval_batch = vals.shape[0]\n\n    tensor_question = np.zeros((eval_batch, 100), dtype=np.long)\n    tensor_answers = np.zeros((eval_batch, 100), dtype=np.long)\n    tensor_ts = np.zeros((eval_batch, 100), dtype=np.long)\n    tensor_user_answer = np.zeros((eval_batch, 100), dtype=np.long)\n\n\n    val_len = []\n    preds = []\n    group_index = group.index\n\n    for i, line in enumerate(vals):\n\n        if line[2] == True:\n            val_len.append(0)\n            continue\n\n        user_id = line[0]\n        question_id = line[1]\n        timestamp = get_timestamp(line[4], user_id) #Compute timestamp difference correctly\n        prior = get_prior(line[5])\n        task_container_id = line[3]\n\n        que_history = np.array([], dtype=np.int32)\n        answers_history = np.array([], dtype=np.int32)  \n        ts_history = np.array([], dtype=np.int32)  \n        user_answer_history = np.array([], dtype=np.int32)  \n\n        if user_id in group_index:\n\n            cap = 99\n            que_history, answers_history, ts_history, user_answer_history = group[user_id]\n\n            que_history = que_history[-cap:]\n            answers_history = answers_history[-cap:]\n            ts_history = ts_history[-cap:]\n            user_answer_history = user_answer_history[-cap:]\n\n\n        a_token = 2\n        user_a_token = 4\n\n        #Decoder data, add start token\n        answers_history = np.concatenate(([a_token],answers_history))\n        user_answer_history = np.concatenate(([user_a_token],user_answer_history))\n\n        #Decoder data\n        que_history = np.concatenate((que_history, [question_id]))  #Add current question\n        ts_history = np.concatenate((ts_history, [timestamp]))  \n\n        tensor_question[i][:len(que_history)] = que_history\n        tensor_answers[i][:len(que_history)] = answers_history\n        tensor_ts[i][:len(que_history)] = ts_history\n        tensor_user_answer[i][:len(que_history)] = user_answer_history\n\n        val_len.append(len(que_history))\n\n    tensor_question = torch.from_numpy(tensor_question).long().T.to(device)\n    tensor_answers = torch.from_numpy(tensor_answers).long().T.to(device)\n    tensor_ts = torch.from_numpy(tensor_ts).long().T.to(device)\n    tensor_user_answer = torch.from_numpy(tensor_user_answer).long().T.to(device)\n\n\n    with torch.no_grad():  \n        out = F.sigmoid(model_saint(tensor_question, tensor_answers, tensor_ts, None, None, tensor_user_answer,2)).squeeze(dim=-1).T\n\n\n    for j in range(len(val_len)):\n        preds.append(out[j][val_len[j]-1].item())\n\n    return preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def split_preds(preds):\n    \n    if preds.shape[0] > 1000:\n        ret = []\n        for i in np.array_split(preds, preds.shape[0]//1000):\n            ret.extend(pred_users(i))\n        return ret\n    else:\n        return pred_users(preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def update_group_var(vals):\n    \n    global group\n    \n    for i, line in enumerate(vals):\n        \n        user_id = line[0]\n        question_id = line[1]\n        \n        content_type_id = line[2]\n        ts = get_timestamp(line[4], user_id)\n        \n        correct = line[6]\n        user_answer = line[7]\n        \n        \n        if content_type_id == True:\n            continue\n\n        if st_user_info.get(user_id, -1) == -1:\n            st_user_info[user_id] = {\"timestamp_ms\":0}\n        else:\n            st_user_info[user_id][\"timestamp_ms\"] = line[4] #Update user info\n            \n        if user_id in group.index:\n            questions= np.append(group[user_id][0],[question_id])\n            answers= np.append(group[user_id][1],[correct])\n            ts= np.append(group[user_id][2],[ts])\n            user_answer= np.append(group[user_id][3],[user_answer])\n            \n            group[user_id] = (questions, answers, ts, user_answer)\n        else:\n            group[user_id] = (np.array([question_id], dtype=np.int32), np.array([correct], dtype=np.int32), np.array([ts], dtype=np.int32)\n                             ,np.array([user_answer], dtype=np.int32))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ordinal_enc = {0: 0, 1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6, 7: 7, 8: 8, 9: 9, 10: 10, 11: 11, 12: 12, 13: 13, 14: 14, 15: 15, 16: 16, 17: 17, 18: 18, 19: 19, 20: 20, 30: 21, 40: 22, 50: 23, 60: 24, 70: 25, 80: 26, 90: 27, 100: 28, 110: 29, 120: 30, 130: 31, 140: 32, 150: 33, 160: 34, 170: 35, 180: 36, 190: 37, 200: 38, 210: 39, 220: 40, 230: 41, 240: 42, 250: 43, 260: 44, 270: 45, 280: 46, 290: 47, 300: 48}\nboundaries = [120,600,1800,3600,10800,43200,86400,259200,604800]\n\ndef get_prior(prior):\n    \n    if prior != prior:\n        return 0\n    \n    prior /= 1000\n    if prior > 20:\n        prior = np.round(prior, decimals=-1)\n    \n    return ordinal_enc.get(round(prior),0)\n\n\ndef get_timestamp(ts, user_id):\n    \n    if st_user_info.get(user_id, -1) == -1:\n        return 0\n    \n    diff = (ts - st_user_info[user_id][\"timestamp_ms\"])/1000\n    \n    if diff < 0:\n        return 0\n    \n    if diff <= 60:\n        return int(diff)\n    \n    for i, boundary in enumerate(boundaries):\n        if boundary > diff:\n            break\n            \n    if i == len(boundaries) - 1:\n        return 60+i+1\n    \n    return 60+i","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prior_part_mean_dict = {1: 22166.159642501425,\n 2: 18714.69673913695,\n 3: 23620.317746179924,\n 4: 23762.753651169547,\n 5: 25094.620302855932,\n 6: 32417.37918735745,\n 7: 47444.16407400242}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pickle\nimport ast\n\nwith open('../input/lgbm-test/repeated_que_count', 'rb') as handle:\n    repeated_que_count = pickle.load(handle)\n\nwith open('../input/lgbm-test/user_info', 'rb') as handle:\n    user_info = pickle.load(handle)\n\nwith open('../input/lgbm-test/watched_tags', 'rb') as handle:\n    watched_tags = pickle.load(handle)\n    \n    \nwith open('../input/lgbm-test/containers_mean', 'rb') as handle:\n    containers_mean = pickle.load(handle)\n    \nwith open('../input/lgbm-test/hardest', 'rb') as handle:\n    hard_questions = pickle.load(handle)\n    \nwith open('../input/lgbm-test/easiest', 'rb') as handle:\n    easy_questions = pickle.load(handle)\n    \nwith open('../input/lgbm-test/que_2_k', 'rb') as handle:\n    que_2_k = pickle.load(handle)\n\ngc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for u in user_info:\n    \n    user_info[u][\"count_2\"] = user_info[u][\"count\"]\n    user_info[u][\"part_count_2\"] = user_info[u][\"part_count\"].copy()\n    user_info[u][\"last_part\"] = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"groups = pd.read_pickle(\"../input/lgbm-test/groups\")\n\ndef numpy_ewma_vectorized_v2(data, window):\n\n    alpha = 2 /(window + 1.0)\n    alpha_rev = 1-alpha\n    n = data.shape[0]\n\n    pows = alpha_rev**(np.arange(n+1))\n\n    scale_arr = 1/pows[:-1]\n    offset = data[0]*pows[1:]\n    pw0 = alpha*alpha_rev**(n-1)\n\n    mult = data*pw0*scale_arr\n    cumsums = mult.cumsum()\n    out = offset + cumsums*scale_arr[::-1]\n    return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features_1_path = '../input/get-features-1/'\nque_data = pd.read_pickle(features_1_path + \"que_data.pickle\")\n\nquestions = que_data.drop(columns=[\"options_number\",\"correctness_number\", \"correct_answer\",\"tags6\",\"tags5\", \"tags4\"]).to_dict(\"index\")\nquestions1 = que_data[[\"tags1\", \"tags2\", \"tags3\",\"tags4\",\"tags5\", \"tags6\"]].to_dict(\"index\")\n\nparts = que_data.part.to_dict()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lec_data = pd.read_csv(\"../input/riiid-test-answer-prediction/lectures.csv\")\nlec_dict = lec_data[[\"lecture_id\", \"tag\"]].set_index(\"lecture_id\").tag.to_dict()\n\n#Add dtype for every feature\nfeatures = [\n    'task_container_id', \"ts_diff_shifted\", \"watched\",\"ts_diff_shifted_2\",\n    'content_id', \"k\", \"k_acc\", \"el_avg\", \"wut\",\n    'prior_question_elapsed_time', \"time_diff2\", \"rolling_mean_5\", \"rolling_mean_10\", \"rolling_mean_15\", \"prior_question_had_explanation_u_part_avg\",\n    'prior_question_had_explanation', \"hard_ratio_opp\", \"easy_ratio_opp\", \"correct_recency\", \"prior_question_elapsed_time_u_part_avg\", \"ewm_mean_10\", \"rolling_mean_5_prior_question\",\n    'last_lecture', \"part_mean\", \"opp_mean\", \"mean_pause\", \"timestamp\", \"prior_part_mean\",\n    \"container_mean\", \"lecs_per\", \"hard_ratio\", \"easy_ratio\",\n    'que_count_user', 'question_repeated', \"rolling_mean\",\"time_diff3\", \"time_diff4\",\n    'user_mean', \"time_diff1\", \"time_diff\", \"sessions\", \"session_count\", \"prior_question_had_explanation_ratio\"\n] + que_data.columns.tolist()[:-1]\n\nfeatures.remove(\"options_number\")\nfeatures.remove(\"correct_answer\")\nfeatures.remove(\"tags6\")\nfeatures.remove(\"tags5\")\nfeatures.remove(\"tags4\")\n\n\ntest_cols = ['row_id','timestamp','user_id','content_id','content_type_id','task_container_id','prior_question_elapsed_time',\n             'prior_question_had_explanation','prior_group_answers_correct','prior_group_responses']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stack_features = [\n    'task_container_id', \"ts_diff_shifted\", \"watched\",\"ts_diff_shifted_2\",\n    'content_id', \"k\", \"k_acc\", \"el_avg\", \"wut\", \"lgb_preds\", \"st_preds\",\n    'prior_question_elapsed_time', \"time_diff2\", \"rolling_mean_5\", \"rolling_mean_10\", \"rolling_mean_15\", \"prior_question_had_explanation_u_part_avg\",\n    'prior_question_had_explanation', \"hard_ratio_opp\", \"easy_ratio_opp\", \"correct_recency\", \"prior_question_elapsed_time_u_part_avg\", \"ewm_mean_10\", \"rolling_mean_5_prior_question\",\n    'last_lecture', \"part_mean\", \"opp_mean\", \"mean_pause\", \"timestamp\", \"prior_part_mean\",\n    \"container_mean\", \"lecs_per\", \"hard_ratio\", \"easy_ratio\",\n    'que_count_user', 'question_repeated', \"rolling_mean\",\"time_diff3\", \"time_diff4\",\n    'user_mean', \"time_diff1\", \"time_diff\", \"sessions\", \"session_count\", \"prior_question_had_explanation_ratio\"\n] + que_data.columns.tolist()[:-1]\n\nstack_features.remove(\"options_number\")\nstack_features.remove(\"correct_answer\")\nstack_features.remove(\"tags6\")\nstack_features.remove(\"tags5\")\nstack_features.remove(\"tags4\")\n\nlgb_preds_idx = stack_features.index(\"lgb_preds\")\nst_preds_idx = stack_features.index(\"st_preds\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"k_size = 20\n\ncols = {test_cols[k]:k for k in range(len(test_cols))}\nfeatures_dict = {features[k]:k for k in range(len(features))}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport ast\nimport copy\nfrom collections import Counter\n\nnew_user = {'count': 0, 'mean_acc':0.5, 'correct_count': 0, 'last_lec':0, 'tmp':0,\"first_timestamp\":0, \"second_timestamp\":0,\n            \"third_timestamp\":0, \"fourth_timestamp\":0, \"fifth_timestamp\":0, \"lecs_n\":0,\"interaction_n\":0, \"ts_diff_shifted\":0.,\n            \"part_corr\":np.zeros((7), dtype=np.uint16), \"part_count\":np.zeros((7), dtype=np.uint16), \"hard_ct\":0, \"hard_cr\":0, \"easy_ct\":0, \"easy_cr\":0,\n           \"sessions\":0, \"session_count\":0, \"sum_pauses\":0., \"had_exp\":0, \"el_sum\":0, \"part_et\": np.zeros((7), dtype=np.float), \n            \"part_explan\": np.zeros((7), dtype=np.uint16), \"k_count\": np.zeros((k_size), dtype=np.uint16), \"k_corr\": np.zeros((k_size), dtype=np.uint16), \n            \"recent_corr\":0, \"priors_5\": [], \"ts_diff_shifted_2\":0., \"count_2\":0, \"part_count_2\":np.zeros((7), dtype=np.uint16), \"last_part\":1}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_meta_data(data_1):\n    \n    user_id = data_1[cols['user_id']]\n    content_type_id = data_1[cols['content_type_id']]\n    content_id = data_1[cols['content_id']]\n    prior_group_answers_correct = data_1[cols['prior_group_answers_correct']]\n    timestamp = data_1[cols['timestamp']]\n    task_container_id = data_1[cols['task_container_id']]\n    prior_question_had_explanation = data_1[cols['prior_question_had_explanation']]\n    elapsed = data_1[cols['prior_question_elapsed_time']]\n    \n    return user_id, content_type_id, content_id, prior_group_answers_correct, timestamp,task_container_id,prior_question_had_explanation, elapsed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_user(user_id): \n    user_info[user_id] = copy.deepcopy(new_user)\n    repeated_que_count[user_id] = {}\n    groups[user_id] = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def update_user_part_acc(user_id, question, answered_correctly, elapsed, explan):\n    \n    part = parts.get(question, -1)\n    \n    user_info[user_id][\"part_count\"][part-1] += 1\n    user_info[user_id][\"part_corr\"][part-1] += answered_correctly\n    \n    if not isinstance(explan, pd._libs.missing.NAType) and explan == explan:\n        user_info[user_id][\"had_exp\"] += explan","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def update_user(user_id, had_exp, elapsed, content_id ,answered_correctly, timestamp):\n        \n    user_info[user_id]['count'] += 1\n\n    if repeated_que_count[user_id].get(content_id,  -1) == -1: #If first time question for the user\n        repeated_que_count[user_id][content_id] = 1\n    else:\n        repeated_que_count[user_id][content_id] += 1\n\n    if answered_correctly:\n        user_info[user_id]['correct_count'] += 1\n\n    user_info[user_id]['mean_acc'] = user_info[user_id]['correct_count']/user_info[user_id]['count']\n    \n    \n    update_user_part_acc(user_id, content_id, answered_correctly, elapsed, had_exp)\n    \n    if hard_questions.get(content_id, False):\n        user_info[user_id][\"hard_ct\"] += 1\n        user_info[user_id][\"hard_cr\"] += answered_correctly\n        \n    if easy_questions.get(content_id, False):\n        user_info[user_id][\"easy_ct\"] += 1\n        user_info[user_id][\"easy_cr\"] += answered_correctly     \n        \n    \n    k = que_2_k[content_id]\n    user_info[user_id][\"k_count\"][k] += 1\n    user_info[user_id][\"k_corr\"][k] += answered_correctly\n    \n    if answered_correctly:\n        user_info[user_id][\"recent_corr\"] = timestamp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def update_lec_data(user_id, content_id):\n    \n    if watched_tags.get(str(user_id), -1) == -1:  #If user's first lecture\n        watched_tags[str(user_id)] = {}\n        \n    if user_info.get(user_id, -1) == -1: #In case first action user does is watching a lecture\n        add_user(user_id)\n        \n        \n    user_info[user_id][\"lecs_n\"] += 1\n    \n    lec_tag  = lec_dict[content_id]\n\n    watched_tags[str(user_id)][str(lec_tag)] = 1\n    user_info[user_id]['last_lec'] = content_id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def non_lag_update(user_id, timestamp, elapsed, explan, lec):\n    \n    \n    timestamp = timestamp/8.64e+7\n    \n    diff_timestamp_1 = timestamp - user_info[user_id][\"first_timestamp\"]\n    diff_timestamp_2 = timestamp - user_info[user_id][\"second_timestamp\"]\n    diff_timestamp_3 = timestamp - user_info[user_id][\"third_timestamp\"]\n    diff_timestamp_4 = timestamp - user_info[user_id][\"fourth_timestamp\"]\n    diff_timestamp_5 = timestamp - user_info[user_id][\"fifth_timestamp\"]\n\n    user_info[user_id][\"fifth_timestamp\"] = user_info[user_id][\"fourth_timestamp\"]\n    user_info[user_id][\"fourth_timestamp\"] = user_info[user_id][\"third_timestamp\"]\n    user_info[user_id][\"third_timestamp\"] = user_info[user_id][\"second_timestamp\"]\n    user_info[user_id][\"second_timestamp\"] = user_info[user_id][\"first_timestamp\"]\n    user_info[user_id][\"first_timestamp\"] = timestamp\n    \n    if (user_info[user_id][\"second_timestamp\"] - user_info[user_id][\"third_timestamp\"]) != 0:\n        user_info[user_id][\"ts_diff_shifted\"] = user_info[user_id][\"second_timestamp\"]*8.64e+7 - user_info[user_id][\"third_timestamp\"]*8.64e+7\n    \n    if (user_info[user_id][\"second_timestamp\"] - user_info[user_id][\"fourth_timestamp\"]) != 0:\n        user_info[user_id][\"ts_diff_shifted_2\"] = user_info[user_id][\"second_timestamp\"]*8.64e+7 - user_info[user_id][\"fourth_timestamp\"]*8.64e+7\n        \n        \n    if not isinstance(explan, pd._libs.missing.NAType) and explan == explan and not lec:\n        user_info[user_id][\"priors_5\"].append(explan) \n    \n    if diff_timestamp_1 > 0.083:\n        user_info[user_id][\"sessions\"] += 1\n        user_info[user_id][\"session_count\"] = 0\n        user_info[user_id][\"sum_pauses\"] += diff_timestamp_1\n        \n    user_info[user_id][\"session_count\"] += 1\n    \n    \n    return  diff_timestamp_1, diff_timestamp_2, diff_timestamp_3, diff_timestamp_4, diff_timestamp_5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def update_data(prior_group_answers_correct):  #Update data retrospectively\n    \n    global tmp_data\n    \n    arr = np.array(ast.literal_eval(prior_group_answers_correct))\n\n    for i, line in enumerate(tmp_data): #Loop through users with correct answers\n        \n        user_id = line[cols['user_id']]\n        content_type_id = line[cols['content_type_id']]\n        content_id = line[cols['content_id']]\n        timestamp = line[cols['timestamp']]\n        task_container_id = line[cols['task_container_id']]\n        \n        explan = line[cols['prior_question_had_explanation']]\n        if isinstance(explan, pd._libs.missing.NAType) or explan == explan:\n            explan = 0\n            \n        elapsed = line[cols['prior_question_elapsed_time']]\n        if isinstance(elapsed, pd._libs.missing.NAType) or elapsed == elapsed:\n            elapsed = 0\n            \n        answered_correctly = arr[i]\n        \n        \n        user_arr = groups[user_id]\n        user_arr.insert(len(user_arr), answered_correctly)\n        groups[user_id] = user_arr\n\n        if content_type_id == False: #If question\n\n            if user_info.get(user_id, -1) == -1: #If user is new\n                add_user(user_id)\n                \n            update_user(user_id, explan, elapsed, content_id, answered_correctly, timestamp)\n\n     \n    tmp_data = []  #Flush tmp_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp_data = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess_1(chunk): #Optimized preprocess function for small batches, for batches > 2500 preprocess is better. Includes adding lecture\n    \n    \n    data_1 = chunk.values\n    out = np.zeros((data_1.shape[0], len(features)))\n    \n    batch_counts = data_1[:, [cols[\"user_id\"],cols[\"content_type_id\"]]]\n    batch_counts = Counter(batch_counts[batch_counts[:, 1] == False][:, 0])\n\n    global tmp_data\n\n\n    for i in range(data_1.shape[0]):\n\n        user_id, content_type_id, content_id ,prior_group_answers_correct, timestamp, task_container_id, explan, elapsed = get_meta_data(data_1[i])\n        part = parts.get(content_id, -1)\n        task_count = batch_counts[user_id]\n        \n        #When users answers are provided, update user states\n        if prior_group_answers_correct == prior_group_answers_correct and prior_group_answers_correct != '[]': #If user's previous responses are there\n            update_data(prior_group_answers_correct)\n\n        tmp_data.append(data_1[i].tolist()) #Append incoming data\n        \n        \n\n        if content_type_id: #If lecture, update the lecture state\n            update_lec_data(user_id, content_id)\n\n            \n        #Timestamp difference\n        if user_info.get(user_id, -1) == -1: #If user is new\n            add_user(user_id)        \n            \n        \n        user_info[user_id][\"interaction_n\"] += 1\n\n\n        _ = non_lag_update(user_id, timestamp, elapsed, explan, content_type_id)\n        diff_timestamp_1, diff_timestamp_2, diff_timestamp_3, diff_timestamp_4, diff_timestamp_5 = _\n\n        if not content_type_id:\n            if not isinstance(elapsed, pd._libs.missing.NAType) and elapsed == elapsed:\n                user_info[user_id][\"el_sum\"] += elapsed\n            else:\n                elapsed = 0\n\n            if isinstance(explan, pd._libs.missing.NAType) or explan != explan:\n                explan = 0 \n\n            if user_info[user_id][\"count_2\"] != 0:\n                last_part = user_info[user_id][\"last_part\"]\n                user_info[user_id][\"part_et\"][last_part-1] += elapsed\n                user_info[user_id][\"part_explan\"][last_part-1] += explan\n            \n            user_info[user_id][\"last_part\"] = part\n            \n            user_info[user_id][\"part_count_2\"][part-1] += 1\n            user_info[user_id][\"count_2\"] += 1\n\n            #-----Filling the array -------------------------------------------->\n\n            out[i, features_dict['content_id']] = content_id\n            out[i, features_dict['task_container_id']] = data_1[i, cols['task_container_id']]\n\n\n            #prior_question_elapsed_time, and prior_question_elapsed_time\n            out[i, features_dict['prior_question_elapsed_time']] = elapsed\n            out[i, features_dict['el_avg']] = (user_info[user_id][\"el_sum\"]/(user_info[user_id][\"count_2\"]))/1000\n            out[i, features_dict['prior_question_elapsed_time_u_part_avg']] = (user_info[user_id][\"part_et\"][part-1])/(user_info[user_id][\"part_count_2\"][part-1])\n\n            out[i, features_dict['prior_question_had_explanation']] = explan\n            out[i, features_dict['prior_question_had_explanation_u_part_avg']] = user_info[user_id][\"part_explan\"][part-1]/(user_info[user_id][\"part_count_2\"][part-1]+1)\n\n\n            #out[i, features_dict[\"content_type_id\"]] = content_type_id\n            #Fill all question features\n            out[i, features_dict['bundle_id']:] = np.array(list(questions.get(content_id).values())) #No unkown question expected\n\n            #que_count_user, question_repeated, user_mean, correct_count\n            out[i, features_dict['que_count_user']] = user_info.get(user_id, {}).get('count',0)\n            out[i, features_dict['question_repeated']] = repeated_que_count.get(user_id, {}).get(content_id, 0) + 1\n\n\n            m = user_info.get(user_id, {}).get('mean_acc', 0)\n            if m == 0: #Usually when user is new, mean drops to zero\n                out[i, features_dict['user_mean']] = 0.55\n            else:\n                out[i, features_dict['user_mean']] = m\n\n            out[i, features_dict['opp_mean']] = 1 - out[i, features_dict['user_mean']]\n\n            #out[i, features_dict['correct_count']] = user_info.get(user_id, {}).get('correct_count',0)\n            out[i, features_dict['last_lecture']] = user_info.get(user_id, {}).get('last_lec',0)\n\n\n            #Time gap\n            out[i, features_dict['time_diff']] = diff_timestamp_1\n            out[i, features_dict['time_diff1']] = diff_timestamp_2\n            out[i, features_dict['time_diff2']] = diff_timestamp_3\n            out[i, features_dict['time_diff3']] = diff_timestamp_4\n            out[i, features_dict['time_diff4']] = diff_timestamp_5\n\n            out[i, features_dict['timestamp']] = timestamp/8.64e+7\n            out[i, features_dict['correct_recency']] = (timestamp - user_info[user_id][\"recent_corr\"])/8.64e+7\n\n            kk = user_info[user_id][\"priors_5\"][-5:]\n            if len(kk) != 0: #If array not empty\n                #print(str(len(kk)) + \" \" + str(np.array(kk).mean()))\n                out[i, features_dict['rolling_mean_5_prior_question']] = np.array(kk).mean()\n\n            out[i, features_dict['ts_diff_shifted']] = user_info[user_id][\"ts_diff_shifted\"]\n            out[i, features_dict['ts_diff_shifted_2']] = user_info[user_id][\"ts_diff_shifted_2\"]\n\n            #Container mean\n            out[i, features_dict['container_mean']] = containers_mean[task_container_id]\n            out[i, features_dict['lecs_per']] = user_info[user_id][\"lecs_n\"]/user_info[user_id][\"interaction_n\"]*100\n            out[i, features_dict['sessions']] = user_info[user_id][\"sessions\"]\n            out[i, features_dict['session_count']] = user_info[user_id][\"session_count\"]\n\n            if user_info[user_id][\"count\"] != 0:\n                out[i, features_dict['prior_question_had_explanation_ratio']] = user_info[user_id][\"had_exp\"]/user_info[user_id][\"count\"]\n\n            if user_info[user_id][\"sessions\"] != 0:\n                out[i, features_dict['mean_pause']] = user_info[user_id][\"sum_pauses\"]/user_info[user_id][\"sessions\"]\n\n\n            #Easy ratio\n            nn = user_info.get(user_id, {}).get('easy_ct',0)\n            if nn != 0:\n                out[i, features_dict['easy_ratio']] = user_info.get(user_id, {}).get('easy_cr',0)/nn\n\n            out[i, features_dict['easy_ratio_opp']] = 1 - out[i, features_dict['easy_ratio']]\n\n\n            #Hard ratio\n            nn = user_info.get(user_id, {}).get('hard_ct',0)\n            if nn != 0:\n                out[i, features_dict['hard_ratio']] = user_info.get(user_id, {}).get('hard_cr',0)/nn\n\n            out[i, features_dict['hard_ratio_opp']] = 1 - out[i, features_dict['hard_ratio']]\n\n\n\n            nn = user_info.get(user_id, {}).get('part_count',[])\n            if nn != []: #User exists\n                part = int(out[i, features_dict['part']] - 1)\n                ct = nn[part]\n                cr = user_info[user_id]['part_corr'][part]\n\n                if ct != 0:\n                    out[i, features_dict['part_mean']] = cr/ct\n\n\n            #Rolling mean\n            if groups.get(user_id, -1) != -1 and groups[user_id] != []:\n\n\n                last_arr = np.array(groups[user_id][-40:])\n\n                out[i, features_dict['rolling_mean']] = numpy_ewma_vectorized_v2(np.array(last_arr[-30:]),5)[-1]  #Limit the length of groups array\n                \n                \n                last_arr = last_arr[last_arr != -1]  #Remove lecture data\n                out[i, features_dict['ewm_mean_10']] = numpy_ewma_vectorized_v2(np.array(last_arr[-30:]),5)[-1]\n\n                out[i, features_dict['rolling_mean_10']] = np.array(last_arr[-10:]).mean()\n                out[i, features_dict['rolling_mean_15']] = np.array(last_arr[-15:]).mean()\n                out[i, features_dict['rolling_mean_5']] = np.array(last_arr[-5:]).mean()\n\n\n            out[i, features_dict['prior_part_mean']] = prior_part_mean_dict[int(out[i, features_dict['part']])]\n\n            k = que_2_k[content_id]\n            out[i, features_dict['k']] = k\n            if user_info[user_id][\"k_count\"][k] != 0:\n                out[i, features_dict['k_acc']] = user_info[user_id][\"k_corr\"][k]/user_info[user_id][\"k_count\"][k]\n\n            \n            out[i, features_dict['wut']] = user_info[user_id][\"ts_diff_shifted\"] - elapsed*task_count\n            if out[i, features_dict['wut']] < 0:\n                out[i, features_dict['wut']] = 0\n                \n             \n            #Watched_n\n            usr = watched_tags.get(str(user_id), -1)\n            if usr != -1:\n\n                nn = 0\n\n                for k in range(6):\n                    nn += usr.get(str(int(questions1[content_id][\"tags\"+str(k+1)])), 0) \n\n                out[i, features_dict['watched']] = nn\n\n\n\n    return out\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\n\nmodel_lgbm = lgb.Booster(model_file='../input/lgbm-test/lgb_classifier.txt')\nstack_lgbm = lgb.Booster(model_file='../input/lgbm-test/lgb_stack.txt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vals = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nfor (test_data,sample_prediction_df) in iter_test:\n    \n    if not isinstance(vals, int): #First value case\n        \n        #vals = vals[vals[:,2] != 1] #Remove lectures from old vals\n        \n        if test_data.iloc[0].prior_group_answers_correct == test_data.iloc[0].prior_group_answers_correct:\n            past_vals = np.array(ast.literal_eval(test_data.iloc[0].prior_group_answers_correct)) \n            past_answers = np.array(ast.literal_eval(test_data.iloc[0].prior_group_responses))\n            \n            past_vals = np.concatenate((vals, past_vals.reshape(len(past_vals),1)), axis=1)\n            past_vals = np.concatenate((past_vals, past_answers.reshape(len(past_answers),1)), axis=1)\n\n            update_group_var(past_vals)  #Update database with the vals of the last batch\n    \n    vals = test_data[[\"user_id\",\"content_id\", \"content_type_id\", \"task_container_id\",\"timestamp\",\"prior_question_elapsed_time\"]].values\n\n    test_transform = preprocess_1(test_data)\n    \n    \n    lgbm_predic = model_lgbm.predict(test_transform)\n    st_predic = np.array(split_preds(vals))\n    \n    \n    #test_transform = np.insert(test_transform, lgb_preds_idx, lgbm_predic, axis=1)\n    #test_transform = np.insert(test_transform, st_preds_idx, st_predic, axis=1)\n    \n    test_data['answered_correctly'] = (lgbm_predic*0.85 + 1.15*st_predic)/2 #stack_lgbm.predict(test_transform)\n    env.predict(test_data.loc[test_data['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!head submission.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}