{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nimport pickle\nimport psutil\nimport joblib\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\n\nimport sqlite3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"c1, c1_2, c2, c3 , c4 = 0.175, 0.075, 0.25, 0.25, 0.25","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## SAINT+"},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_SEQ = 100\nn_part = 7\nD_MODEL = 256\nN_LAYER = 2\nDROPOUT = 0.1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def feature_time_lag(df, time_dict):\n\n tt = np.zeros(len(df), dtype=np.int64)\n\n for ind, row in enumerate(df[['user_id','timestamp','task_container_id']].values):\n\n if row[0] in time_dict.keys():\n if row[2]-time_dict[row[0]][1] == 0:\n\n tt[ind] = time_dict[row[0]][2]\n\n else:\n t_last = time_dict[row[0]][0]\n task_ind_last = time_dict[row[0]][1]\n tt[ind] = row[1]-t_last\n time_dict[row[0]] = (row[1], row[2], tt[ind])\n else:\n # time_dict : timestamp, task_container_id, lag_time\n time_dict[row[0]] = (row[1], row[2], -1)\n tt[ind] = 0\n\n df[\"time_lag\"] = tt\n return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FFN(nn.Module):\n def __init__(self, state_size=200):\n super(FFN, self).__init__()\n self.state_size = state_size\n\n self.lr1 = nn.Linear(state_size, state_size)\n self.relu = nn.ReLU()\n self.lr2 = nn.Linear(state_size, state_size)\n self.dropout = nn.Dropout(DROPOUT)\n \n def forward(self, x):\n x = self.lr1(x)\n x = self.relu(x)\n x = self.lr2(x)\n return self.dropout(x)\n\ndef future_mask(seq_length):\n future_mask = np.triu(np.ones((seq_length, seq_length)), k=1).astype('bool')\n return torch.from_numpy(future_mask)\n\n\nclass SAINTModel(nn.Module):\n def __init__(self, n_skill, n_part, max_seq=MAX_SEQ, embed_dim= 128, elapsed_time_cat_flag = True):\n super(SAINTModel, self).__init__()\n\n self.n_skill = n_skill\n self.embed_dim = embed_dim\n self.n_cat = n_part\n self.elapsed_time_cat_flag = elapsed_time_cat_flag\n\n self.e_embedding = nn.Embedding(self.n_skill+1, embed_dim) ## exercise\n self.c_embedding = nn.Embedding(self.n_cat+1, embed_dim) ## category\n self.pos_embedding = nn.Embedding(max_seq-1, embed_dim) ## position\n self.res_embedding = nn.Embedding(2+1, embed_dim) ## response\n\n\n if self.elapsed_time_cat_flag == True:\n self.elapsed_time_embedding = nn.Embedding(300+1, embed_dim) ## elapsed time (the maximum elasped time is 300)\n self.lag_embedding1 = nn.Embedding(300+1, embed_dim) ## lag time1 for 300 seconds\n self.lag_embedding2 = nn.Embedding(1440+1, embed_dim) ## lag time2 for 1440 minutes\n self.lag_embedding3 = nn.Embedding(365+1, embed_dim) ## lag time3 for 365 days\n\n else:\n self.elapsed_time_embedding = nn.Linear(1, embed_dim, bias=False) ## elapsed time\n self.lag_embedding = nn.Linear(1, embed_dim, bias=False) ## lag time\n\n\n self.exp_embedding = nn.Embedding(2+1, embed_dim) ## user had explain\n\n self.transformer = nn.Transformer(nhead=8, d_model = embed_dim, num_encoder_layers= N_LAYER, num_decoder_layers= N_LAYER, dropout = DROPOUT)\n\n self.dropout = nn.Dropout(DROPOUT)\n self.layer_normal = nn.LayerNorm(embed_dim) \n self.ffn = FFN(embed_dim)\n self.pred = nn.Linear(embed_dim, 1)\n \n def forward(self, question, part, response, elapsed_time, lag_time, exp):\n\n device = question.device \n\n ## embedding layer\n question = self.e_embedding(question)\n part = self.c_embedding(part)\n pos_id = torch.arange(question.size(1)).unsqueeze(0).to(device)\n pos_id = self.pos_embedding(pos_id)\n res = self.res_embedding(response)\n exp = self.exp_embedding(exp)\n\n if self.elapsed_time_cat_flag == True:\n\n ## feature engineering\n ## elasped time\n elapsed_time = torch.true_divide(elapsed_time, 1000)\n elapsed_time = torch.round(elapsed_time)\n elapsed_time = torch.where(elapsed_time.float() <= 300, elapsed_time, torch.tensor(300.0).to(device)).long()\n elapsed_time = self.elapsed_time_embedding(elapsed_time)\n\n ## lag_time1\n lag_time = torch.true_divide(lag_time, 1000)\n lag_time = torch.round(lag_time)\n lag_time1 = torch.where(lag_time.float() <= 300, lag_time, torch.tensor(300.0).to(device)).long()\n\n ## lag_time2\n lag_time = torch.true_divide(lag_time, 60)\n lag_time = torch.round(lag_time)\n lag_time2 = torch.where(lag_time.float() <= 1440, lag_time, torch.tensor(1440.0).to(device)).long()\n\n ## lag_time3\n lag_time = torch.true_divide(lag_time, 1440)\n lag_time = torch.round(lag_time)\n lag_time3 = torch.where(lag_time.float() <= 365, lag_time, torch.tensor(365.0).to(device)).long()\n\n ## lag time\n lag_time1 = self.lag_embedding1(lag_time1) \n lag_time2 = self.lag_embedding2(lag_time2) \n lag_time3 = self.lag_embedding3(lag_time3)\n\n else:\n\n elapsed_time = elapsed_time.view(-1,1)\n elapsed_time = self.elapsed_time_embedding(elapsed_time)\n elapsed_time = elapsed_time.view(-1, MAX_SEQ-1, self.embed_dim)\n\n lag_time = lag_time.view(-1,1)\n lag_time = self.lag_embedding(lag_time)\n lag_time = lag_time.view(-1, MAX_SEQ-1, self.embed_dim)\n\n # elapsed_time = elapsed_time.view(-1, MAX_SEQ-1, 1) ## [batch, s_len] => [batch, s_len, 1]\n # elapsed_time = self.elapsed_time_embedding(elapsed_time)\n\n\n enc = question + part + pos_id + exp\n dec = pos_id + res + elapsed_time + lag_time1 + lag_time2 + lag_time3\n\n enc = enc.permute(1, 0, 2) # x: [bs, s_len, embed] => [s_len, bs, embed]\n dec = dec.permute(1, 0, 2)\n mask = future_mask(enc.size(0)).to(device)\n\n att_output = self.transformer(enc, dec, src_mask=mask, tgt_mask=mask, memory_mask = mask)\n att_output = self.layer_normal(att_output)\n att_output = att_output.permute(1, 0, 2) # att_output: [s_len, bs, embed] => [bs, s_len, embed]\n\n x = self.ffn(att_output)\n x = self.layer_normal(x + att_output)\n x = self.pred(x)\n\n return x.squeeze(-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_skill = 13523\ngroup = joblib.load(\"../input/saint-plus-data-new/group_20210102.pkl.zip\")\nquestions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')\ntime_dict = joblib.load(\"../input/saint-plus-data-new/time_dict.pkl.zip\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nmodel1 = SAINTModel(n_skill, n_part, embed_dim= D_MODEL)\ntry:\n model1.load_state_dict(torch.load(\"../input/saint-plus-model/saint_plus_model_20210102_padding_v2.pt\"))\nexcept:\n model1.load_state_dict(torch.load(\"../input/saint-plus-model/saint_plus_model_20210102_padding_v2.pt\", map_location='cpu'))\nmodel1.to(device)\nmodel1.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nmodel3 = SAINTModel(n_skill, n_part, embed_dim= D_MODEL)\ntry:\n model3.load_state_dict(torch.load(\"../input/saint-plus-model/saint_plus_model_20210107_v3.pt\"))\nexcept:\n model3.load_state_dict(torch.load(\"../input/saint-plus-model/saint_plus_model_20210107_v3.pt\", map_location='cpu'))\nmodel3.to(device)\nmodel3.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"N_LAYER = 3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nmodel2 = SAINTModel(n_skill, n_part, embed_dim= D_MODEL)\ntry:\n model2.load_state_dict(torch.load(\"../input/saint-plus-model/saint_plus_model_20210103.pt_v2\"))\nexcept:\n model2.load_state_dict(torch.load(\"../input/saint-plus-model/saint_plus_model_20210103.pt_v2\", map_location='cpu'))\nmodel2.to(device)\nmodel2.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestDataset(Dataset):\n def __init__(self, samples, test_df, n_skills, max_seq=MAX_SEQ): \n super(TestDataset, self).__init__()\n self.samples = samples\n self.user_ids = [x for x in test_df[\"user_id\"].unique()]\n self.test_df = test_df\n self.n_skill = n_skills\n self.max_seq = max_seq\n\n def __len__(self):\n return self.test_df.shape[0]\n\n def __getitem__(self, index):\n test_info = self.test_df.iloc[index]\n\n user_id = test_info[\"user_id\"]\n target_id = test_info[\"content_id\"]\n part = test_info[\"part\"]\n pri_quest_elap = test_info[\"prior_question_elapsed_time\"]\n time_lag = test_info[\"time_lag\"]\n pri_quest_exp = test_info[\"prior_question_had_explanation\"]\n \n q = np.zeros(self.max_seq, dtype=int)\n qa = np.zeros(self.max_seq, dtype=int)\n res = np.zeros(self.max_seq, dtype=int)\n p = np.zeros(self.max_seq, dtype=int)\n pri_elap = np.zeros(self.max_seq, dtype=int)\n lag = np.zeros(self.max_seq, dtype=int)\n pri_exp = np.zeros(self.max_seq, dtype=int)\n\n if user_id in self.samples.index:\n q_, qa_, p_, pri_elap_, lag_, pri_exp_ = self.samples[user_id]\n \n seq_len = len(q_)\n \n ## for zero padding\n q_ = q_+1\n pri_exp_ = pri_exp_ + 1\n res_ = qa_ + 1\n \n\n if seq_len >= self.max_seq:\n q = q_[-self.max_seq:]\n qa = qa_[-self.max_seq:]\n res = res_[-self.max_seq:]\n p = p_[-self.max_seq:]\n pri_elap = pri_elap_[-self.max_seq:]\n lag = lag_[-self.max_seq:]\n pri_exp = pri_exp_[-self.max_seq:]\n \n else:\n q[-seq_len:] = q_\n qa[-seq_len:] = qa_\n res[-seq_len:] = res_\n p[-seq_len:] = p_\n pri_elap[-seq_len:] = pri_elap_\n lag[-seq_len:] = lag_\n pri_exp[-seq_len:] = pri_exp_\n \n \n\n exercise = np.append(q[2:], [target_id+1])\n part = np.append(p[2:], [part])\n elap = np.append(pri_elap[2:], [pri_quest_elap])\n lag = np.append(lag[2:], [time_lag])\n pri_exp = np.append(pri_exp[2:], [pri_quest_exp+1])\n\n response = res[1:]\n\n return exercise, part, response, elap, lag, pri_exp","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Catboost Feature Pipeline"},{"metadata":{},"cell_type":"markdown","source":"# new_feature 0104 "},{"metadata":{"trusted":true},"cell_type":"code","source":"lt_correct_dict = pickle.load(open('../input/arvis-feature/last_timestamp_correct.pkl', 'rb'))\nnp_uq_td = pickle.load(open(\"../input/uq-data/np_uq_td_0518.pkl.data\",\"rb\")) \ncurr_u_dict = pickle.load(open(\"../input/uq-data/curr_u_dict_0614_only_user_three_time_diff.pkl.data\",\"rb\"))\nmax_timestamp_u_dict = pickle.load(open(\"../input/arvis-feature/max_timestamp_u_dict_2015.pkl\",\"rb\")) \nmax_timestamp_u_dict2 = pickle.load(open(\"../input/arvis-feature/max_timestamp_u_dict2_2015.pkl\",\"rb\")) \nmax_timestamp_u_dict3 = pickle.load(open(\"../input/arvis-feature/max_timestamp_u_dict3_2015.pkl\",\"rb\")) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cp ../input/uq-data/user_ques_db.db ./user_ques_db.db","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_uq_feats_and_update(df):\n conn = sqlite3.connect('user_ques_db.db')\n cursor = conn.cursor()\n global idx\n uq_timediff = np.zeros(len(df), dtype=np.uint64) \n for cnt,row in enumerate(df[['user_id','content_id','timestamp']].itertuples(index=False)): \n cursor.execute(f'select idx from user where user_id = {row[0]} and content_id = {row[1]}')\n tmp_idx = cursor.fetchall()\n if tmp_idx == []: # no got the idx for user_id-content_id pair\n uq_timediff[cnt] = 0\n np_uq_td[idx] = row[2]\n cursor.execute(f'insert into user (user_id, content_id, idx) values ({row[0]}, {row[1]}, {idx})')\n idx += 1\n else: # got the idx for user_id-content_id pair\n tmp_idx = tmp_idx[0][0]\n uq_timediff[cnt] = row[2] - np_uq_td[tmp_idx]\n np_uq_td[tmp_idx] = row[2]\n\n cursor.close()\n conn.commit()\n conn.close()\n \n uq_feats_df = pd.DataFrame({'curr_uq_time_diff':uq_timediff}) \n df = pd.concat([df, uq_feats_df], axis=1)\n return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_user_feats_without_update(df):\n utdiff = np.zeros(len(df), dtype=np.uint64)\n utdiff_mean = np.zeros(len(df), dtype=np.uint64) \n uelapdiff = np.zeros(len(df), dtype=np.float32) \n for cnt,row in enumerate(df[['user_id','timestamp','prior_question_elapsed_time']].itertuples(index=False)): \n if row[0] in curr_u_dict:\n utdiff[cnt] = row[1] - curr_u_dict[row[0]][\"uts\"]\n utdiff_mean[cnt] = curr_u_dict[row[0]][\"utsdiff\"][1] / curr_u_dict[row[0]][\"utsdiff\"][0]\n uelapdiff[cnt] = row[2] - curr_u_dict[row[0]][\"uelapdiff\"]\n else:\n utdiff[cnt] = 0; utdiff_mean[cnt] = 0; uelapdiff[cnt] = 0;\n \n user_feats_df = pd.DataFrame({'curr_user_time_diff':utdiff, 'curr_user_time_diff_mean':utdiff_mean, \n 'curr_user_elapsed_time_diff':uelapdiff\n }) \n user_feats_df['curr_user_elapsed_time_diff'].fillna(0, inplace=True) \n df = pd.concat([df, user_feats_df], axis=1)\n return df\n\ndef update_user_feats(df):\n for cnt,row in enumerate(df[['user_id','content_id','answered_correctly','timestamp','prior_question_elapsed_time']].itertuples(index=False)): \n if row[0] in curr_u_dict:\n curr_u_dict[row[0]][\"uts\"] = row[3]\n curr_u_dict[row[0]][\"utsdiff\"][0] += 1 \n curr_u_dict[row[0]][\"utsdiff\"][1] += row[3] \n curr_u_dict[row[0]][\"uelapdiff\"] = row[4] \n else:\n curr_u_dict[row[0]] = {}\n curr_u_dict[row[0]][\"uts\"] = row[3]\n curr_u_dict[row[0]][\"utsdiff\"] = [1, row[3]] \n curr_u_dict[row[0]][\"uelapdiff\"] = row[4] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## only in training！！\ndef add_user_feats(df):\n utdiff = np.zeros(len(df), dtype=np.uint64)\n utdiff_mean = np.zeros(len(df), dtype=np.uint64) \n uelapdiff = np.zeros(len(df), dtype=np.float32) \n for cnt,row in enumerate(tqdm(df[['user_id','content_id','answered_correctly',\n 'timestamp','prior_question_elapsed_time',\n ]].itertuples(index=False),total=df.shape[0])): \n if row[0] in curr_u_dict:\n # 写入np\n utdiff[cnt] = row[3] - curr_u_dict[row[0]][\"uts\"]\n utdiff_mean[cnt] = curr_u_dict[row[0]][\"utsdiff\"][1] / curr_u_dict[row[0]][\"utsdiff\"][0]\n uelapdiff[cnt] = row[4] - curr_u_dict[row[0]][\"uelapdiff\"]\n # 写入字典\n curr_u_dict[row[0]][\"uts\"] = row[3]\n curr_u_dict[row[0]][\"utsdiff\"][0] += 1 \n curr_u_dict[row[0]][\"utsdiff\"][1] += row[3] \n curr_u_dict[row[0]][\"uelapdiff\"] = row[4] \n else:\n # 写入np\n utdiff[cnt] = 0; utdiff_mean[cnt] = 0; uelapdiff[cnt] = 0;\n # 写入字典\n curr_u_dict[row[0]] = {}\n curr_u_dict[row[0]][\"uts\"] = row[3]\n curr_u_dict[row[0]][\"utsdiff\"] = [1, row[3]] \n curr_u_dict[row[0]][\"uelapdiff\"] = row[4] \n \n user_feats_df = pd.DataFrame({\n 'curr_user_time_diff':utdiff, 'curr_user_time_diff_mean':utdiff_mean, \n 'curr_user_elapsed_time_diff':uelapdiff\n }) \n\n user_feats_df['curr_user_elapsed_time_diff'].fillna(0, inplace=True) \n df = pd.concat([df, user_feats_df], axis=1)\n return df\n# curr_u_dict = {}\n# train_df = add_user_feats(train_df)\n# valid_df = add_user_feats(valid_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def lagtime_for_test(df):\n lagtime_mean = 0\n lagtime_mean2 = 0\n lagtime_mean3 = 0\n lagtime = np.zeros(len(df), dtype=np.float32)\n lagtime2 = np.zeros(len(df), dtype=np.float32)\n lagtime3 = np.zeros(len(df), dtype=np.float32)\n for i, (user_id,\n content_type_id,\n timestamp,\n content_id,) in enumerate(zip(df['user_id'].values, df['content_type_id'].values, df['timestamp'].values, df['content_id'].values)):\n if content_type_id==0:\n if user_id in max_timestamp_u_dict['max_time_stamp'].keys():\n lagtime[i]=timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]\n if(max_timestamp_u_dict2['max_time_stamp2'][user_id]==lagtime_mean2):\n lagtime2[i]=lagtime_mean2\n lagtime3[i]=lagtime_mean3\n else:\n lagtime2[i]=timestamp-max_timestamp_u_dict2['max_time_stamp2'][user_id]\n if(max_timestamp_u_dict3['max_time_stamp3'][user_id]==lagtime_mean3):\n lagtime3[i]=lagtime_mean3\n else:\n lagtime3[i]=timestamp-max_timestamp_u_dict3['max_time_stamp3'][user_id]\n max_timestamp_u_dict3['max_time_stamp3'][user_id]=max_timestamp_u_dict2['max_time_stamp2'][user_id]\n max_timestamp_u_dict2['max_time_stamp2'][user_id]=max_timestamp_u_dict['max_time_stamp'][user_id]\n max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp\n else:\n lagtime[i]=lagtime_mean\n max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})\n lagtime2[i]=lagtime_mean2\n max_timestamp_u_dict2['max_time_stamp2'].update({user_id:lagtime_mean2})\n lagtime3[i]=lagtime_mean3\n max_timestamp_u_dict3['max_time_stamp3'].update({user_id:lagtime_mean3})\n df[\"lag_time\"]= lagtime\n df[\"lag_time2\"]= lagtime2\n df[\"lag_time3\"]= lagtime3\n df[\"lag_time\"].fillna(-1, inplace=True)\n df[\"lag_time2\"].fillna(-1, inplace=True)\n df[\"lag_time3\"].fillna(-1, inplace=True)\n df['lag_time'] = df['lag_time'].replace(0, method=\"ffill\")\n df['lag_time2'] = df['lag_time2'].replace(0, method=\"ffill\")\n df['lag_time3'] = df['lag_time3'].replace(0, method=\"ffill\")\n \n df[\"lag_time\"] = df[\"lag_time\"].astype(\"uint64\")\n df[\"lag_time2\"] = df[\"lag_time2\"].astype(\"uint64\")\n df[\"lag_time3\"] = df[\"lag_time3\"].astype(\"uint64\")\n return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# функция расчета кумулятивной суммы по переменной и обновления словаря\n# cumulative sum and dict update\ndef add_feats(df_np, feat_dict, col_idx, col_feat):\n '''\n df_np - pandas датафрейм приведенный к numpy .to_numpy()\n feat_dict - numpy словарь вида [idx - feat]\n col_idx - индекс столбца переменной по которой будем агрегировать\n col_feat - индекс столбца переменной по которой будем считать кумулятивную сумму\n ''' \n current_feat_value = np.zeros(len(df_np))\n for cnt, row in enumerate(df_np[:,[col_idx, col_feat]]):\n current_feat_value[cnt] = feat_dict[row[0]] # текущее значение из словаря\n feat_dict[row[0]] += row[1] # в словарь добавляем текущее значение строки\n # заменяем обновленными значениями\n df_np[:, col_feat] = current_feat_value\n \n return df_np\n\n\n# функция добавления данных из словаря (новый пользователь = добавляем с 0)\ndef add_feats_from_dict(df_np, feat_dict, col_idx, col_dict=-1):\n '''\n df_np - pandas датафрейм приведенный к numpy .to_numpy()\n feat_dict - numpy словарь вида [idx - feat]\n col_idx - индекс столбца переменной по которой добавлять данные из словаря\n ''' \n current_feat_value = np.zeros(len(df_np))\n for cnt, idx in enumerate(df_np[:,col_idx]):\n if col_dict == -1: current_feat_value[cnt] = feat_dict[idx] # текущее значение из словаря = значение\n else: current_feat_value[cnt] = feat_dict[idx][col_dict] # текущее значение из словаря = значение из списка с индексом col_dict\n \n return (np.c_[ df_np, current_feat_value ])\n #return np.concatenate((df_np, current_feat_value), axis=1)\n\n# функция добавления данных из словаря (новый индекс = забираем данные с индексом = -100)\ndef add_feats_from_dict_got_new_user(df_np, feat_dict, col_idx, col_dict=-1):\n '''\n df_np - pandas датафрейм приведенный к numpy .to_numpy()\n feat_dict - numpy словарь вида [idx - feat]\n col_idx - индекс столбца переменной по которой добавлять данные из словаря\n ''' \n current_feat_value = np.zeros(len(df_np))\n for cnt, idx in enumerate(df_np[:,col_idx]):\n # row[0] = index\n # row[1] = value\n if idx in feat_dict.keys(): # если индекс есть в словаре - берем данные по его id \n if col_dict == -1: current_feat_value[cnt] = feat_dict[idx] # текущее значение из словаря = значение\n else: current_feat_value[cnt] = feat_dict[idx][col_dict] # текущее значение из словаря = значение из списка с индексом col_dict\n else: # если индекса нет в словаре - берем данные по id = -100\n if col_dict == -1: current_feat_value[cnt] = feat_dict[-100] # текущее значение из словаря = значение\n else: current_feat_value[cnt] = feat_dict[-100][col_dict] # текущее значение из словаря = значение из списка с индексом col_dict\n feat_dict[idx] = feat_dict[-100] # добавляем новый индекс\n \n return (np.c_[ df_np, current_feat_value ])\n #return np.concatenate((df_np, current_feat_value), axis=1)\n\n\n# функция обновления словаря\ndef update_dict(df_pd, feat_dict, col_idx, col_feat, col_dict=-1):\n for row in df_pd[['content_type_id', col_idx, col_feat]].values:\n if row[0] == 0:\n if col_dict == -1: feat_dict[row[1]] += row[2] # добавляем текущее значение\n else: feat_dict[row[1]][col_dict] += row[2] # добавляем текущее значение в стобец\n \n# добавление ohe во фрейм для заданной переменной и ее значения\ndef add_ohe(df, col_feat, oh_value):\n # df - общий фрейм np array\n # col_feat - индекс переменной\n return (np.c_[ df, np.array([int(i == oh_value) for i in df[:,col_feat]]) ]) \n #return np.concatenate((df, np.array([int(i == oh_value) for i in df[:,col_feat]])), axis=1)\n\n\n# user_slice_accuracy_n - точность ответа на последних N вопросах\n# функция добавления данных из словаря\ndef user_slice_accuracy_n_get(df_pd, feat_dict):\n global_first_question_accuracy = 0.6453965159034877 \n current_list = np.zeros(len(df_pd))\n for cnt, (user, content_type_id) in enumerate(df_pd[['user_id', 'content_type_id']].values): \n # если вопрос\n if content_type_id == 0:\n # если user_id есть в словаре\n if user in feat_dict: \n current_list[cnt] = np.mean(feat_dict[user])\n else: # если user_id нет в словаре - берем global_first_question_accuracy\n current_list[cnt] = 0.6454\n else:\n current_list[cnt] = 0\n return current_list\n# user_slice_accuracy_n - точность ответа на последних N вопросах \n# функция добавления обновления словаря\ndef user_slice_accuracy_n_update(df_pd, feat_dict, border=5): \n for cnt, (user, answer) in enumerate(df_pd[['user_id', 'answered_correctly']].values): \n # если user_id есть в словаре\n if user in feat_dict: \n feat_dict[user].append(answer) # добавляем текущий ответ\n feat_dict[user] = feat_dict[user][-border:] # ограничиваем количество\n # если user_id нет в словаре\n else: \n feat_dict[user] = [answer] # добавляем текущий ответ \n return feat_dict\n\n# user_slice_accuracy_session - точность ответа на последней сессии\n# функция добавления данных из словаря\ndef user_slice_accuracy_session_get(df_pd, feat_dict, session_max_time=12): \n current_list = np.zeros(len(df_pd))\n for cnt, (user, timestamp, content_type_id) in enumerate(df_pd[['user_id', 'timestamp', 'content_type_id']].values): \n # если вопрос\n if content_type_id == 0:\n # если user_id есть в словаре\n if user in feat_dict: \n # считаем дельту timestamp в часах\n time_delta_h = (timestamp - feat_dict[user][1]) / 1000 / 60 / 60\n if time_delta_h < session_max_time: \n current_list[cnt] = np.mean(feat_dict[user][0])\n else: \n current_list[cnt] = 0.67\n # если user_id нет в словаре\n else: \n current_list[cnt] = 0.67 \n # если лекция\n else: current_list[cnt] = 0\n return current_list\n# user_slice_accuracy_session - точность ответа на последней сессии\n# функция обновления словаря\ndef user_slice_accuracy_session_update(df_pd, feat_dict, session_max_time=12): \n for cnt, (user, answer, timestamp) in enumerate(df_pd[['user_id', 'answered_correctly', 'timestamp']].values): \n # если user_id есть в словаре\n if user in feat_dict: \n # считаем дельту timestamp в часах\n time_delta_h = (timestamp - feat_dict[user][1]) / 1000 / 60 / 60\n if time_delta_h < session_max_time: \n feat_dict[user][0].append(answer)\n feat_dict[user][1] = timestamp\n else: \n feat_dict[user][0] = [answer]\n feat_dict[user][1] = timestamp\n # если user_id нет в словаре\n else: \n feat_dict[user] = [[answer], timestamp]\n return feat_dict\n \n\n# количество вопросов по content_id на которые отвечал каждый user_id накопленно: берем данные + обновляем словарь \ndef user_question_attempt_cnt_get_update(df_pd, feat_dict):\n current_feat_value = np.zeros(len(df_pd))\n for idx, (user_id, content_id, content_type_id) in enumerate(df_pd[['user_id', 'content_id', 'content_type_id']].values):\n \n # если вопрос\n if content_type_id == 0: \n current_feat_value[idx] = list(feat_dict[user_id]).count(content_id)\n # обновляем = добавляем content_id в лист\n feat_dict[user_id] = np.append(feat_dict[user_id], content_id)\n # если лекция - ставим 0\n else:\n current_feat_value[idx] = 0\n\n return current_feat_value\n\n# part_l_q_cnt - функция добавления данных из словаря + обновление словаря\ndef user_lectures_part(df_pd, feat_dict): \n current_list = np.zeros(len(df_pd))\n for cnt, (user, content_type_id, part_q, part_l) in enumerate(df_pd[['user_id', 'content_type_id', 'part_q', 'part_l']].values):\n part_q = max(0, part_q) # для значения -100 (новый id)\n \n # если вопрос\n if content_type_id == 0:\n # если user_id есть в словаре - берем данные по его id \n if user in feat_dict: \n current_list[cnt] = feat_dict[user][part_q] \n # если user_id нет в словаре - добавляем с нулями, забираем 0\n else: \n feat_dict[user] = [0] * 8\n current_list[cnt] = 0 \n\n # если лекция\n else:\n # если user_id есть в словаре - обновляем словарь\n if user in feat_dict: \n feat_dict[user][part_l] += 1 \n # если user_id нет в словаре - добавляем с нулями\n else: \n feat_dict[user] = [0] * 8\n feat_dict[user][part_l] += 1 \n # забираем 0\n current_list[cnt] = 0\n \n return current_list\n\n# lecture_cnt - функция добавления данных из словаря\ndef user_lecture_cnt(df_pd, feat_dict):\n '''\n df_pd - pandas датафрейм приведенный к numpy .to_numpy()\n feat_dict - словарь\n ''' \n current_list = np.zeros(len(df_pd))\n for cnt, user in enumerate(df_pd['user_id'].values):\n if user in feat_dict: # если user_id есть в словаре - берем данные по его id \n current_list[cnt] = sum(feat_dict[user]) \n else: # если user_id нет в словаре - забираем 0\n current_list[cnt] = 0 \n \n return current_list\n\n\n# tag_l_q_equal_cnt - слушал ли юзер лекцию по тегу равную тегу вопроса (количество) - функция добавления данных из словаря + обновление словаря\ndef user_l_q_tag_equal(df_pd, feat_dict):\n '''\n df_pd - pandas датафрейм приведенный к numpy .to_numpy()\n feat_dict - словарь\n border - количество последних ответов к отслеживанию\n ''' \n current_list = np.zeros(len(df_pd))\n for idx, (user, content_type_id, tag, tags_list) in enumerate(df_pd[['user_id', 'content_type_id', 'tag_l', 'tags_list']].values):\n # если вопрос\n if content_type_id == 0:\n # количество тегов = тегу прослушенной лекции\n current_list[idx] = len(set(feat_dict[user]) & set(tags_list))\n # если лекция\n else:\n feat_dict[user].append(int(tag)) # добавляем текущий тег лекции\n current_list[idx] = 0\n \n return current_list\n\n# данные из словарей вопросов и лекций\ndef get_q_l(df_pd, feat_dict, key):\n current_list = []\n for idx, content_id in enumerate(df_pd['content_id'].values):\n if content_id in feat_dict:\n current_list.append(feat_dict[content_id][key])\n else:\n current_list.append(-100)\n return current_list\n\n# функция обновления словаря - возвращает датафрейм с новым значением\ndef dict_user_timestampsdelta_get_update_3(df, feat_dict):\n #if need_sort: df = df.sort_values(['user_id', 'timestamp'])\n q_list = np.zeros((len(df), 3), dtype = np.float32)\n l_list = np.zeros((len(df), 3), dtype = np.float32)\n for cnt, (user_id, timestamp, content_type_id) in enumerate(df[['user_id', 'timestamp', 'content_type_id']].values):\n # переводим в минуты\n timestamp = timestamp / 1000 / 60\n # если user_id есть в словаре - берем значения, обновляем словарь\n if user_id in feat_dict:\n # вычитаем из текущего времени каждый элемент из словаря\n q_list[cnt] = np.array([timestamp - t for t in feat_dict[user_id][0]]) # вопросы\n l_list[cnt] = np.array([timestamp - t for t in feat_dict[user_id][1]]) # лекции\n # обновляем значения в словаре (content_type_id==0 вопрос, content_type_id==1 лекция)\n feat_dict[user_id][int(content_type_id)].pop(0) # удаляем первый элемент\n feat_dict[user_id][int(content_type_id)].append(timestamp) # добавляем в конец текущее время\n \n # если user_id нет - добавляем и забираем с np.nan\n else:\n if content_type_id == 1: feat_dict[user_id] = [[np.nan, np.nan, np.nan], [np.nan, np.nan, 0]]\n else: feat_dict[user_id] = [[np.nan, np.nan, 0], [np.nan, np.nan, np.nan]]\n q_list[cnt] = np.array([np.nan, np.nan, np.nan])\n l_list[cnt] = np.array([np.nan, np.nan, np.nan])\n \n # добавляем в основной датафрейм\n for i in [0, 1, 2]:\n df['prior_question_' + str(i+1) + '_timedelta_min'] = q_list[:, i]\n df['prior_lecture_' + str(i+1) + '_timedelta_min'] = l_list[:, i]\n df['prior_question_' + str(i+1) + '_timedelta_min'] = df['prior_question_' + str(i+1) + '_timedelta_min'].fillna(-100).replace(0, method='ffill')\n df['prior_lecture_' + str(i+1) + '_timedelta_min'] = df['prior_lecture_' + str(i+1) + '_timedelta_min'].fillna(-100).replace(0, method='ffill')\n \n del [q_list, l_list]\n return df\n\n# {user: [[теги 0-N], [количество вопросав по тегу], [количество правильных ответов по тегу]]}\n# для работы нужен подгруженный словарь dict_global_question_tag_accuracy\ndef user_question_tag_accuracy_get(df, feat_calc):\n # задаем веса для тегов\n tags_w = [0.43, 0.27, 0.18, 0.08, 0.03, 0.01] \n values = np.zeros(len(df))\n feat_list = ['user_id', 'content_type_id', 'tags_list']\n for cnt, (user, content_type_id, tags_list) in enumerate(df[feat_list].values):\n if tags_list == -100: tags_list = [-100]\n # если вопрос:\n if content_type_id == 0:\n # если юзер в словаре\n if user in feat_calc:\n user_tags_accuracy = 0\n # для каждого тега\n for tag_i in tags_list:\n tags_accuracy_list = []\n # если тег есть в словаре - забирем текущие значения\n if tag_i in feat_calc[user][0]:\n # находим индекс тега\n tag_i_idx = feat_calc[user][0].index(tag_i)\n # добавляем точность текущего тега по юзеру\n tags_accuracy_list.append(feat_calc[user][2][tag_i_idx] / feat_calc[user][1][tag_i_idx])\n # если тега нет в словаре\n else:\n # добавляем среднюю точность текущего тега из словаря\n tags_accuracy_list.append(dict_global_question_tag_accuracy[tag_i][2])\n # считаем общую точность по текущим тегам для юзеру\n l = len(tags_accuracy_list)\n tags_w_l = tags_w[:l]\n tags_w_l_sum = sum(tags_w_l)\n tags_w_current = [x/tags_w_l_sum for x in tags_w_l] \n user_tags_accuracy = sum([tag * w for tag, w in zip(tags_accuracy_list, tags_w_current)])\n\n # если юзера нет в словаре - добавляем\n else:\n # для каждого тега\n for tag_i in tags_list:\n tags_accuracy_list = []\n # добавляем среднюю точность текущего тега из словаря\n tags_accuracy_list.append(dict_global_question_tag_accuracy[tag_i][2])\n # считаем общую точность по текущим тегам для юзера\n l = len(tags_accuracy_list)\n tags_w_l = tags_w[:l]\n tags_w_l_sum = sum(tags_w_l)\n tags_w_current = [x/tags_w_l_sum for x in tags_w_l] \n user_tags_accuracy = sum([tag * w for tag, w in zip(tags_accuracy_list, tags_w_current)])\n\n values[cnt] = user_tags_accuracy\n # если лекция:\n else:\n values[cnt] = 0\n \n return values\n\ndef user_question_tag_accuracy_update(df, feat_calc):\n feat_list = ['user_id', 'answered_correctly', 'content_type_id', 'tags_list']\n for cnt, (user, answer, content_type_id, tags_list) in enumerate(df[feat_list].values):\n if tags_list == -100: tags_list = [-100]\n # если вопрос:\n if content_type_id == 0:\n # если юзер в словаре\n if user in feat_calc:\n user_tags_accuracy = 0\n # для каждого тега\n for tag_i in tags_list:\n tags_accuracy_list = []\n # если тег есть в словаре\n if tag_i in feat_calc[user][0]:\n # находим индекс тега\n tag_i_idx = feat_calc[user][0].index(tag_i)\n # обновляем словарь\n feat_calc[user][1][tag_i_idx] += 1\n feat_calc[user][2][tag_i_idx] += answer\n # если тега нет в словаре\n else:\n # добавляем тег в словарь\n feat_calc[user][0].append(tag_i)\n feat_calc[user][1].append(1)\n feat_calc[user][2].append(answer)\n\n # если юзера нет в словаре - добавляем\n else:\n feat_calc[user] = [[], [], []]\n # для каждого тега\n for tag_i in tags_list:\n feat_calc[user][0].append(tag_i)\n feat_calc[user][1].append(1)\n feat_calc[user][2].append(answer)\n \n return feat_calc\n\n\ndef user_correct_incorrect_timestamp_get(df, feat_dict):\n '''\n Время до последнего правильного и неправильного ответа\n Функция возвращает лист с данными\n '''\n incorrect_list = []\n correct_list = []\n for (user_id, timestamp, content_type_id) in df[['user_id', 'timestamp', 'content_type_id']].values:\n # переводим в минуты\n timestamp = timestamp / 1000 / 60\n correct_value, incorrect_value = 0, 0\n # если лекция\n if content_type_id: \n incorrect_list.append(-100)\n correct_list.append(-100)\n # если вопрос\n else:\n # если user_id есть в словаре - берем значения\n if user_id in feat_dict:\n # забираем текущие значения timestamp из словаря\n incorrect_value, correct_value = feat_dict[user_id][0], feat_dict[user_id][1]\n # если user_id нет\n else: \n # забираем np.nan\n incorrect_value, correct_value = np.nan, np.nan\n\n # считаем дельту \n incorrect_value = (timestamp - incorrect_value)\n correct_value = (timestamp - correct_value)\n # добавляем \n incorrect_list.append(incorrect_value)\n correct_list.append(correct_value)\n\n # добавляем в датафрейм\n df['prior_question_incorrect_timedelta_min'] = incorrect_list\n df['prior_question_correct_timedelta_min'] = correct_list\n # заполянем пропуски, заменяем 0 на предыдущее значение отличное от 0\n df['prior_question_incorrect_timedelta_min'] = df['prior_question_incorrect_timedelta_min'].fillna(-100).replace(0, method='ffill')\n df['prior_question_correct_timedelta_min'] = df['prior_question_correct_timedelta_min'].fillna(-100).replace(0, method='ffill')\n \n return df\n\ndef user_correct_incorrect_timestamp_update(df, feat_dict):\n '''\n Время до последнего правильного и неправильного ответа\n Функция возвращает лист с данными и обновляет словарь\n '''\n incorrect_list = []\n correct_list = []\n for (user_id, timestamp, content_type_id, answer) in df[['user_id', 'timestamp', 'content_type_id', 'answered_correctly']].values:\n # переводим в минуты\n timestamp = timestamp / 1000 / 60\n # если вопрос\n if content_type_id == 0: \n # если user_id есть в словаре - обновляем словарь\n if user_id in feat_dict:\n if answer: feat_dict[user_id][1] = timestamp # если ответ верный\n else: feat_dict[user_id][0] = timestamp # если ответ не верный\n # если user_id нет\n else: \n # добавляем юзера в словарь\n if answer: feat_dict[user_id] = [np.nan, timestamp] # если ответ верный\n else: feat_dict[user_id] = [timestamp, np.nan] # если ответ не верный\n\n return feat_dict\n\n\ntime_session_map = {'q_count_all' : 0,\n 'q_count_n' : 1,\n 'time_all_n' : 2,\n 'time_n' : 3,\n 'time_dict_all' : 4,\n 'time_dict_n' : 5,\n 'prior_timestamp' : 6,\n 'prior_container' : 7,\n 'prior_container_shape' : 8,\n 'prior_content_id' : 9,\n }\ndef user_slice_question_time_mean_session(df_pd, feat_dict, session_max_time_min = 180): \n prior_question_elapsed_time_mean = 25452.541\n out_mean_n = np.zeros(len(df_pd))\n out_mean_all = np.zeros(len(df_pd))\n out_delta_n = np.zeros(len(df_pd))\n out_delta_all = np.zeros(len(df_pd))\n \n calc_list = ['user_id', 'timestamp', 'task_container_id', 'content_id', \n 'content_type_id', 'prior_question_elapsed_time', 'task_container_freq']\n for cnt, (user, timestamp, task_container_id, content_id, content_type_id, prior_question_elapsed_time, task_container_freq) in enumerate(df_pd[calc_list].values):\n # время в минуты\n timestamp = timestamp / 1000 / 60\n if content_id not in dict_content_elapsed_time_mean: content_id = -100\n # если вопрос\n if content_type_id == 0:\n # если user_id есть в словаре\n if user in feat_dict: \n # считаем дельту timestamp\n time_delta = (timestamp - feat_dict[user][time_session_map['prior_timestamp']])\n if time_delta < session_max_time_min: \n # если новый контейнер\n if task_container_id != feat_dict[user][time_session_map['prior_container']]:\n s = feat_dict[user][time_session_map['prior_container_shape']]\n c = feat_dict[user][time_session_map['prior_content_id']]\n # обновляем словарь\n feat_dict[user][time_session_map['time_all_n']] += prior_question_elapsed_time * s\n feat_dict[user][time_session_map['time_n']] += prior_question_elapsed_time * s\n feat_dict[user][time_session_map['q_count_all']] += s\n feat_dict[user][time_session_map['q_count_n']] += s\n feat_dict[user][time_session_map['time_dict_all']] += dict_content_elapsed_time_mean[c] * s\n feat_dict[user][time_session_map['time_dict_n']] += dict_content_elapsed_time_mean[c] * s\n feat_dict[user][time_session_map['prior_timestamp']] = timestamp\n feat_dict[user][time_session_map['prior_container']] = task_container_id \n feat_dict[user][time_session_map['prior_content_id']] = content_id \n feat_dict[user][time_session_map['prior_container_shape']] = task_container_freq\n # считаем фичи \n out_mean_n[cnt] = feat_dict[user][time_session_map['time_n']] / feat_dict[user][time_session_map['q_count_n']]\n out_mean_all[cnt] = feat_dict[user][time_session_map['time_all_n']] / feat_dict[user][time_session_map['q_count_all']]\n out_delta_n[cnt] = feat_dict[user][time_session_map['time_n']] / feat_dict[user][time_session_map['time_dict_n']]\n out_delta_all[cnt] = feat_dict[user][time_session_map['time_all_n']] / feat_dict[user][time_session_map['time_dict_all']]\n\n # если контейнер прежний - берем предыдущее значение\n else:\n out_mean_n[cnt] = out_mean_n[cnt-1]\n out_mean_all[cnt] = out_mean_all[cnt-1]\n out_delta_n[cnt] = out_delta_n[cnt-1]\n out_delta_all[cnt] = out_delta_all[cnt-1]\n else: \n # если новый контейнер\n if task_container_id != feat_dict[user][time_session_map['prior_container']]:\n s = feat_dict[user][time_session_map['prior_container_shape']]\n c = feat_dict[user][time_session_map['prior_content_id']]\n # обновляем словарь\n feat_dict[user][time_session_map['time_all_n']] += prior_question_elapsed_time * s\n feat_dict[user][time_session_map['time_n']] = 0\n feat_dict[user][time_session_map['q_count_all']] += s\n feat_dict[user][time_session_map['q_count_n']] = 0\n feat_dict[user][time_session_map['time_dict_all']] += dict_content_elapsed_time_mean[c] * s\n feat_dict[user][time_session_map['time_dict_n']] = 0\n feat_dict[user][time_session_map['prior_timestamp']] = timestamp\n feat_dict[user][time_session_map['prior_container']] = task_container_id\n feat_dict[user][time_session_map['prior_content_id']] = content_id \n feat_dict[user][time_session_map['prior_container_shape']] = task_container_freq\n # считаем фичи \n out_mean_n[cnt] = feat_dict[user][time_session_map['time_all_n']] / feat_dict[user][time_session_map['q_count_all']]\n out_mean_all[cnt] = feat_dict[user][time_session_map['time_all_n']] / feat_dict[user][time_session_map['q_count_all']]\n out_delta_n[cnt] = feat_dict[user][time_session_map['time_all_n']] / feat_dict[user][time_session_map['time_dict_all']]\n out_delta_all[cnt] = feat_dict[user][time_session_map['time_all_n']] / feat_dict[user][time_session_map['time_dict_all']]\n\n # если контейнер прежний - берем предыдущее значение\n else:\n out_mean_n[cnt] = out_mean_n[cnt-1]\n out_mean_all[cnt] = out_mean_all[cnt-1]\n out_delta_n[cnt] = out_delta_n[cnt-1]\n out_delta_all[cnt] = out_delta_all[cnt-1]\n # если user_id нет в словаре\n else:\n out_mean_n[cnt] = prior_question_elapsed_time_mean\n out_mean_all[cnt] = prior_question_elapsed_time_mean\n out_delta_n[cnt] = prior_question_elapsed_time_mean / dict_content_elapsed_time_mean[content_id] \n out_delta_all[cnt] = prior_question_elapsed_time_mean / dict_content_elapsed_time_mean[content_id] \n feat_dict[user] = [1, 1, 0, 0, \n dict_content_elapsed_time_mean[content_id], dict_content_elapsed_time_mean[content_id], \n timestamp, task_container_id, task_container_freq, content_id]\n # если лекция\n else: out_mean_n[cnt], out_mean_all[cnt], out_delta_n[cnt], out_delta_all[cnt] = 0, 0, 0, 0\n \n df_pd['user_question_time_mean_n_session'] = out_mean_n\n df_pd['user_question_time_mean_all_session'] = out_mean_all\n df_pd['user_question_time_delta_n_session'] = out_delta_n\n df_pd['user_question_time_delta_all_session'] = out_delta_all\n return df_pd\n\n\n# prior_question_had_explanation counter features\nuser_priorq_expl_types_map = {'q_count' : 0,\n 'c_t_cnt' : 1,\n 'c_f_cnt' : 2,\n 'w_t_cnt' : 3,\n 'w_f_cnt' : 4,\n 'prior_container' : 5,\n 'prior_container_shape' : 6,\n 'prior_answer_expl_type' : 7,\n }\n\ndef user_priorq_expl_types_get(df_pd, feat_dict): \n c_t_cnt = np.zeros(len(df_pd))\n c_f_cnt = np.zeros(len(df_pd))\n w_t_cnt = np.zeros(len(df_pd))\n w_f_cnt = np.zeros(len(df_pd))\n cw_tf_type = np.zeros(len(df_pd))\n \n calc_list = ['user_id', 'task_container_id', 'content_type_id', \n 'prior_question_had_explanation', 'task_container_freq']\n for cnt, (user, task_container_id, content_type_id, priorq_had_expl, task_container_freq) in enumerate(df_pd[calc_list].values):\n # если вопрос\n if content_type_id == 0:\n # если user_id есть в словаре\n if user in feat_dict:\n # если контейнер новый\n if task_container_id != feat_dict[user][user_priorq_expl_types_map['prior_container']]:\n # количество вопросов в предыдущем контейнере \n prior_c_shape = feat_dict[user][user_priorq_expl_types_map['prior_container_shape']]\n # общее количество вопросов\n q_count = feat_dict[user][user_priorq_expl_types_map['q_count']] + 0.0001\n # 1.1. возвращаем: c_t_cnt/cnt, c_f_cnt/cnt, w_t_cnt/cnt, w_f_cnt/cnt, cw_tf_type\n c_t_cnt[cnt] = feat_dict[user][user_priorq_expl_types_map['c_t_cnt']] / q_count\n c_f_cnt[cnt] = feat_dict[user][user_priorq_expl_types_map['c_f_cnt']] / q_count\n w_t_cnt[cnt] = feat_dict[user][user_priorq_expl_types_map['w_t_cnt']] / q_count\n w_f_cnt[cnt] = feat_dict[user][user_priorq_expl_types_map['w_f_cnt']] / q_count\n cw_tf_type[cnt] = feat_dict[user][user_priorq_expl_types_map['prior_answer_expl_type']]\n # если контейнер старый\n else:\n # 2.1. возвращаем: предыдущие значения (счетчик -1)\n c_t_cnt[cnt] = c_t_cnt[cnt - 1]\n c_f_cnt[cnt] = c_f_cnt[cnt - 1]\n w_t_cnt[cnt] = w_t_cnt[cnt - 1]\n w_f_cnt[cnt] = w_f_cnt[cnt - 1]\n cw_tf_type[cnt] = cw_tf_type[cnt - 1]\n # если user_id нет в словаре\n else:\n c_t_cnt[cnt], c_f_cnt[cnt], w_t_cnt[cnt], w_f_cnt[cnt], w_f_cnt[cnt] = 0, 0, 0, 0, 0\n\n # если лекция\n else: c_t_cnt[cnt], c_f_cnt[cnt], w_t_cnt[cnt], w_f_cnt[cnt], cw_tf_type[cnt] = 0, 0, 0, 0, 0\n \n df_pd['user_prior_correct_expl_prc'] = c_t_cnt\n df_pd['user_prior_correct_noexpl_prc'] = c_f_cnt\n df_pd['user_prior_wrong_expl_prc'] = w_t_cnt\n df_pd['user_prior_wrong_noexpl_prc'] = w_f_cnt\n df_pd['user_prior_answer_expl_type'] = cw_tf_type\n\n return df_pd\n\ndef user_priorq_expl_types_update(df_pd, feat_dict): \n calc_list = ['user_id', 'task_container_id', 'content_type_id', \n 'prior_question_had_explanation', 'answered_correctly', 'task_container_freq']\n for cnt, (user, task_container_id, content_type_id, priorq_had_expl, answer, task_container_freq) in enumerate(df_pd[calc_list].values):\n if answer: t1 = 'c_'\n else: t1 = 'w_'\n if priorq_had_expl: t2 = 't_cnt'\n else: t2 = 'f_cnt'\n col = t1 + t2\n # если вопрос\n if content_type_id == 0:\n # если user_id есть в словаре\n if user in feat_dict:\n # если контейнер новый\n if task_container_id != feat_dict[user][user_priorq_expl_types_map['prior_container']]:\n # количество вопросов в предыдущем контейнере \n prior_c_shape = feat_dict[user][user_priorq_expl_types_map['prior_container_shape']]\n # 1.2. обновляем словарь: q_count += prior_c_shape\n feat_dict[user][user_priorq_expl_types_map['q_count']] += prior_c_shape\n # в зависимости от ответа и флага обновляем счетчик c/w_t/f_cnt += prior_c_shape \n feat_dict[user][user_priorq_expl_types_map[col]] += prior_c_shape\n # 2.3. обновляем словарь: контейнер, количество вопросов в предыдущем контейнере\n feat_dict[user][user_priorq_expl_types_map['prior_container']] = task_container_id\n feat_dict[user][user_priorq_expl_types_map['prior_container_shape']] = task_container_freq\n feat_dict[user][user_priorq_expl_types_map['prior_answer_expl_type']] = user_priorq_expl_types_map[col]\n\n # если user_id нет в словаре\n else:\n feat_dict[user] = [0, 0, 0, 0, 0, \n task_container_id, task_container_freq, 0]\n\n return feat_dict\n\n\n# функция обновления словаря - возвращает датафрейм с новым значением\ndef user_lectures_typeof_cnt(df, feat_dict):\n concept = np.zeros(len(df))\n solving_question = np.zeros(len(df))\n for cnt, (user, content_id, content_type_id) in enumerate(df[['user_id', 'content_id', 'content_type_id']].values):\n # если лекция\n if content_type_id:\n # если user есть в словаре\n if user in feat_dict:\n # если content_id есть в словаре dict_lectures\n if content_id in dict_lectures:\n # обновляем словарь\n if dict_lectures[content_id]['type_of'] == 'concept': feat_dict[user][0] += 1\n elif dict_lectures[content_id]['type_of'] == 'solving question': feat_dict[user][1] += 1\n # если user нет в словаре\n else:\n # если content_id есть в словаре dict_lectures\n if content_id in dict_lectures:\n # создаем юзера\n feat_dict[user] = [0, 0]\n # обновляем словарь\n if dict_lectures[content_id]['type_of'] == 'concept': feat_dict[user][0] += 1\n elif dict_lectures[content_id]['type_of'] == 'solving question': feat_dict[user][1] += 1\n # возвращаем\n concept[cnt] = 0\n solving_question[cnt] = 0\n # если вопрос\n else:\n # если user есть в словаре\n if user in feat_dict:\n # возвращаем\n concept[cnt] = feat_dict[user][0]\n solving_question[cnt] = feat_dict[user][1]\n # если user есть в словаре\n else:\n # возвращаем\n concept[cnt] = 0\n solving_question[cnt] = 0\n \n df['lecture_concept_cnt'] = concept.astype(np.uint16)\n df['lecture_solving_question_cnt'] = solving_question.astype(np.uint16)\n return df\n\ndef user_answer_mode_n_get(df_pd, feat_dict, border=10): \n current_list = np.zeros(len(df_pd), dtype = np.uint8)\n for cnt, (user, content_type_id, content_id) in enumerate(df_pd[['user_id', 'content_type_id', 'content_id']].values): \n # если вопрос\n if content_type_id == 0:\n # если user_id есть в словаре\n if user in feat_dict:\n # если content_id есть в словаре dict_questions\n if content_id in dict_questions:\n current_list[cnt] = max(set(feat_dict[user]), key=feat_dict[user].count) == dict_questions[content_id]['correct_answer']\n # если content_id нет в словаре dict_questions\n else:\n current_list[cnt] = 100\n else: # если user_id нет в словаре\n current_list[cnt] = 100\n else:\n current_list[cnt] = 0\n \n return current_list\n\ndef user_answer_mode_n_update(df_pd, feat_dict, border=10): \n for cnt, (user, user_answer, content_type_id, content_id) in enumerate(df_pd[['user_id', 'user_answer', 'content_type_id', 'content_id']].values): \n # если вопрос\n if content_type_id == 0:\n # обновляем словарь\n feat_dict[user].append(user_answer) # добавляем текущий ответ\n feat_dict[user] = feat_dict[user][-border:] # ограничиваем количество\n \n return feat_dict\n\n\n# создаем map по самым крупным стартовым bundle_id\nfirst_bundle_id_map = {7900 : 1,\n 128 : 2,\n 5692 : 3,\n -100 : 4,\n }\n\n# тянем bundle_id\ndef question_bundle_id_get(df_pd, feat_dict): \n current_list = np.zeros(len(df_pd))\n for cnt, (content_type_id, content_id) in enumerate(df_pd[['content_type_id', 'content_id']].values):\n # если вопрос\n if content_type_id == 0: \n # если content_id есть в словаре dict_questions\n if content_id in feat_dict:\n current_list[cnt] = feat_dict[content_id]['bundle_id']\n # если content_id нет в словаре dict_questions\n else:\n current_list[cnt] = -100\n # если лекция\n else: \n current_list[cnt] = 0\n df_pd['bundle_id'] = current_list.astype(np.int32)\n return df_pd\n\n# функция расчета переменной и обновления словаря {user : cluster} - кластер юзера по первому bundle_id\ndef user_bundle_cluster_get_update(df_pd, feat_dict): \n l = ['user_id', 'content_type_id', 'bundle_id', 'timestamp']\n current_list = np.zeros(len(df_pd))\n for cnt, (user, content_type_id, bundle, timestamp) in enumerate(df_pd[l].values): \n # если user нет в словаре - обновляем словарь\n if user not in feat_dict: \n # если вопрос и он первый для юзера\n if content_type_id == 0 and timestamp == 0: \n # если bundle есть в first_bundle_id_map\n if bundle in first_bundle_id_map:\n # добавляем кластер\n feat_dict[user] = first_bundle_id_map[bundle]\n # если bundle нет в first_bundle_id_map\n else: feat_dict[user] = 4 \n # иначе добавляем с 4\n else: feat_dict[user] = 4 \n\n # возвращаем данные\n current_list[cnt] = feat_dict[user]\n \n df_pd['first_bundle_id_cluster'] = current_list.astype(np.uint8)\n return df_pd\n\n# функция обновления словаря - точность общая и по bundle_id для кластера\n# {bundle_id : [[all_cnt, all_corr], [cl_1_cnt, cl_1_corr], [cl_2_cnt, cl_2_corr], [cl_3_cnt, cl_3_corr], [cl_4_cnt, cl_4_corr]]}\ndef question_bundle_accuracy_update(df_pd, feat_dict): \n l = ['content_type_id', 'bundle_id', 'answered_correctly', 'first_bundle_id_cluster']\n for cnt, (content_type_id, bundle_id, answer, cluster) in enumerate(df_pd[l].values):\n # если вопрос\n if content_type_id == 0: \n # если bundle_id есть в словаре\n if bundle_id in feat_dict:\n # обновляем словарь\n feat_dict[bundle_id][0][0] += 1\n feat_dict[bundle_id][0][1] += answer\n feat_dict[bundle_id][cluster][0] += 1\n feat_dict[bundle_id][cluster][1] += answer\n # если bundle_id нет в словаре - добавляем с кластером = 4\n else:\n feat_dict[bundle_id] = [[1, answer], [0, 0], [0, 0], [0, 0], [1, answer]]\n \n return feat_dict\n\n# тянем точность по bundle_id в train из словаря dict_question_bundle_accuracy\ndef question_bundle_accuracy_get(df_pd, feat_dict): \n current_list = np.zeros((len(df_pd), 2), dtype = np.float32)\n l = ['content_type_id', 'bundle_id', 'first_bundle_id_cluster']\n for cnt, (content_type_id, bundle_id, cluster) in enumerate(df_pd[l].values):\n # если вопрос\n if content_type_id == 0: \n # если bundle_id есть в словаре\n if bundle_id in feat_dict: \n # возвращаем значения\n current_list[cnt, 0] = feat_dict[bundle_id][0][1] / (feat_dict[bundle_id][0][0] + 0.000001)\n current_list[cnt, 1] = feat_dict[bundle_id][cluster][1] / (feat_dict[bundle_id][cluster][0] + 0.000001)\n # если bundle_id нет в словаре\n else:\n current_list[cnt, 0] = 0.67\n current_list[cnt, 1] = 0.67\n\n # если лекция\n else: \n current_list[cnt, 0] = 0\n current_list[cnt, 1] = 0\n \n df_pd['bundle_id_all_accuracy'] = current_list[:,0].astype(np.float32)\n df_pd['bundle_id_cluster_accuracy'] = current_list[:,1].astype(np.float32)\n\n return df_pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# user_question_part_accuracy - точность ответа в разрезе part вопроса по юзеру \nquestion_part_map = {\n 'part_-100_count' : 0,\n 'part_-100_count_correct' : 1,\n 'part_-100_accuracy' : 2,\n \n 'part_1_count' : 3,\n 'part_1_count_correct' : 4,\n 'part_1_accuracy' : 5,\n \n 'part_2_count' : 6,\n 'part_2_count_correct' : 7,\n 'part_2_accuracy' : 8,\n \n 'part_3_count' : 9,\n 'part_3_count_correct' : 10,\n 'part_3_accuracy' : 11,\n \n 'part_4_count' : 12,\n 'part_4_count_correct' : 13,\n 'part_4_accuracy' : 14,\n \n 'part_5_count' : 15,\n 'part_5_count_correct' : 16,\n 'part_5_accuracy' : 17,\n \n 'part_6_count' : 18,\n 'part_6_count_correct' : 19,\n 'part_6_accuracy' : 20,\n \n 'part_7_count' : 21,\n 'part_7_count_correct' : 22,\n 'part_7_accuracy' : 23\n}\nquestion_part_def_accuracy = {\n 1 : 0.745,\n 2 : 0.709,\n 3 : 0.701,\n 4 : 0.631,\n 5 : 0.610,\n 6 : 0.669,\n 7 : 0.660,\n -100 : 0.6,\n}\n\n\ndef dict_user_question_part_accuracy_get(df_pd, feat_dict):\n current_list = np.zeros(len(df_pd))\n for idx, (user, part_q, content_type_id) in enumerate(df_pd[['user_id', 'part_q', 'content_type_id']].values):\n part_q = int(part_q) \n # если вопрос\n if content_type_id == 0:\n map_accuracy = question_part_map['part_' + str(part_q) + '_accuracy']\n # если юзер есть в словаре - берем из словаря\n if user in feat_dict:\n current_list[idx] = feat_dict[user][map_accuracy]\n else: # если user нет в словаре - берем дефолт из словаря дефолта по part_q\n current_list[idx] = question_part_def_accuracy[part_q] \n # если лекция - ставим 0\n else:\n current_list[idx] = 0\n\n return current_list\n\ndef dict_user_question_part_accuracy_update(df_pd, feat_dict, trust_border=10):\n\n for _, (user, part_q, content_type_id, ans_corr) in enumerate(df_pd[['user_id', 'part_q', 'content_type_id', 'answered_correctly']].values):\n part_q = int(part_q)\n # если вопрос\n if content_type_id == 0:\n \n map_cnt = question_part_map['part_' + str(part_q) + '_count']\n map_cnt_correct = question_part_map['part_' + str(part_q) + '_count_correct']\n map_accuracy = question_part_map['part_' + str(part_q) + '_accuracy']\n \n # если юзер есть в словаре\n if user in feat_dict:\n feat_dict[user][map_cnt] += 1\n feat_dict[user][map_cnt_correct] += ans_corr\n feat_dict[user][map_accuracy] = feat_dict[user][map_cnt_correct] / feat_dict[user][map_cnt]\n # если количество вопросов меньше trust_border - сглаживаем средним по part\n if feat_dict[user][map_cnt] < trust_border:\n feat_dict[user][map_accuracy] = ((feat_dict[user][map_accuracy] * feat_dict[user][map_cnt] +\n question_part_def_accuracy[part_q] * (trust_border - feat_dict[user][map_cnt])) / trust_border)\n\n else: # если user нет в словаре - добавляем\n feat_dict[user] = [0] * len(question_part_map)\n for i in range(1, 7):\n feat_dict[user][question_part_map['part_' + str(i) + '_accuracy']] = question_part_def_accuracy[i]\n feat_dict[user][map_cnt] += 1\n feat_dict[user][map_cnt_correct] += ans_corr\n feat_dict[user][map_accuracy] = feat_dict[user][map_cnt_correct] / feat_dict[user][map_cnt]\n # сглаживаем средним по part\n feat_dict[user][map_accuracy] = ((feat_dict[user][map_accuracy] * feat_dict[user][map_cnt] +\n question_part_def_accuracy[part_q] * (trust_border - feat_dict[user][map_cnt])) / trust_border)\n\n return feat_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_obj(name ):\n with open('../input/riiid-numpy-df-3/' + name + '.pkl', 'rb') as f:\n return pickle.load(f)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load boostings"},{"metadata":{"trusted":true},"cell_type":"code","source":"from catboost import CatBoostClassifier\n\ncat_model = CatBoostClassifier()\ncat_model.load_model('../input/riiid-lgb-v1/cat_arvis_v4.cbm')\n\nimport lightgbm as lgb\nlgb_model = lgb.Booster(model_file='../input/riiid-lgb-v1/model_lgb_7946_v8_full_data_arvis.txt')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Some dicts"},{"metadata":{"trusted":true},"cell_type":"code","source":"# импорт словарей\ndict_lectures = load_obj('dict_lectures')\ndict_questions = load_obj('dict_questions')\n\ndict_question_user_cnt = load_obj('dict_question_user_cnt')\ndict_correct_answers_user_cnt = load_obj('dict_correct_answers_user_cnt')\ndict_question_explonation_user_cnt = load_obj('dict_question_explonation_user_cnt')\ndict_questionid_part_tag12_avgtarget = load_obj('dict_questionid_part_tag12_avgtarget_5')\n\ndict_user_question_attempt_cnt = load_obj('dict_user_question_attempt_cnt')\ndict_user_lectures_part = load_obj('dict_user_lectures_part')\ndict_user_question_part_accuracy = load_obj('dict_user_question_part_accuracy')\ndict_user_l_q_tag_equal = load_obj('dict_user_l_q_tag_equal')\n\ndict_user_slice_accuracy_5 = load_obj('dict_user_slice_accuracy_5')\ndict_user_slice_accuracy_20 = load_obj('dict_user_slice_accuracy_20')\ndict_user_slice_accuracy_50 = load_obj('dict_user_slice_accuracy_50')\ndict_user_slice_accuracy_session_3 = load_obj('dict_user_slice_accuracy_session_3')\ndict_user_slice_accuracy_session_12 = load_obj('dict_user_slice_accuracy_session_12')\ndict_user_slice_accuracy_session_48 = load_obj('dict_user_slice_accuracy_session_48')\ndict_user_timestampsdelta_3 = load_obj('dict_user_timestampsdelta_3')\n\ndict_global_question_tag_accuracy = load_obj('dict_global_question_tag_accuracy')\ndict_global_question_tag_accuracy[-100] = 0.64\ndict_user_question_tag_accuracy = load_obj('dict_user_question_tag_accuracy')\n\ndict_user_correct_incorrect_timestamp = load_obj('dict_user_correct_incorrect_timestamp')\ndict_content_elapsed_time_mean = load_obj('dict_content_elapsed_time_mean')\ndict_user_slice_question_time_mean_session = load_obj('dict_user_slice_question_time_mean_session')\n\ndict_user_priorq_expl_types = load_obj('dict_user_priorq_expl_types')\n\ndict_user_lectures_typeof_cnt = load_obj('dict_user_lectures_typeof_cnt')\ndict_user_answer_mode_10 = load_obj('dict_user_answer_mode_10')\ndict_user_answer_mode_50 = load_obj('dict_user_answer_mode_50')\n\ndict_question_bundle_accuracy = load_obj('dict_question_bundle_accuracy')\ndict_user_bundle_cluster = load_obj('dict_user_bundle_cluster')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* 0 - row_id\n* 1 - timestamp\n* 2 - user_id\n* 3 - content_id\n* 4 - content_type_id\n* 5 - task_container_id\n* 6 - prior_question_elapsed_time\n* 7 - prior_question_had_explanation\n* 8 - prior_group_answers_correct\n* 9 - prior_group_responses"},{"metadata":{"trusted":true},"cell_type":"code","source":"features_map = {# input\n 'row_id' : 0,\n 'timestamp' : 1,\n 'user_id' : 2,\n 'content_id' : 3,\n 'content_type_id' : 4,\n 'task_container_id' : 5,\n 'prior_question_elapsed_time' : 6,\n 'prior_question_had_explanation' : 7,\n 'prior_group_answers_correct' : 8,\n 'prior_group_responses' : 9,\n # 1. dict_user_timestampsdelta_3\n 'prior_question_1_timedelta_min' : 10,\n 'prior_lecture_1_timedelta_min' : 11,\n 'prior_question_2_timedelta_min' : 12,\n 'prior_lecture_2_timedelta_min' : 13,\n 'prior_question_3_timedelta_min' : 14,\n 'prior_lecture_3_timedelta_min' : 15,\n # 2. скользящая точность по user_id\n 'user_slice_accuracy_5' : 16,\n 'user_slice_accuracy_20' : 17,\n 'user_slice_accuracy_50' : 18, \n 'user_slice_accuracy_session_3' : 19,\n 'user_slice_accuracy_session_12' : 20,\n 'user_slice_accuracy_session_48' : 21,\n # 3.1. количество вопросов в группе \n 'task_container_freq' : 22,\n # 3.2. порядковый номер вопроса в группе\n 'task_container_counter' : 23,\n # 4. количество вопросов по content_id на которые отвечал каждый user_id накопленно (добавляем + обновляем словарь) \n 'user_question_attempt_cnt' : 24,\n # 5. количество лекций по пользователю накопленно\n 'lecture_cnt' : 25,\n 'lecture_concept_cnt' : 26,\n 'lecture_solving_question_cnt' : 27,\n # 6. сколько лекций с part равным part вопроса пользователь прослушал до вопроса (добавляем + обновляем словарь)\n 'part_l_q_cnt' : 28,\n # 7. слушал ли юзер лекцию по тегу равную тегу вопроса (количество) (добавляем + обновляем словарь)\n 'tag_l_q_equal_cnt' : 29,\n # 8. точность ответа в разрезе part вопроса по юзеру (добавляем)\n 'user_question_part_accuracy' : 30, \n # 9. точность ответа в разрезе tag вопроса по юзеру (добавляем)\n 'user_question_tag_accuracy' : 31, \n # 10. время до последнего правильного и неправильного ответа\n 'prior_question_incorrect_timedelta_min' : 32,\n 'prior_question_correct_timedelta_min' : 33,\n # 11. среднее время ответа на вопросы за последние N минут; отношение к среднему по content_id\n 'user_question_time_mean_n_session' : 34,\n 'user_question_time_mean_all_session' : 35,\n 'user_question_time_delta_n_session' : 36,\n 'user_question_time_delta_all_session' : 37,\n # 12 prior_question_had_explanation counter features\n 'user_prior_correct_expl_prc' : 38,\n 'user_prior_correct_noexpl_prc' : 39,\n 'user_prior_wrong_expl_prc' : 40,\n 'user_prior_wrong_noexpl_prc' : 41,\n 'user_prior_answer_expl_type' : 42,\n # 13 мода ответа на последние N вопросов == ответу на вопрос\n 'user_answer_mode_10' : 43,\n 'user_answer_mode_50' : 44,\n # 14 точность по bundle_id вопроса\n 'first_bundle_id_cluster' : 45,\n 'bundle_id_all_accuracy' : 46,\n 'bundle_id_cluster_accuracy' : 47,\n # 15. количество вопросов по пользователю накопленно (добавляем из словаря) \n 'question_user_cnt' : 48,\n # 16. количество правильных ответов по пользователю накопленно (добавляем из словаря)\n 'correct_answers_user_cnt' : 49,\n # 17. доля правильных ответов по пользователю накопленно = correct_answers_user_cnt / question_user_cnt\n 'correct_answers_user_prc' : 50,\n # 18. количество вопросов с объяснением по пользователю накопленно (добавляем из словаря)\n 'prior_question_had_explanation_user_cnt' : 51,\n # 19. доля вопросов с объяснением по пользователю накопленно\n 'prior_question_had_explanation_user_prc' : 52,\n # 20. {question_id : [content_id_mean, part, part_mean, tag_1_mean, tag_2_mean, part_tag_1_mean, part_tag_2_mean]} \n 'content_id_mean' : 53,\n 'part' : 54,\n 'part_mean' : 55,\n 'tag_1_mean' : 56,\n 'tag_2_mean' : 57,\n# 'part_tag_1_mean' : 35,\n# 'part_tag_2_mean' : 36, \n# 'tags_encoded' : 37, \n # 21. делим точность пользователя на точность вопроса\n 'user_to_question_accuracy' : 58, \n # 22. среднее гармоническое из точности пользователя и точности вопроса\n 'hmean_user_content_accuracy' : 59 \n }\n\ndict_questionid_part_tag12_avgtarget_map = {\n 'content_id_cnt' : 0,\n 'content_correct_cnt' : 1,\n \n 'answered_correctly_avg_content_smooth' : 2,\n 'part' : 3,\n 'answered_correctly_avg_part' : 4,\n 'answered_correctly_avg_tag_1' : 5,\n 'answered_correctly_avg_tag_2' : 6,\n 'answered_correctly_avg_part_tag_1' : 7,\n 'answered_correctly_avg_part_tag_2' : 8,\n 'tags_encoded' : 9 \n}\n\ntrain_cols_clf = [features_map['prior_question_1_timedelta_min'],\n features_map['prior_question_2_timedelta_min'],\n features_map['prior_question_3_timedelta_min'],\n features_map['prior_lecture_1_timedelta_min'],\n features_map['prior_lecture_2_timedelta_min'],\n features_map['prior_lecture_3_timedelta_min'],\n features_map['task_container_freq'],\n features_map['task_container_counter'], \n features_map['user_question_attempt_cnt'],\n \n # features_2\n features_map['prior_question_elapsed_time'],\n features_map['prior_question_had_explanation'],\n features_map['question_user_cnt'],\n features_map['correct_answers_user_cnt'],\n features_map['correct_answers_user_prc'],\n features_map['prior_question_had_explanation_user_cnt'],\n features_map['prior_question_had_explanation_user_prc'],\n \n # features_3\n features_map['user_slice_accuracy_5'],\n features_map['user_slice_accuracy_20'],\n features_map['user_slice_accuracy_50'],\n \n features_map['user_slice_accuracy_session_3'],\n features_map['user_slice_accuracy_session_12'],\n features_map['user_slice_accuracy_session_48'],\n \n # features_4\n features_map['lecture_cnt'],\n features_map['lecture_concept_cnt'],\n features_map['lecture_solving_question_cnt'],\n features_map['part_l_q_cnt'],\n features_map['tag_l_q_equal_cnt'], \n \n # features_5\n features_map['user_question_part_accuracy'],\n \n # features_6\n features_map['user_question_tag_accuracy'],\n \n # features_8\n features_map['prior_question_incorrect_timedelta_min'],\n features_map['prior_question_correct_timedelta_min'],\n features_map['user_question_time_mean_n_session'],\n features_map['user_question_time_mean_all_session'],\n features_map['user_question_time_delta_n_session'],\n features_map['user_question_time_delta_all_session'],\n \n # features_9\n features_map['user_prior_correct_expl_prc'],\n features_map['user_prior_correct_noexpl_prc'],\n features_map['user_prior_wrong_expl_prc'],\n features_map['user_prior_wrong_noexpl_prc'],\n features_map['user_prior_answer_expl_type'],\n \n # features_10\n features_map['user_answer_mode_10'],\n features_map['user_answer_mode_50'], \n \n # features_11\n features_map['first_bundle_id_cluster'],\n features_map['bundle_id_all_accuracy'],\n features_map['bundle_id_cluster_accuracy'],\n \n # dict\n features_map['content_id_mean'],\n features_map['part'],\n features_map['part_mean'],\n features_map['tag_1_mean'],\n features_map['tag_2_mean'],\n #features_map['part_tag_1_mean'],\n #features_map['part_tag_2_mean'],\n #features_map['tags_encoded'],\n \n # calc\n features_map['user_to_question_accuracy'], \n features_map['hmean_user_content_accuracy'], \n \n ]\n\ntrain_cols_lgb = [features_map['prior_question_1_timedelta_min'],\n features_map['prior_question_2_timedelta_min'],\n features_map['prior_question_3_timedelta_min'],\n features_map['prior_lecture_1_timedelta_min'],\n features_map['prior_lecture_2_timedelta_min'],\n features_map['prior_lecture_3_timedelta_min'],\n features_map['task_container_freq'],\n features_map['task_container_counter'], \n features_map['user_question_attempt_cnt'],\n \n # features_2\n features_map['prior_question_elapsed_time'],\n features_map['prior_question_had_explanation'],\n features_map['question_user_cnt'],\n features_map['correct_answers_user_cnt'],\n features_map['correct_answers_user_prc'],\n features_map['prior_question_had_explanation_user_cnt'],\n features_map['prior_question_had_explanation_user_prc'],\n \n # features_3\n features_map['user_slice_accuracy_5'],\n features_map['user_slice_accuracy_20'],\n features_map['user_slice_accuracy_50'],\n \n features_map['user_slice_accuracy_session_3'],\n features_map['user_slice_accuracy_session_12'],\n features_map['user_slice_accuracy_session_48'],\n \n # features_4\n features_map['lecture_cnt'],\n features_map['lecture_concept_cnt'],\n features_map['lecture_solving_question_cnt'],\n features_map['part_l_q_cnt'],\n features_map['tag_l_q_equal_cnt'], \n \n # features_5\n features_map['user_question_part_accuracy'],\n \n # features_6\n features_map['user_question_tag_accuracy'],\n \n # features_8\n features_map['prior_question_incorrect_timedelta_min'],\n features_map['prior_question_correct_timedelta_min'],\n features_map['user_question_time_mean_n_session'],\n features_map['user_question_time_mean_all_session'],\n features_map['user_question_time_delta_n_session'],\n features_map['user_question_time_delta_all_session'],\n \n # features_9\n features_map['user_prior_correct_expl_prc'],\n features_map['user_prior_correct_noexpl_prc'],\n features_map['user_prior_wrong_expl_prc'],\n features_map['user_prior_wrong_noexpl_prc'],\n features_map['user_prior_answer_expl_type'],\n \n # features_10\n features_map['user_answer_mode_10'],\n features_map['user_answer_mode_50'], \n \n # features_11\n features_map['first_bundle_id_cluster'],\n features_map['bundle_id_all_accuracy'],\n features_map['bundle_id_cluster_accuracy'],\n \n # dict\n features_map['content_id_mean'],\n features_map['part'],\n features_map['part_mean'],\n features_map['tag_1_mean'],\n features_map['tag_2_mean'],\n #features_map['part_tag_1_mean'],\n #features_map['part_tag_2_mean'],\n #features_map['tags_encoded'],\n \n # calc\n features_map['user_to_question_accuracy'], \n features_map['hmean_user_content_accuracy'], \n \n ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"idx = 86867031","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\nenv = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for boosting\nprevious_test_df = pd.DataFrame()\n\n# for saint\nprev_test_df = None\n\nfor (test_df, sample_prediction_df) in iter_test:\n # make a copy, because preprocessing could be different\n test_df_saint = test_df.copy()\n ## SAINT PART\n if (prev_test_df is not None) & (psutil.virtual_memory().percent < 90):\n prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])\n prev_test_df = prev_test_df[prev_test_df.content_type_id == False]\n \n ## lag time\n prev_test_df = feature_time_lag(prev_test_df, time_dict)\n\n prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly', 'part', 'prior_question_elapsed_time', 'time_lag', 'prior_question_had_explanation']].groupby('user_id').apply(lambda r: (\n r['content_id'].values,\n r['answered_correctly'].values,\n r['part'].values,\n r['prior_question_elapsed_time'].values,\n r['time_lag'].values,\n r['prior_question_had_explanation'].values))\n \n for prev_user_id in prev_group.index:\n if prev_user_id in group.index:\n group[prev_user_id] = (\n np.append(group[prev_user_id][0], prev_group[prev_user_id][0])[-MAX_SEQ:], \n np.append(group[prev_user_id][1], prev_group[prev_user_id][1])[-MAX_SEQ:],\n np.append(group[prev_user_id][2], prev_group[prev_user_id][2])[-MAX_SEQ:],\n np.append(group[prev_user_id][3], prev_group[prev_user_id][3])[-MAX_SEQ:],\n np.append(group[prev_user_id][4], prev_group[prev_user_id][4])[-MAX_SEQ:],\n np.append(group[prev_user_id][5], prev_group[prev_user_id][5])[-MAX_SEQ:]\n )\n \n else:\n group[prev_user_id] = (\n prev_group[prev_user_id][0], \n prev_group[prev_user_id][1],\n prev_group[prev_user_id][2],\n prev_group[prev_user_id][3],\n prev_group[prev_user_id][4],\n prev_group[prev_user_id][5]\n )\n\n \n ## elapsed time\n test_df_saint.prior_question_elapsed_time = test_df_saint.prior_question_elapsed_time.fillna(0)\n \n ## prior_question_had_explanation\n test_df_saint['prior_question_had_explanation'] = test_df_saint['prior_question_had_explanation'].fillna(value = False).astype(int)\n \n test_df_saint = test_df_saint.merge(questions_df[[\"question_id\",\"part\"]], how = \"left\",left_on = 'content_id', right_on = 'question_id') \n \n prev_test_df = test_df_saint.copy()\n \n ## drop lecture\n test_df_saint = test_df_saint[test_df_saint.content_type_id == False]\n \n \n ## lag time\n test_df_saint = feature_time_lag(test_df_saint, time_dict)\n \n test_dataset = TestDataset(group, test_df_saint, n_skill)\n test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)\n \n outs1 = [] \n outs2 = []\n outs3 = []\n \n for item in test_dataloader:\n exercise = item[0].to(device).long()\n part = item[1].to(device).long()\n response = item[2].to(device).long()\n elapsed_time = item[3].to(device).long()\n lag_time = item[4].to(device).long()\n pri_exp = item[5].to(device).long()\n \n with torch.no_grad():\n output1 = model1(exercise, part, response, elapsed_time, lag_time, pri_exp)\n output2 = model2(exercise, part, response, elapsed_time, lag_time, pri_exp)\n output3 = model3(exercise, part, response, elapsed_time, lag_time, pri_exp)\n outs1.extend(torch.sigmoid(output1)[:, -1].view(-1).data.cpu().numpy())\n outs2.extend(torch.sigmoid(output2)[:, -1].view(-1).data.cpu().numpy())\n outs3.extend(torch.sigmoid(output3)[:, -1].view(-1).data.cpu().numpy())\n\n ## Catboost part\n # если previous_test_df существует и не пустой - обновляем словари\n if previous_test_df.shape[0] != 0:\n # get correct answers for previous group\n previous_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])\n previous_test_df['user_answer'] = eval(test_df['prior_group_responses'].iloc[0])\n # оставляем только вопросы\n previous_test_df = previous_test_df[previous_test_df['content_type_id'] == 0]\n previous_test_df['q_counter'] = 1\n \n # 1. количество вопросов по пользователю накопленно - обновляем словарь\n update_dict(previous_test_df, dict_question_user_cnt, 'user_id', 'q_counter')\n # 2. количество правильных ответов по пользователю накопленно - обновляем словарь\n update_dict(previous_test_df, dict_correct_answers_user_cnt, 'user_id', 'answered_correctly')\n # 3. количество вопросов с объяснением по пользователю накопленно - обновляем словарь\n previous_test_df['prior_question_had_explanation'] = previous_test_df['prior_question_had_explanation'].fillna(0).astype(int)\n update_dict(previous_test_df, dict_question_explonation_user_cnt, 'user_id', 'prior_question_had_explanation')\n # 4. количество вопросов по content_id - обновляем словарь\n update_dict(previous_test_df, dict_questionid_part_tag12_avgtarget, 'content_id', 'q_counter', 0)\n # 5. количество правильных ответов по content_id - обновляем словарь\n update_dict(previous_test_df, dict_questionid_part_tag12_avgtarget, 'content_id', 'answered_correctly', 1)\n # 6. сглаженная доля правильных ответов по content_id - обновляем словарь\n answered_correctly_avg_content_global = 0.5 # средняя точность для нового типа вопросов - на шару\n trust_border = 5 # требуемое количество вопросов нового типа для расчета полной статистики без сглаживания\n for c in previous_test_df['content_id'].unique():\n #dict_questionid_part_tag12_avgtarget[c][2] = dict_questionid_part_tag12_avgtarget[c][1] / dict_questionid_part_tag12_avgtarget[c][0] \n # несглаженная точность\n answered_correctly_avg_content = dict_questionid_part_tag12_avgtarget[c][1] / dict_questionid_part_tag12_avgtarget[c][0]\n # если количество вопросов больше trust_border - сглаживать не надо, просто считаем точность\n if dict_questionid_part_tag12_avgtarget[c][0] >= trust_border:\n dict_questionid_part_tag12_avgtarget[c][2] = answered_correctly_avg_content\n # если количество вопросов меньше trust_border - сглаживаем\n else: \n border_calc_K = np.minimum(dict_questionid_part_tag12_avgtarget[c][0], trust_border)\n border_calc_L = (trust_border - border_calc_K) / trust_border\n dict_questionid_part_tag12_avgtarget[c][2] = (\n answered_correctly_avg_content * border_calc_K / trust_border +\n answered_correctly_avg_content_global * border_calc_L\n ) \n \n # 7. user_question_part_accuracy - точность ответа в разрезе part вопроса по юзеру - обновляем словарь\n dict_user_question_part_accuracy = dict_user_question_part_accuracy_update(previous_test_df, dict_user_question_part_accuracy)\n \n # 8. скользящая точность по user_id - обновляем словарь\n dict_user_slice_accuracy_5 = user_slice_accuracy_n_update(previous_test_df, dict_user_slice_accuracy_5, border=5)\n dict_user_slice_accuracy_20 = user_slice_accuracy_n_update(previous_test_df, dict_user_slice_accuracy_20, border=20)\n dict_user_slice_accuracy_50 = user_slice_accuracy_n_update(previous_test_df, dict_user_slice_accuracy_50, border=50)\n dict_user_slice_accuracy_session_3 = user_slice_accuracy_session_update(previous_test_df, dict_user_slice_accuracy_session_3, session_max_time=3)\n dict_user_slice_accuracy_session_12 = user_slice_accuracy_session_update(previous_test_df, dict_user_slice_accuracy_session_12, session_max_time=12)\n dict_user_slice_accuracy_session_48 = user_slice_accuracy_session_update(previous_test_df, dict_user_slice_accuracy_session_48, session_max_time=48)\n # 9. время до последнего правильного и неправильного ответа - обновляем словарь\n dict_user_correct_incorrect_timestamp = user_correct_incorrect_timestamp_update(previous_test_df, dict_user_correct_incorrect_timestamp)\n # 10. user_question_tag_accuracy - точность ответа в разрезе part вопроса по юзеру - обновляем словарь\n dict_user_question_tag_accuracy = user_question_tag_accuracy_update(previous_test_df, dict_user_question_tag_accuracy)\n # 11. prior_question_had_explanation - counter features - обновляем словарь\n dict_user_priorq_expl_types = user_priorq_expl_types_update(previous_test_df, dict_user_priorq_expl_types)\n # 12. user_answer_mode_10/50 - мода ответа на последние N вопросов == номер правильного ответа по content_id - обновляем словарь\n dict_user_answer_mode_10 = user_answer_mode_n_update(previous_test_df, dict_user_answer_mode_10, border=10)\n dict_user_answer_mode_50 = user_answer_mode_n_update(previous_test_df, dict_user_answer_mode_50, border=50)\n # 13. bundle_id_accuracy - точность по bundle_id вопроса - обновляем словарь\n dict_question_bundle_accuracy = question_bundle_accuracy_update(previous_test_df, dict_question_bundle_accuracy)\n \n ### new feature\n update_user_feats(previous_test_df)\n\n for user_id, answered_correctly, t in zip(previous_test_df['user_id'].values, previous_test_df['answered_correctly'].values, previous_test_df['timestamp'].values):\n if user_id in lt_correct_dict['timestamp']:\n if t == lt_correct_dict['timestamp'][user_id]:\n lt_correct_dict['last_timestamp_correct_cnt'][user_id] += 1\n lt_correct_dict['last_timestamp_correct_sum'][user_id] += answered_correctly\n else:\n lt_correct_dict['timestamp'].update({user_id:t})\n lt_correct_dict['last_timestamp_correct_cnt'][user_id] = 1\n lt_correct_dict['last_timestamp_correct_sum'][user_id] = answered_correctly\n else:\n lt_correct_dict['timestamp'].update({user_id:t})\n lt_correct_dict['last_timestamp_correct_cnt'].update({user_id:1})\n lt_correct_dict['last_timestamp_correct_sum'].update({user_id:answered_correctly})\n \n lt_correct_dict['last_timestamp_correct_pct'][user_id] = lt_correct_dict['last_timestamp_correct_sum'][user_id] / lt_correct_dict['last_timestamp_correct_cnt'][user_id]\n \n \n \n # сохраняем копию\n previous_test_df = test_df.copy()\n test_df2 = test_df.copy()\n # если батч пустой - не делаем расчеты\n# if test_df.shape[0] == 0: \n# test_df['answered_correctly'] = []\n# env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n# else: \n \n ########## РАСЧЕТ ФИЧЕЙ ##########\n \n # заполняем пропуски\n prior_question_elapsed_time_mean = 25452.541\n\n \n ##### new feature part\n test_df2 = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n lt_correct_cnt = np.zeros(len(test_df2), dtype=np.int8)\n lt_correct_sum = np.zeros(len(test_df2), dtype=np.int8)\n lt_correct_pct = np.zeros(len(test_df2), dtype=np.float16)\n test_df2 = add_user_feats_without_update(test_df2)\n test_df2 = add_uq_feats_and_update(test_df2) # new\n test_df2 = lagtime_for_test(test_df2)\n\n for i, (user_id, t) in enumerate(zip(test_df2['user_id'].values, test_df2['timestamp'].values)):\n if user_id in lt_correct_dict['timestamp']:\n lt_correct_cnt[i] = lt_correct_dict['last_timestamp_correct_cnt'][user_id]\n lt_correct_sum[i] = lt_correct_dict['last_timestamp_correct_sum'][user_id]\n lt_correct_pct[i] = lt_correct_dict['last_timestamp_correct_pct'][user_id]\n else:\n lt_correct_cnt[i] = -1\n lt_correct_sum[i] = -1\n lt_correct_pct[i] = -1\n\n test_df2['last_timestamp_correct_cnt'] = lt_correct_cnt\n test_df2['last_timestamp_correct_sum'] = lt_correct_sum\n test_df2['last_timestamp_correct_pct'] = lt_correct_pct\n\n additional_test_feature = test_df2[['last_timestamp_correct_cnt', 'last_timestamp_correct_sum',\n 'last_timestamp_correct_pct', 'lag_time', 'lag_time2',\n 'lag_time3', 'curr_user_time_diff', 'curr_user_time_diff_mean',\n 'curr_user_elapsed_time_diff', 'curr_uq_time_diff']].values \n \n test_df['prior_question_elapsed_time'] = test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean)\n test_df['prior_group_answers_correct'] = 0\n test_df['prior_group_responses'] = 0\n test_df = test_df.replace([np.inf, -np.inf], np.nan)\n test_df = test_df.fillna(0) \n ##### 1. сколько времени прошло с момента последнего вопроса и лекции: prior_question_timedelta_min, prior_lecture_timedelta_min\n # считаем фичи, заполняем словарь\n test_df = dict_user_timestampsdelta_get_update_3(test_df, dict_user_timestampsdelta_3)\n\n ##### 2. скользящая точность по user_id\n test_df['user_slice_accuracy_5'] = user_slice_accuracy_n_get(test_df, dict_user_slice_accuracy_5)\n test_df['user_slice_accuracy_20'] = user_slice_accuracy_n_get(test_df, dict_user_slice_accuracy_20)\n test_df['user_slice_accuracy_50'] = user_slice_accuracy_n_get(test_df, dict_user_slice_accuracy_50)\n test_df['user_slice_accuracy_session_3'] = user_slice_accuracy_session_get(test_df, dict_user_slice_accuracy_session_3, session_max_time=3)\n test_df['user_slice_accuracy_session_12'] = user_slice_accuracy_session_get(test_df, dict_user_slice_accuracy_session_12, session_max_time=12)\n test_df['user_slice_accuracy_session_48'] = user_slice_accuracy_session_get(test_df, dict_user_slice_accuracy_session_48, session_max_time=48)\n\n ##### 3.1. количество вопросов в группе\n test_df['task_container_freq'] = test_df.groupby(['user_id', 'task_container_id'])['task_container_id'].transform('count')\n ##### 3.2. порядковый номер вопроса в группе\n test_df['task_container_counter'] = test_df[['user_id', 'task_container_id', 'content_id']].groupby(['user_id', 'task_container_id'], as_index=False).agg(['cumcount']) + 1 \n ##### 4. количество вопросов по content_id на которые отвечал каждый user_id накопленно (добавляем + обновляем словарь) \n test_df['user_question_attempt_cnt'] = user_question_attempt_cnt_get_update(test_df, dict_user_question_attempt_cnt).astype(np.int16)\n ##### 5. количество лекций по пользователю накопленно\n test_df['lecture_cnt'] = user_lecture_cnt(test_df, dict_user_lectures_part)\n test_df = user_lectures_typeof_cnt(test_df, dict_user_lectures_typeof_cnt)\n ##### 6. part_l_q_cnt - сколько лекций с part равным part вопроса пользователь прослушал до вопроса (добавляем + обновляем словарь) \n # добавляем part и tags_list вопроса\n test_df['part_q'] = get_q_l(test_df, dict_questions, 'part')\n test_df['tags_list'] = get_q_l(test_df, dict_questions, 'tags_list')\n # добавляем part и tag лекций\n test_df['part_l'] = get_q_l(test_df, dict_lectures, 'part')\n test_df['tag_l'] = get_q_l(test_df, dict_lectures, 'tag') \n test_df['part_l_q_cnt'] = user_lectures_part(test_df, dict_user_lectures_part).astype(np.int16)\n\n ##### 7. tag_l_q_equal_cnt - слушал ли юзер лекцию по тегу равную тегу вопроса (количество) (добавляем + обновляем словарь)\n test_df['tag_l_q_equal_cnt'] = user_l_q_tag_equal(test_df, dict_user_l_q_tag_equal).astype(np.int16)\n\n ##### 8. user_question_part_accuracy - точность ответа в разрезе part вопроса по юзеру (добавляем)\n test_df['user_question_part_accuracy'] = dict_user_question_part_accuracy_get(test_df, dict_user_question_part_accuracy)\n ##### 9. user_question_tag_accuracy - точность ответа в разрезе tag вопроса по юзеру (добавляем)\n test_df['user_question_tag_accuracy'] = user_question_tag_accuracy_get(test_df, dict_user_question_tag_accuracy)\n ##### 10. время до последнего правильного и неправильного ответа\n test_df = user_correct_incorrect_timestamp_get(test_df, dict_user_correct_incorrect_timestamp)\n ##### 11. среднее время ответа на вопросы за последние N минут; отношение к среднему по content_id\n test_df = user_slice_question_time_mean_session(test_df, dict_user_slice_question_time_mean_session)\n ##### 12. prior_question_had_explanation - counter features\n test_df = user_priorq_expl_types_get(test_df, dict_user_priorq_expl_types)\n ##### 13. user_answer_mode_10/50 - мода ответа на последние N вопросов == номер правильного ответа по content_id\n test_df['user_answer_mode_10'] = user_answer_mode_n_get(test_df, dict_user_answer_mode_10, border=10)\n test_df['user_answer_mode_50'] = user_answer_mode_n_get(test_df, dict_user_answer_mode_50, border=50)\n ##### 14 точность по bundle_id вопроса\n # тянем bundle_id\n test_df = question_bundle_id_get(test_df, dict_questions)\n # кластер юзера по первому bundle_id - расчет фичи + заполнение словаря\n test_df = user_bundle_cluster_get_update(test_df, dict_user_bundle_cluster)\n # тянем точность по кластеру юзера по первому bundle_id\n test_df = question_bundle_accuracy_get(test_df, dict_question_bundle_accuracy)\n \n # сохраняем part_q, tags_list, task_container_freq, first_bundle_id_cluster в previous_test_df\n previous_test_df['part_q'] = test_df['part_q'].values\n previous_test_df['tags_list'] = test_df['tags_list'].values\n previous_test_df['task_container_freq'] = test_df['task_container_freq'].values\n previous_test_df['bundle_id'] = test_df['bundle_id'].values\n previous_test_df['first_bundle_id_cluster'] = test_df['first_bundle_id_cluster'].values\n # удаляем временные столбцы\n test_df.drop(columns=['part_l', 'part_q', 'tag_l', 'tags_list', 'bundle_id'], inplace=True)\n # оставляем только вопросы\n test_df = test_df[test_df['content_type_id'] == 0]\n \n # если фрейм не пустой (кейс когда в батче только лекция)\n if test_df.shape[0] > 0:\n \n # преобразуем в numpy\n np_test_df = test_df.to_numpy(dtype=np.float64, na_value=0)\n col_idx = features_map['user_id']\n\n ##### 15. количество вопросов по пользователю накопленно (добавляем из словаря)\n np_test_df = add_feats_from_dict(np_test_df, dict_question_user_cnt, col_idx) \n ##### 16. количество правильных ответов по пользователю накопленно (добавляем из словаря)\n np_test_df = add_feats_from_dict(np_test_df, dict_correct_answers_user_cnt, col_idx)\n ##### 17. доля правильных ответов по пользователю накопленно\n # делим количество правильных ответов на количество вопросов, если знаменатель = 0, тогда 0\n col_numerator = features_map['correct_answers_user_cnt']\n col_denominator = features_map['question_user_cnt']\n np_test_df = np.c_[ np_test_df, np.divide(np_test_df[:,col_numerator], np_test_df[:,col_denominator], \n out=np.zeros_like(np_test_df[:,col_denominator]), \n where=np_test_df[:,col_denominator]!=0) ]\n\n ##### 18. количество вопросов с объяснением по пользователю накопленно (добавляем из словаря)\n np_test_df = add_feats_from_dict(np_test_df, dict_question_explonation_user_cnt, col_idx)\n ##### 19. доля вопросов с объяснением по пользователю накопленно\n # делим количество вопросов с объяснением на количество вопросов, если знаменатель = 0, тогда 0\n col_numerator = features_map['prior_question_had_explanation_user_cnt']\n col_denominator = features_map['question_user_cnt']\n np_test_df = np.c_[ np_test_df, np.divide(np_test_df[:,col_numerator], np_test_df[:,col_denominator], \n out=np.zeros_like(np_test_df[:,col_denominator]), \n where=np_test_df[:,col_denominator]!=0) ] \n ##### 20. {question_id : [content_id_mean, part, part_mean, tag_1_mean, tag_2_mean, part_tag_1_mean, part_tag_2_mean]}\n for i in [dict_questionid_part_tag12_avgtarget_map['answered_correctly_avg_content_smooth'],\n dict_questionid_part_tag12_avgtarget_map['part'],\n dict_questionid_part_tag12_avgtarget_map['answered_correctly_avg_part'],\n dict_questionid_part_tag12_avgtarget_map['answered_correctly_avg_tag_1'],\n dict_questionid_part_tag12_avgtarget_map['answered_correctly_avg_tag_2'],\n # dict_questionid_part_tag12_avgtarget_map['answered_correctly_avg_part_tag_1'],\n # dict_questionid_part_tag12_avgtarget_map['answered_correctly_avg_part_tag_2'],\n # dict_questionid_part_tag12_avgtarget_map['tags_encoded'],\n ]:\n np_test_df = add_feats_from_dict_got_new_user(np_test_df, dict_questionid_part_tag12_avgtarget, col_idx=features_map['content_id'], col_dict=i)\n\n ##### 21. user_to_question_accuracy\n # делим точность пользователя на точность вопроса\n # если знаменатель = 0, тогда 0\n col_numerator = features_map['correct_answers_user_prc']\n col_denominator = features_map['content_id_mean']\n np_test_df = np.c_[ np_test_df, np.divide(np_test_df[:,col_numerator], np_test_df[:,col_denominator], out=np.zeros_like(np_test_df[:,col_denominator]), where=np_test_df[:,col_denominator]!=0) ]\n\n ##### 22. hmean_user_content_accuracy\n # среднее гармоническое из точности пользователя и точности вопроса\n user_acc = features_map['correct_answers_user_prc']\n question_acc = features_map['content_id_mean']\n np_test_df = np.c_[ np_test_df, 2*(np_test_df[:,user_acc] * np_test_df[:,question_acc])/(np_test_df[:,user_acc] + np_test_df[:,question_acc]) ]\n \n\n pred_feature = np_test_df[:,train_cols_clf]\n pred_feature = np.concatenate((pred_feature, additional_test_feature), axis=1)\n ########## GET MODEL PREDICT ##########\n# dict_keys(['timestamp', 'last_timestamp_correct_cnt', 'last_timestamp_correct_sum', 'last_timestamp_correct_pct'])\n\n\n pred = pd.DataFrame()\n pred['score_lgbm'] = lgb_model.predict(pred_feature)\n pred['score_cat'] = cat_model.predict_proba(pred_feature)[:,1]\n pred['score_saint1'] = np.array(outs1)\n pred['score_saint2'] = np.array(outs2)\n pred['score_saint3'] = np.array(outs3)\n \n test_df['answered_correctly'] = (c1*pred['score_lgbm'] + c1_2 * pred['score_cat'] + c2*pred['score_saint1'] + c3*pred['score_saint2']+ c4*pred['score_saint3']).values\n\n \n else:\n test_df['answered_correctly'] = np.array([], dtype = np.float32)\n\n env.predict(test_df[['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"0.25 * 0.7, 0.25 * 0.3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}