{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"global_test = 2\n\n#1 - feature generation, #2 - submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport glob\nimport riiideducation\n#from sklearn.linear_model import LogisticRegression\n#from sklearn.metrics import roc_auc_score\nimport matplotlib.pyplot as plt\n#from sklearn.model_selection import train_test_split\nfrom tqdm import tqdm\n#from sklearn.preprocessing import StandardScaler\n#from sklearn.inspection import permutation_importance\nfrom catboost import CatBoostClassifier\n\nfrom sklearn.decomposition import PCA\nfrom sklearn.preprocessing import StandardScaler\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.options.display.max_rows = 100\npd.options.display.max_columns = 100","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## file access functions:"},{"metadata":{"trusted":true},"cell_type":"code","source":"def df_to_np(df, filter_lectures:bool, convert_answers:bool):\n\n    tmstmp = (df['timestamp']/3600000).to_numpy(dtype = np.float32) \n    userid = df['user_id'].to_numpy()\n    ctntid = df['content_id'].to_numpy()\n    ctnttp = df['content_type_id'].to_numpy()\n    contnr = df['task_container_id'].to_numpy()\n    \n    pqtime = np.nan_to_num(df['prior_question_elapsed_time']\\\n                           .to_numpy(dtype = np.float32), nan = float32m1)\n    \n    pqexpl = df['prior_question_had_explanation']\\\n             .to_numpy(dtype = np.int8, na_value = 1)\n    \n    if convert_answers:\n        usrans = df['user_answer'].to_numpy()              \n        anscor = df['answered_correctly'].to_numpy()\n           \n    if filter_lectures:\n        f = ctnttp == int8_0\n        \n        if convert_answers: \n            return tmstmp[f], userid[f], ctntid[f], ctnttp[f],\\\n            contnr[f], pqtime[f], pqexpl[f], usrans[f], anscor[f]\n        else:\n            return tmstmp[f], userid[f], ctntid[f], ctnttp[f],\\\n            contnr[f], pqtime[f], pqexpl[f]\n    else:\n        return tmstmp, userid, ctntid, ctnttp, contnr, pqtime, pqexpl, usrans, anscor","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## initialization functions - run once:"},{"metadata":{"trusted":true},"cell_type":"code","source":"def question_maps():\n    \n    global qestn_tagsmap, qestn_partmap, qestn_bndlmap, qestn_cansmap\n    \n    df = pd.read_csv('../input/riiid-test-answer-prediction/questions.csv')\n\n    tags_list = df['tags'].replace(np.nan, 188).to_list()\n\n    qestn_tagsmap = [[np.uint8(y) for y in str(x).split()] for x in tags_list]\n    qestn_partmap = df['part'].to_numpy().astype(np.uint8)-uint8_1\n    qestn_bndlmap = df['bundle_id'].to_numpy().astype(np.uint16)\n    qestn_cansmap = df['correct_answer'].to_numpy().astype(np.uint8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def lecture_maps():\n    \n    global lectr_tag_map, lectr_partmap, lectr_typemap\n\n    df = pd.read_csv('../input/riiid-test-answer-prediction/lectures.csv')\n\n    df_lecture_id = df.lecture_id\\\n    .to_numpy(dtype = np.int16)\n\n    df_tag  = df.tag\\\n    .to_numpy(dtype=np.uint8)\n\n    df_part = df.part\\\n    .to_numpy(dtype=np.uint8)\n\n    df_type_of = df.type_of\\\n    .replace({'starter':0,'concept':1,'solving question':2,'intention':3})\\\n    .to_numpy(dtype = np.uint8)\n\n    lectr_tag_map = np.zeros(np.iinfo(np.int16).max, dtype = np.uint8)\n    lectr_partmap = np.zeros(np.iinfo(np.int16).max, dtype = np.uint8)\n    lectr_typemap = np.zeros(np.iinfo(np.int16).max, dtype = np.uint8)\n\n\n    for i in range(len(df)):\n        lectr_tag_map[df_lecture_id[i]] = df_tag[i]\n        lectr_partmap[df_lecture_id[i]] = df_part[i]\n        lectr_typemap[df_lecture_id[i]] = df_type_of[i]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_cor_table():\n     \n    max_neigbrs = 1000\n    \n    global cor_table\n    \n    map1 = np.load('../input/content-correlation-100to300/ctnt_map.npy')\n    map2 = np.load('../input/content-correlation/ctnt_map.npy')\n\n    cor1 = np.load('../input/content-correlation-100to300/result.npy')\n    cor2 = np.load('../input/content-correlation/result.npy')\n\n    cor = (cor2[map2[:max_content],:,:][:,map2[:max_content],:].astype(np.uint32)\n          +cor1[map1[:max_content],:,:][:,map1[:max_content],:].astype(np.uint32))\n\n    #correction for correlated content size, \n    #giving more weight to questions with a lot of answers\n    size = np.log(np.log(cor[:,:,0].sum(axis = 1)+1))\n    size = size/size.max()/10\n    corrs = (cor[:,:,0]+5)/(cor[:,:,1]*1.7+cor[:,:,0]+10)+size\n    \n    cor_table = corrs.argsort(axis = 1)[:,-max_neigbrs:].astype(np.int32)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_content_answer_shares():\n    \n    global ca_shares_all\n\n    columns = ['user_id',\n                   'content_id',\n                   'content_type_id',\n                   'user_answer',\n                   'answered_correctly']\n\n    df = get_train_large(t_part=99, columns=columns) \n    df = df.loc[df.content_type_id == 0, df.columns != 'content_type_id']\n\n    ca_shares_all = pd.pivot_table(df,\n                                   values='answered_correctly',\n                                   index='content_id',\n                                   columns='user_answer',\n                                   aggfunc='count',\n                                   fill_value=0).to_numpy()+1\n    \n    ca_shares_all = (ca_shares_all.T/ca_shares_all.sum(axis = 1)).T.astype(np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_content_first_answer_mean():\n    \n    global ctnt_fam\n    \n    columns = ['user_id',\n            'new_order',\n            'answered_correctly',\n            'content_id',\n            'content_type_id']\n        \n    df = get_train_large(t_part = 99, columns = columns)\n\n    df = df\\\n    .loc[df.content_type_id==0, df.columns!='content_type_id']\\\n    .sort_values(by = 'new_order')\n\n    df = df.groupby(['user_id','content_id']).first()\n\n    df = df.groupby('content_id').answered_correctly.mean().sort_index()\n    \n    ctnt_fam = df.to_numpy(dtype = np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_ctnt_enc():\n    \n    global ctnt_enc\n    \n    qestn_tagsmap_ohe = np.zeros((len(qestn_tagsmap), 189), dtype = np.bool)\n\n    for i,j in enumerate(qestn_tagsmap):\n        for k in j:\n            qestn_tagsmap_ohe[i,k] = True\n    \n    \n    tags_comps = StandardScaler().fit_transform(\n        PCA(n_components=3, random_state=0).fit_transform(qestn_tagsmap_ohe)\n    )\n\n    corr_comps = StandardScaler().fit_transform(\n        PCA(n_components=9, random_state=0).fit_transform(cor_table)\n    )\n\n    \n    comb_comps = np.concatenate([tags_comps,corr_comps], axis = 1)\n\n    ctnt_enc = PCA(n_components=1, random_state=0)\\\n    .fit_transform(comb_comps).astype(np.float32).ravel()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_ac_pqexpl():\n    \n    global ac_pqexpl\n    \n    ac_pqexpl = np.zeros((max_content, 2, 2), dtype = np.int32)\n    \n    columns = ['content_id',\n               'prior_question_had_explanation',\n               'content_type_id',\n               'answered_correctly']\n\n    df = get_train_large(t_part = 99, columns = columns)\n\n    df = df\\\n    .loc[df.content_type_id==0]\\\n    .groupby(['content_id','prior_question_had_explanation'])\\\n    .agg({'answered_correctly':['sum','count']})\\\n    .droplevel(0, axis  =1).reset_index()\\\n    .fillna(1)\n\n    ctntid = df['content_id'].to_numpy()\n    pqexpl = df['prior_question_had_explanation'].to_numpy(dtype = np.int8)\n    anssum = df['sum'].to_numpy(dtype = np.int32)\n    anscnt = df['count'].to_numpy(dtype = np.int32)     \n    ansfls = anscnt - anssum\n\n    for r in range(len(df)):\n        ac_pqexpl[ctntid[r],pqexpl[r],1] += ansfls[r]\n        ac_pqexpl[ctntid[r],pqexpl[r],0] += anssum[r]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## pretrain functions, run every part:"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_small(t_part:int):\n\n    all_files  = glob.glob('../input/riiid-parquets-v5/df_*')\n    read_files = [file for file in all_files if file.endswith('_'+str(t_part))]\n    df = pd.read_parquet(read_files[0])\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_large(t_part:int, columns:list):\n    \n    all_files  = glob.glob('../input/riiid-parquets-v5/df_*')\n    read_files = [file for file in all_files if not file.endswith('_'+str(t_part))]\n    df = pd.concat([pd.read_parquet(file, columns = columns) for file in read_files])\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_groups(t_part:int):\n    \n    df = get_train_small(t_part)\n\n    groups = []\n    for i in np.arange(0, 10000, dtype = np.int16):\n        group = df.loc[df.new_order == i].reset_index(drop = True)\n        groups.append(group)\n    \n    return groups","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_arrays_and_lists():\n    \n    global next_uplace,\\\n    au_ctntid,\\\n    a_userid,\\\n    lu_seq,\\\n    lu_seq_part,\\\n    au_anshar,\\\n    au_ctshar,\\\n    user_map,\\\n    au_tmstmp_prv\n    \n    au_ctntid     = np.zeros((max_users, max_content, 3), dtype = np.int8)\n    a_userid      = np.zeros((max_users, 2), dtype = np.int16)\n    \n    au_anshar     = np.zeros((max_users, 2), dtype = np.float32) \n    au_ctshar     = np.zeros((max_users, 2), dtype = np.float32) \n\n    user_map      = np.zeros(np.iinfo(np.int32).max,dtype = np.int32)\n    next_uplace   = np.int32(1)\n\n    au_tmstmp_prv = np.zeros((max_users,3), dtype = np.float32)\n    \n    lu_seq        = [[] for _ in range(max_users)]\n    lu_seq_part   = [[[],[],[],[],[],[],[]] for _ in range(max_users)]\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## iterational functions, run every iteration:"},{"metadata":{"trusted":true},"cell_type":"code","source":"def update_user_map(unique_users):\n    \n    global next_uplace\n    \n    for u in unique_users:\n        if user_map[u] == int32_0:\n            user_map[u] = next_uplace\n            next_uplace += int32_1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def update_arrays(df):\n\n    tmstmp,userid,ctntid,ctnttp,contnr,pqtime,pqexpl,usrans,anscor = df_to_np(df,False,True)\n    \n    for r in range(len(df)):\n        \n        user_ = user_map[userid[r]]\n\n        if tmstmp[r] > au_tmstmp_prv[user_,0]:\n            au_tmstmp_prv[user_,2] = au_tmstmp_prv[user_,1]\n            au_tmstmp_prv[user_,1] = au_tmstmp_prv[user_,0]\n            au_tmstmp_prv[user_,0] = tmstmp[r]\n  \n        if ctnttp[r] == int8_0:\n            \n            lsu   = lu_seq[user_]\n            lsup  = lu_seq_part[user_][qestn_partmap[ctntid[r]]]\n            \n            bndl_ = qestn_bndlmap[ctntid[r]]\n            ctnt_ = ctntid[r]\n                        \n            if len(lsu)>m: lsu.pop(0)\n            if len(lsup)>m: lsup.pop(0)\n            \n      \n            au_ctntid[user_,ctnt_,1]         += int8_1\n            au_ctntid[user_,ctnt_,2]          = usrans[r]\n            \n            \n            if anscor[r] == int8_1:\n                a_userid[user_,0]            += int16_1\n                au_ctntid[user_,ctnt_,0]      = int8_1\n                lsu.append(True)\n                lsup.append(True)\n                au_anshar[user_, 0]           += ca_shares_all[ctnt_,usrans[r]]\n                au_ctshar[user_, 0]           += ctnt_fam[ctnt_]\n                \n            else:\n                a_userid[user_,1]             += int16_1\n                au_ctntid[user_,ctnt_,0]       = int8_0\n                lsu.append(False)\n                lsup.append(False)\n                au_anshar[user_, 1]           += ca_shares_all[ctnt_,usrans[r]]\n                au_ctshar[user_, 1]           += ctnt_fam[ctnt_]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_features(df, is_test:bool):\n    \n    if is_test:\n        tmstmp,userid,ctntid,ctnttp,contnr,pqtime,pqexpl=\\\n        df_to_np(df,True,False)\n    else:\n        tmstmp,userid,ctntid,ctnttp,contnr,pqtime,pqexpl,usrans,anscor=\\\n        df_to_np(df,True,True)\n\n    user = user_map[userid]\n    part = qestn_partmap[ctntid]\n    userid_ctntid_ = au_ctntid[user,ctntid]  \n    \n    userid_ = a_userid[user,:]\n    userid_avg_ = (userid_[:,0]/(userid_[:,0]+userid_[:,1]+int16_1)).astype(np.float32)\n    \n    cp_ = ac_pqexpl[ctntid,pqexpl,:]\n    ctntid_pqexpl_avg_ = (cp_[:,0]/(cp_[:,0]+cp_[:,1]+int32_1)).astype(np.float32)\n    \n    \n    #based on answer and content solve probability\n    \n    uanshar_ = au_anshar[user, :]\n    #uanshar_slf = (uanshar_[:,0]/(uanshar_[:,0]+uanshar_[:,1]+e)).astype(np.float32)\n    #uanshar_cor = (uanshar_[:,0]/(userid_[:,0]+e)).astype(np.float32)\n    uanshar_fls = (uanshar_[:,1]/(userid_[:,1]+e)).astype(np.float32)\n    \n    uctshar_ = au_ctshar[user, :]\n    uctshar_slf = (uctshar_[:,0]/(uctshar_[:,0]+uctshar_[:,1]+e)).astype(np.float32)\n    uctshar_cor = (uctshar_[:,0]/(userid_[:,0]+e)).astype(np.float32)\n    uctshar_fls = (uctshar_[:,1]/(userid_[:,1]+e)).astype(np.float32)\n    \n    \n    #user features based on neighboring questions:\n    \n    correlation_ids = cor_table[ctntid]\n    neigh = au_ctntid[user.reshape(-1,1),correlation_ids,:]\n    \n    all_ans_cnt = np.count_nonzero(neigh[:,:,1],axis = 1).astype(np.int16)\n    cor_ans_cnt = np.count_nonzero(neigh[:,:,0],axis = 1).astype(np.int16)\n    fls_ans_cnt = all_ans_cnt - cor_ans_cnt\n    \n    neigh_ca_shrs_all = ca_shares_all[correlation_ids,neigh[:,:,2]]*(neigh[:,:,1]!=int8_0)\n    \n    cor_shrs_all = ((neigh_ca_shrs_all*(neigh[:,:,0]==int8_1)).sum(axis = 1)/\n                    (cor_ans_cnt+e)).astype(np.float32)\n    \n    fls_shrs_all = ((neigh_ca_shrs_all*(neigh[:,:,0]==int8_0)).sum(axis = 1)/\n                    (fls_ans_cnt+e)).astype(np.float32)\n\n    \n    #user features based on last n questions:\n    \n    lu_seq_        = [lu_seq[u] for u in user]\n    \n    lst_m_avg      = np.array(\n        [x.count(True)/(len(x)+e) for x in lu_seq_],\n        dtype = np.float32)\n    \n    lst_s_avg      = np.array(\n        [x[-s:].count(True)/(len(x[-s:])+e) for x in lu_seq_],\n        dtype = np.float32)\n       \n        \n    lu_seq_part_   = [lu_seq_part[u][part[_]] for _, u in enumerate(user)]\n    \n    lst_part_m_avg = np.array(\n        [x.count(True)/(len(x)+e) for x in lu_seq_part_],\n        dtype = np.float32)\n    \n    lst_part_s_avg = np.array(\n        [x[-s:].count(True)/(len(x[-s:])+e) for x in lu_seq_part_],\n        dtype = np.float32)\n\n\n\n    X = pd.DataFrame({\n        'part':part,\n        'prior_explanation':pqexpl,\n        'prior_elapsed_time':pqtime,\n        'content':ctntid,\n        'ctntent_encoded':ctnt_enc[ctntid],\n        'task_container':contnr,\n        \n        'time_to_cont_1':tmstmp - au_tmstmp_prv[user,0],\n        'time_to_cont_3':tmstmp - au_tmstmp_prv[user,2],\n        'time_cont1_to_cont2':au_tmstmp_prv[user,0] - au_tmstmp_prv[user,1],\n        'time_cont2_to_cont3':au_tmstmp_prv[user,1] - au_tmstmp_prv[user,2],\n        \n        'user_content_attempts':userid_ctntid_[:,1],\n        'user_content_last_1':userid_ctntid_[:,0],\n        \n        'user_part_last_m_avg':lst_part_m_avg, \n        'user_part_last_s_avg':lst_part_s_avg,\n        'user_last_m_avg':lst_m_avg,\n        'user_last_s_avg':lst_s_avg, \n\n        'content_explanation_avg':ctntid_pqexpl_avg_,\n        'content_first_answer_avg':ctnt_fam[ctntid],\n        'content_avg_time':ctnt_mtime[ctntid],\n\n        'user_relative_content_avg':uctshar_slf,        \n        'user_true_content_avg':uctshar_cor, \n        'user_false_content_avg':uctshar_fls,\n        'user_false_answer_avg':uanshar_fls,\n        \n        'neighbor_content_true_shares':cor_shrs_all,\n        'neighbor_content_false_shares':fls_shrs_all,\n    })\n    \n    if is_test:\n        return X\n    else:\n        return X, anscor\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## execution:"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nuint8_0 = np.uint8(0)\nuint8_1 = np.uint8(1)\n\nuint16_0 = np.uint16(0)\nuint16_1 = np.uint16(1)\n\nint8_0  = np.int8(0)\nint8_1  = np.int8(1)\n\nint16_0 = np.int16(0)\nint16_1 = np.int16(1)\n\nint32_0 = np.int32(0)\nint32_1 = np.int32(1)\n\nfloat32m1 = np.float32(-1)\n\nmax_users   = 450000\nmax_content = 13523\n\nm = 100\ns = 20\ne = 0.1\n\nquestion_maps()\nlecture_maps()\nget_cor_table()\nget_content_answer_shares()\nget_ac_pqexpl()\nget_content_first_answer_mean()\nget_ctnt_enc()\n\nctnt_mtime = np.load('../input/question-duration/question_mean_time.npy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nif global_test == 1:\n\n    for i in tqdm(range(10)):\n\n        X = []\n        y = []\n\n        get_arrays_and_lists()\n        groups = get_train_groups(i)\n\n        for df in groups:\n\n            update_user_map(df.user_id.unique())\n            X_, y_ = get_features(df,False)\n            X.append(X_)\n            y.append(y_)\n            update_arrays(df)\n        \n        del(groups)\n\n        X = pd.concat(X)\n        y = np.concatenate(y)\n\n        X.to_parquet('X_'+str(i))\n        np.save('y_'+str(i), y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nif global_test == 2:\n    \n    get_arrays_and_lists()\n    \n    for i in tqdm(range(10)):\n\n        groups = get_train_groups(i)\n        \n\n        for df in groups:\n\n            update_user_map(df.user_id.unique())\n            update_arrays(df)\n\n        del(groups)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if global_test == 2:\n    \n    model1 = CatBoostClassifier()\n    model1.load_model(fname='../input/riiid-model/cb1')\n    \n    model2 = CatBoostClassifier()\n    model2.load_model(fname='../input/riiid-model/cb2')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if global_test == 2:\n    \n    env = riiideducation.make_env()\n    iter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nif global_test == 2:\n    \n    old_df = None\n\n    for (new_df, sample) in iter_test:\n        \n        \n        if old_df is not None:\n            old_df['user_answer'] = np.array(\n                [int(x) for x in new_df.iloc[0,9][1:-1].split(', ')], dtype = np.int8)\n            old_df['answered_correctly'] = np.array(\n                [int(x) for x in new_df.iloc[0,8][1:-1].split(', ')], dtype = np.int8)\n            \n            update_arrays(old_df)\n            \n            \n        old_df = new_df.iloc[:,:8].copy()\n        update_user_map(new_df.user_id.unique())\n        X = get_features(new_df, True)\n\n        sample['answered_correctly'] =  (\n            model1.predict_proba(X)[:,1]/2 + model2.predict_proba(X)[:,1]/2\n        )\n        \n        env.predict(sample)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}