{"cells":[{"metadata":{"_kg_hide-input":false,"execution":{"iopub.execute_input":"2020-12-16T14:27:03.065408Z","iopub.status.busy":"2020-12-16T14:27:03.064445Z","iopub.status.idle":"2020-12-16T14:27:04.258134Z","shell.execute_reply":"2020-12-16T14:27:04.257475Z"},"papermill":{"duration":1.215355,"end_time":"2020-12-16T14:27:04.258268","exception":false,"start_time":"2020-12-16T14:27:03.042913","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import gc\nimport numpy as np\nimport pandas as pd\nimport pickle\nfrom sklearn.metrics import roc_auc_score\nimport lightgbm as lgb\nimport joblib\nfrom tqdm.notebook import tqdm\n\nimport os\nimport riiideducation","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### This kernel is written because reading pkl(type is dict) will used soooooooo exaggerated RAM."},{"metadata":{},"cell_type":"markdown","source":"# method 1"},{"metadata":{},"cell_type":"markdown","source":"1. Create several required ndarray (there is 3)\n2. Dict just store index of ndarray\n3. read them now\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# one dict\ncurr_u_dict = pickle.load(open(\"../input/merge-data1615/curr_u_dict_1615.pkl.data\",\"rb\")) # dict\n\n# three ndarray\nnp_cor_cnt = pickle.load(open(\"../input/merge-data1615/np_cor_cnt_1615.pkl.data\",\"rb\")) # ndarray\nnp_all_cnt = pickle.load(open(\"../input/merge-data1615/np_all_cnt_1615.pkl.data\",\"rb\")) # ndarray\nnp_uctdiff_cnt = pickle.load(open(\"../input/merge-data1615/np_uctdiff_cnt_1615.pkl.data\",\"rb\")) # ndarray","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The following is the function used with the dict and ndarray when testing"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-16T14:27:04.298536Z","iopub.status.busy":"2020-12-16T14:27:04.293273Z","iopub.status.idle":"2020-12-16T14:27:04.342218Z","shell.execute_reply":"2020-12-16T14:27:04.341441Z"},"papermill":{"duration":0.068371,"end_time":"2020-12-16T14:27:04.342346","exception":false,"start_time":"2020-12-16T14:27:04.273975","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def add_user_feats_without_update(df):\n    ucc = np.zeros(len(df), dtype=np.uint16)\n    uac = np.zeros(len(df), dtype=np.uint16)\n    uqcor = np.zeros(len(df), dtype=np.uint8)\n    uqcnt = np.zeros(len(df), dtype=np.uint8)\n    utdiff = np.zeros(len(df), dtype=np.uint64)\n    utdiff_mean = np.zeros(len(df), dtype=np.uint64) \n    uelapdiff = np.zeros(len(df), dtype=np.float32)  \n    uq_timediff = np.zeros(len(df), dtype=np.uint64) \n    for cnt,row in enumerate(df[['user_id', 'content_id','timestamp','prior_question_elapsed_time']].itertuples(index=False)): \n        if row[0] in curr_u_dict:\n            ucc[cnt] = curr_u_dict[row[0]][\"ucc\"]\n            uac[cnt] = curr_u_dict[row[0]][\"uac\"]\n            utdiff[cnt] = row[2] - curr_u_dict[row[0]][\"uts\"]\n            utdiff_mean[cnt] = curr_u_dict[row[0]][\"utsdiff\"][1] / curr_u_dict[row[0]][\"utsdiff\"][0] \n            uelapdiff[cnt] = row[3] - curr_u_dict[row[0]][\"uelapdiff\"] \n            if row[1] in curr_u_dict[row[0]]:\n                tmp_idx = curr_u_dict[row[0]][row[1]]\n                uq_timediff[cnt] =  row[2] - np_uctdiff_cnt[tmp_idx] \n                uqcor[cnt] = np_cor_cnt[tmp_idx]\n                uqcnt[cnt] = np_all_cnt[tmp_idx]\n            else:\n                uqcor[cnt] = 0; uqcnt[cnt] = 0\n                uq_timediff[cnt] = 0 \n        else:\n            ucc[cnt] = 0; uac[cnt] = 0\n            uqcor[cnt] = 0; uqcnt[cnt] = 0\n            utdiff[cnt] = 0; utdiff_mean[cnt] = 0; \n            uelapdiff[cnt] = 0; uq_timediff[cnt] = 0 \n            \n    user_feats_df = pd.DataFrame({'curr_user_correct_cnt':ucc, 'curr_user_answer_cnt':uac,\n                                  'curr_uq_correct_cnt':uqcor, 'curr_uq_answer_cnt':uqcnt,\n                                  'curr_user_time_diff':utdiff, 'curr_user_time_diff_mean':utdiff_mean, \n                                  'curr_user_elapsed_time_diff':uelapdiff, 'curr_uq_time_diff':uq_timediff \n                                 }) \n    user_feats_df['curr_uq_acc'] = user_feats_df['curr_uq_correct_cnt'] / user_feats_df['curr_uq_answer_cnt']\n    user_feats_df['curr_uq_acc'].fillna(0.680, inplace=True)\n    user_feats_df['curr_uq_acc'] = user_feats_df['curr_uq_acc'].astype(np.float32)\n    user_feats_df['curr_uq_correct_cnt'] = user_feats_df['curr_uq_correct_cnt'].where(user_feats_df['curr_uq_correct_cnt'] <= 4, 4)\n    user_feats_df['curr_uq_answer_cnt'] = user_feats_df['curr_uq_answer_cnt'].where(user_feats_df['curr_uq_answer_cnt'] <= 4, 4)\n    user_feats_df['curr_user_acc'] = user_feats_df['curr_user_correct_cnt'] / user_feats_df['curr_user_answer_cnt']\n    user_feats_df['curr_user_acc'].fillna(0.680, inplace=True)\n    user_feats_df['curr_user_acc'] = user_feats_df['curr_user_acc'].astype(np.float32)\n    user_feats_df['curr_user_time_diff_mean'].fillna(0, inplace=True)  \n    user_feats_df['curr_user_elapsed_time_diff'].fillna(0, inplace=True) \n    user_feats_df['curr_uq_time_diff'].fillna(0, inplace=True)  \n    df = pd.concat([df, user_feats_df], axis=1)\n    return df\n\n\ndef update_user_feats(df):\n    global idx\n    for row in df[['user_id','content_id','answered_correctly','timestamp', 'content_type_id','prior_question_elapsed_time',]].values: \n        if row[4] == 0:\n            if row[0] in curr_u_dict:\n                curr_u_dict[row[0]][\"ucc\"] += row[2]\n                curr_u_dict[row[0]][\"uac\"] += 1\n                curr_u_dict[row[0]][\"uts\"] = row[3]\n                curr_u_dict[row[0]][\"utsdiff\"][0] += 1 \n                curr_u_dict[row[0]][\"utsdiff\"][1] += row[3] \n                curr_u_dict[row[0]][\"uelapdiff\"] = row[5] \n                if row[1] in curr_u_dict[row[0]]:\n                    tmp_idx = curr_u_dict[row[0]][row[1]]\n                    np_uctdiff_cnt[tmp_idx] = row[3] \n                    np_cor_cnt[tmp_idx] += row[2]\n                    np_all_cnt[tmp_idx] += 1\n                else:\n                    curr_u_dict[row[0]][row[1]] = idx\n                    np_uctdiff_cnt[idx] = row[3] \n                    np_cor_cnt[idx] += row[2]\n                    np_all_cnt[idx] += 1\n                    idx += 1\n            else:\n                curr_u_dict[row[0]] = {}\n                curr_u_dict[row[0]][\"ucc\"] = row[2]\n                curr_u_dict[row[0]][\"uac\"] = 1\n                curr_u_dict[row[0]][\"uts\"] = row[3]\n                curr_u_dict[row[0]][\"utsdiff\"] = [1, row[3]] \n                curr_u_dict[row[0]][\"uelapdiff\"] = row[5] \n                curr_u_dict[row[0]][row[1]] = idx\n                np_uctdiff_cnt[idx] = row[3] \n                np_cor_cnt[idx] += row[2]\n                np_all_cnt[idx] += 1\n                idx += 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# method 2"},{"metadata":{},"cell_type":"markdown","source":"1. convert the dictionary into a pandas dataframe.(pandas.to_dict)\n2. save it and then upload it to kaggle.\n3. read it by writing your own function, as shown below.\n4. Of course pandas has pandas.from_dict ,but it does not apply to the case of multiple nesting.(like me)."},{"metadata":{"trusted":true},"cell_type":"code","source":"# \ndef to_letures_dict(df):\n    da = {}\n    for t in tqdm(df.itertuples(name=None), total=df.shape[0]):\n        key = t[0]\n        sub_dict = {}\n        for i, col in enumerate(df.columns, 1):\n            if col == \"has_tags\":\n                sub_dict[col] = eval(t[i])\n            else:\n                sub_dict[col] = t[i]\n        da[key] = sub_dict\n    return da\n\ncurr_lectures_dict = pd.read_csv(\"../input/merge-data1615/curr_lectures_dict_1615.csv.data\", index_col=0)\ncurr_lectures_dict = to_letures_dict(curr_lectures_dict)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The following is the function used with the dict and ndarray when testing"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-16T14:27:04.397659Z","iopub.status.busy":"2020-12-16T14:27:04.392221Z","iopub.status.idle":"2020-12-16T14:27:18.713442Z","shell.execute_reply":"2020-12-16T14:27:18.710606Z"},"papermill":{"duration":14.355603,"end_time":"2020-12-16T14:27:18.713612","exception":false,"start_time":"2020-12-16T14:27:04.358009","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv')\nlectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')\nlectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])\nlectures_df['content_type_id'] = 1\n\nq_taglist_df = pd.read_csv(\"/kaggle/input/riiid-test-answer-prediction/questions.csv\")[['tags']].astype(str)\nq_taglist_df[\"tags_l\"] = [x.split() for x in q_taglist_df.tags.values]\nq_taglist_df['content_type_id'] = 0\nq_taglist_df.drop(\"tags\", axis=1, inplace=True)\nq_taglist_df.drop(10033, axis=0, inplace=True) # nan\n\ndef add_lectures_feats(df, curr_dict):\n    new_df = df[[\"row_id\", \"user_id\", \"timestamp\", \"content_id\", \"content_type_id\"]]\n    new_df = new_df.merge(lectures_df, how=\"left\", left_on = [\"content_id\",\"content_type_id\"], right_on = [\"lecture_id\",\"content_type_id\"])\n    new_df = new_df.merge(q_taglist_df, how=\"left\", left_on = [\"content_id\",\"content_type_id\"], right_on = [q_taglist_df.index,\"content_type_id\"])\n    new_df = new_df.sort_values([\"timestamp\"])\n    new_df = new_df[['timestamp', 'user_id', 'content_type_id','tag','part_1','part_2','part_3','part_4','part_5','part_6','part_7',\n                     'type_of_concept','type_of_intention','type_of_solving_question','type_of_starter','tags_l','row_id']]\n    ulc_lb = np.zeros(len(df), dtype=\"int8\")\n    part1_l = np.zeros(len(df), dtype=\"uint16\")\n    part2_l = np.zeros(len(df), dtype=\"uint16\")\n    part3_l = np.zeros(len(df), dtype=\"uint16\")\n    part4_l = np.zeros(len(df), dtype=\"uint16\")\n    part5_l = np.zeros(len(df), dtype=\"uint16\")\n    part6_l = np.zeros(len(df), dtype=\"uint16\")\n    part7_l = np.zeros(len(df), dtype=\"uint16\")\n    type_of_concept_l = np.zeros(len(df), dtype=\"uint16\")\n    type_of_intention_l = np.zeros(len(df), dtype=\"uint16\")\n    type_of_solving_question_l = np.zeros(len(df), dtype=\"uint16\")\n    type_of_starter_l = np.zeros(len(df), dtype=\"uint16\")\n    has_tags_l = np.zeros(len(df), dtype=\"float32\")\n    \n    # 0.'timestamp', 1.'user_id', 2.'content_type_id',3.'tag',4.'part_1',5.'part_2',6.'part_3',7.'part_4',8.'part_5',9.'part_6',10.'part_7',\n    # 11.'type_of_concept',12.'type_of_intention',13.'type_of_solving_question',14.'type_of_starter',15.'tags_l', 16.'row_id'\n    for cnt,row in enumerate(new_df.itertuples(index=False)):\n        if row[1] in curr_dict:\n            if row[2] == 1:\n                curr_dict[row[1]][\"lecture_bool\"] = 1\n                curr_dict[row[1]][\"part_1_cnt\"] += int(row[4])\n                curr_dict[row[1]][\"part_2_cnt\"] += int(row[5])\n                curr_dict[row[1]][\"part_3_cnt\"] += int(row[6])\n                curr_dict[row[1]][\"part_4_cnt\"] += int(row[7])\n                curr_dict[row[1]][\"part_5_cnt\"] += int(row[8])\n                curr_dict[row[1]][\"part_6_cnt\"] += int(row[9])\n                curr_dict[row[1]][\"part_7_cnt\"] += int(row[10])\n                curr_dict[row[1]][\"type_of_concept_cnt\"] += int(row[11])\n                curr_dict[row[1]][\"type_of_intention_cnt\"] += int(row[12])\n                curr_dict[row[1]][\"type_of_solving_question_cnt\"] += int(row[13])\n                curr_dict[row[1]][\"type_of_starter_cnt\"] += int(row[14])\n                curr_dict[row[1]][\"has_tags\"].add(int(row[3]))\n        else:\n            curr_dict[row[1]] = {}\n            if row[2] == 1:\n                curr_dict[row[1]][\"lecture_bool\"] = 1\n                curr_dict[row[1]][\"part_1_cnt\"] = int(row[4])\n                curr_dict[row[1]][\"part_2_cnt\"] = int(row[5])\n                curr_dict[row[1]][\"part_3_cnt\"] = int(row[6])\n                curr_dict[row[1]][\"part_4_cnt\"] = int(row[7])\n                curr_dict[row[1]][\"part_5_cnt\"] = int(row[8])\n                curr_dict[row[1]][\"part_6_cnt\"] = int(row[9])\n                curr_dict[row[1]][\"part_7_cnt\"] = int(row[10])\n                curr_dict[row[1]][\"type_of_concept_cnt\"] = int(row[11])\n                curr_dict[row[1]][\"type_of_intention_cnt\"] = int(row[12])\n                curr_dict[row[1]][\"type_of_solving_question_cnt\"] = int(row[13])\n                curr_dict[row[1]][\"type_of_starter_cnt\"] = int(row[14])\n                curr_dict[row[1]][\"has_tags\"] = set([int(row[3])])\n            else:\n                curr_dict[row[1]][\"lecture_bool\"] = 0\n                curr_dict[row[1]][\"part_1_cnt\"] = 0\n                curr_dict[row[1]][\"part_2_cnt\"] = 0\n                curr_dict[row[1]][\"part_3_cnt\"] = 0\n                curr_dict[row[1]][\"part_4_cnt\"] = 0\n                curr_dict[row[1]][\"part_5_cnt\"] = 0\n                curr_dict[row[1]][\"part_6_cnt\"] = 0\n                curr_dict[row[1]][\"part_7_cnt\"] = 0\n                curr_dict[row[1]][\"type_of_concept_cnt\"] = 0\n                curr_dict[row[1]][\"type_of_intention_cnt\"] = 0\n                curr_dict[row[1]][\"type_of_solving_question_cnt\"] = 0\n                curr_dict[row[1]][\"type_of_starter_cnt\"] = 0\n                curr_dict[row[1]][\"has_tags\"] = set()\n        \n        ulc_lb[cnt] = curr_dict[row[1]][\"lecture_bool\"]\n        part1_l[cnt] = curr_dict[row[1]][\"part_1_cnt\"]\n        part2_l[cnt] = curr_dict[row[1]][\"part_2_cnt\"]\n        part3_l[cnt] = curr_dict[row[1]][\"part_3_cnt\"]\n        part4_l[cnt] = curr_dict[row[1]][\"part_4_cnt\"]\n        part5_l[cnt] = curr_dict[row[1]][\"part_5_cnt\"]\n        part6_l[cnt] = curr_dict[row[1]][\"part_6_cnt\"]\n        part7_l[cnt] = curr_dict[row[1]][\"part_7_cnt\"]\n        type_of_concept_l[cnt] = curr_dict[row[1]][\"type_of_concept_cnt\"]\n        type_of_intention_l[cnt] = curr_dict[row[1]][\"type_of_intention_cnt\"]\n        type_of_solving_question_l[cnt] = curr_dict[row[1]][\"type_of_solving_question_cnt\"]\n        type_of_starter_l[cnt] = curr_dict[row[1]][\"type_of_starter_cnt\"]\n        \n        if type(row[15]) == list:\n            tags_has = 0\n            for tag in row[15]:\n                if int(tag) in curr_dict[row[1]][\"has_tags\"]:\n                    tags_has += 1\n            has_tags_l[cnt] = tags_has/len(row[15])\n\n    has_tags_lb = (has_tags_l > 0).astype(\"int8\")\n\n    lectures_feats_df = pd.DataFrame({\"curr_lecture_bool\":ulc_lb,\n                                      \"part_1_cnt\":part1_l,\n                                      \"part_2_cnt\":part2_l,\n                                      \"part_3_cnt\":part3_l,\n                                      \"part_4_cnt\":part4_l,\n                                      \"part_5_cnt\":part5_l,\n                                      \"part_6_cnt\":part6_l,\n                                      \"part_7_cnt\":part7_l,\n                                      \"type_of_concept_cnt\":type_of_concept_l,\n                                      \"type_of_intention_cnt\":type_of_intention_l,\n                                      \"type_of_solving_question_cnt\":type_of_solving_question_l,\n                                      \"type_of_starter_cnt\":type_of_starter_l,\n                                      \"watched_tags_rate\":has_tags_l,\n                                      \"watched_tags_bool\":has_tags_lb,\n                                     }).set_index(new_df[\"row_id\"])\n\n    df = df.merge(lectures_feats_df,how=\"left\",left_on=\"row_id\",right_index=True)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.017415,"end_time":"2020-12-16T14:28:19.210425","exception":false,"start_time":"2020-12-16T14:28:19.19301","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Testing\n"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-16T14:28:19.881377Z","iopub.status.busy":"2020-12-16T14:28:19.880486Z","iopub.status.idle":"2020-12-16T14:28:19.882572Z","shell.execute_reply":"2020-12-16T14:28:19.883406Z"},"papermill":{"duration":0.031336,"end_time":"2020-12-16T14:28:19.883592","exception":false,"start_time":"2020-12-16T14:28:19.852256","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-16T14:28:19.960409Z","iopub.status.busy":"2020-12-16T14:28:19.959561Z","iopub.status.idle":"2020-12-16T14:28:20.953772Z","shell.execute_reply":"2020-12-16T14:28:20.952989Z"},"papermill":{"duration":1.048227,"end_time":"2020-12-16T14:28:20.953925","exception":false,"start_time":"2020-12-16T14:28:19.905698","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"previous_test_df = None\nfor (test_df, sample_prediction_df) in iter_test:\n    if previous_test_df is not None:\n        previous_test_df[target] = eval(test_df[\"prior_group_answers_correct\"].iloc[0])\n        update_user_feats(previous_test_df)  # use the function\n        previous_test_df = previous_test_df[previous_test_df[target] != -1]\n        \n    previous_test_df = test_df.copy()\n    \n    test_df = add_lectures_feats(test_df, curr_lectures_dict) # use the function\n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n    test_df = add_user_feats_without_update(test_df) # use the function\n    \n    test_df = add_up_without_update(test_df) # use the function\n    \n    # ...balabalabala...(Other codes omitted)\n\n    test_df['answered_correctly'] = model.predict(test_df[features])\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}