{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/lgbm-inference-db-full-data/pickle5-0.0.11/","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-12T18:21:01.107461Z","iopub.status.busy":"2020-10-12T18:21:01.1067Z","iopub.status.idle":"2020-10-12T18:21:02.19541Z","shell.execute_reply":"2020-10-12T18:21:02.194464Z"},"lines_to_next_cell":2,"papermill":{"duration":1.131405,"end_time":"2020-10-12T18:21:02.195556","exception":false,"start_time":"2020-10-12T18:21:01.064151","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nfrom sklearn.metrics import roc_auc_score\nfrom collections import defaultdict\nfrom tqdm.notebook import tqdm\nimport lightgbm as lgb\nimport pickle5 as pickle\nfrom numba import jit","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from contextlib import contextmanager\nimport time\n\n\n@contextmanager\ndef timer(name):\n    \"\"\"\n    Time Each Process\n    \"\"\"\n    t0 = time.time()\n    yield\n    print('\\n[{}] done in {} Minutes\\n'.format(name, round((time.time() - t0) / 60, 2)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_pickle = '../input/lgbm-inference-db-full-data/train_df.pickle'\nquestion_file = '../input/lgbm-inference-db-full-data/question_features.csv'\n# feature_file = '../input/lgbm-inference-db-full-data/pre_features.csv'\nms_in_a_day = 8.64 * 10 ** 7\n\nprior_question_elapsed_time_mean = 25439.41","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## feature engineering"},{"metadata":{"trusted":true},"cell_type":"code","source":"left_asymptote = 0.25\n\n\n@jit(nopython=True)\ndef get_new_theta(is_good_answer, beta, left_asymptote, theta, nb_previous_answers):\n    return theta + learning_rate_theta(nb_previous_answers) * (\n            is_good_answer - probability_of_good_answer(theta, beta, left_asymptote))\n\n\n@jit(nopython=True)\ndef learning_rate_theta(nb_answers):\n    return max(0.3 / (1 + 0.01 * nb_answers), 0.04)\n\n\n@jit(nopython=True)\ndef probability_of_good_answer(theta, beta, left_asymptote):\n    return left_asymptote + (1 - left_asymptote) * sigmoid(theta - beta)\n\n\n@jit(nopython=True)\ndef sigmoid(x):\n    return 1 / (1 + np.exp(-x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# funcs for user stats with loop\n# def count_attempts(df):\n#     cols = ['user_id', 'content_id', 'content_type_id']\n#     for cnt, row in enumerate(tqdm(df[cols].values)):\n#         if row[2] == 0:\n#             attempt_dict[row[0]][row[1]] += 1\n#     return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calc_user_feats_test(df, bundle_count, temp_values):\n    attempt_no_array = np.zeros(len(df), dtype=np.int32)\n    last_lecture_time_array = np.zeros(len(df), dtype=np.float64)\n    last_incorrect_time_array = np.zeros(len(df), dtype=np.float64)\n    acsu = np.zeros(len(df), dtype=np.int32)\n    acsu_part = np.zeros(len(df), dtype=np.int32)\n    cu = np.zeros(len(df), dtype=np.int32)\n    cu_part = np.zeros(len(df), dtype=np.int32)\n    tu_part = np.zeros(len(df), dtype=np.int32)\n    lag_time_array = np.zeros(len(df), dtype=np.int64)  # diff between timestamps\n    wait_time_array = np.zeros(len(df), dtype=np.float64)  # the time a student waits before starts the next question\n    theta_array = np.zeros(len(df), dtype=np.float32)\n    beta_array = np.zeros(len(df), dtype=np.float32)\n    difficulty_correct_array = np.zeros(len(df), dtype=np.float64)\n    difficulty_incorrect_array = np.zeros(len(df), dtype=np.float64)\n\n    feature_cols = ['user_id', 'prior_question_elapsed_time', 'timestamp',\n                    'content_id', 'content_type_id', 'part', 'bundle_id',\n                    'mean_content_accuracy_sm']\n\n    for cnt, row in enumerate(df[feature_cols].values):\n        if row[2] == 0:\n            lag_time_array[cnt] = 0\n            prior_question_lag_time[row[0]] = np.nan\n            wait_time_array[cnt] = np.nan\n        elif row[2] == user_last_timestamp[row[0]]:  # if question is in the same bundle as the previous one\n            wait_time_array[cnt] = temp_values[5]\n            lag_time_array[cnt] = row[2] - user_last_timestamp_traceback[row[0]]\n        else:\n            lag_time_array[cnt] = row[2] - user_last_timestamp[row[0]]\n            if (lag_time_array[cnt] == 0) | (lag_time_array[cnt] == row[2]) | (len(prior_bundle_count[row[0]]) != 2):\n                wait_time_array[cnt] = np.nan\n            else:\n                wait_time_array[cnt] = prior_question_lag_time[row[0]] - prior_bundle_count[row[0]][1] * row[1]\n\n            user_last_timestamp_traceback[row[0]] = user_last_timestamp[\n                row[0]]  # assign last time stamp to track back dict\n        user_last_timestamp[row[0]] = row[2]  # assign the latest timestamp to user\n\n        # count attempts for the same question, lecture attendance (all) and lecture attendance (solving problem)\n        if row[4] == 1:\n            last_lecture_time[row[0]] = row[3]\n        else:\n            if row[6] in bundles:\n                if len(prior_bundle_count[row[0]]) == 2:\n                    if row[6] == prior_bundle_count[row[0]][0]:\n                        bundle_count += 1\n                        bundle_flg = True\n                        save_temp_value_flg = False\n                    else:\n                        bundle_count = 1\n                        bundle_flg = False\n                        save_temp_value_flg = True\n                else:\n                    bundle_count = 1\n                    bundle_flg = False\n                    save_temp_value_flg = True\n            else:\n                bundle_count = 1\n                bundle_flg = False\n                save_temp_value_flg = False\n\n            prior_question_lag_time[row[0]] = lag_time_array[cnt]\n            if save_temp_value_flg:\n                temp_values[0] = answered_correctly_sum_user_dict['total'][row[0]]\n                temp_values[1] = answered_correctly_sum_user_dict[int(row[5])][row[0]]\n                temp_values[2] = question_count_dict['total'][row[0]]\n                temp_values[3] = question_count_dict[int(row[5])][row[0]]\n                # temp_values[4] = sum(answer_list_20[row[0]])\n                # temp_values[5] = len(answer_list_20[row[0]])\n                temp_values[4] = last_incorrect_time[row[0]]   # 6\n                temp_values[5] = wait_time_array[cnt]    # 7\n                temp_values[6] = beta_dict[row[3]]     #  8\n                temp_values[7] = theta_dict[row[0]]    # 9\n                temp_values[8] = difficulty_dict[row[0]]['correct']\n                temp_values[9] = difficulty_dict[row[0]]['incorrect']\n\n            if bundle_flg:  # assign fixed temp values\n                acsu[cnt] = temp_values[0]\n                cu[cnt] = temp_values[2]\n                difficulty_correct_array[cnt] = temp_values[8]\n                difficulty_incorrect_array[cnt] = temp_values[9]\n\n                # part feature\n                acsu_part[cnt] = temp_values[1]\n                cu_part[cnt] = temp_values[3]\n                \n                theta_array[cnt] = temp_values[7]\n                beta_array[cnt] = temp_values[6]\n\n                if row[2] == 0:\n                    last_incorrect_time_array[cnt] = np.nan\n                else:\n                    last_incorrect_time_array[cnt] = row[2] - temp_values[4]\n\n            else:\n                acsu[cnt] = answered_correctly_sum_user_dict['total'][row[0]]\n                cu[cnt] = question_count_dict['total'][row[0]]\n                difficulty_correct_array[cnt] = difficulty_dict[row[0]]['correct']\n                difficulty_incorrect_array[cnt] = difficulty_dict[row[0]]['incorrect']\n\n                # part feature\n                acsu_part[cnt] = answered_correctly_sum_user_dict[int(row[5])][row[0]]\n                cu_part[cnt] = question_count_dict[int(row[5])][row[0]]\n\n                if row[2] == 0:\n                    last_incorrect_time_array[cnt] = np.nan\n                else:\n                    last_incorrect_time_array[cnt] = row[2] - last_incorrect_time[row[0]]\n\n                # keep track of the last 20 questions\n#                 if len(answer_list_20[row[0]]) == 0:\n#                     last_20_accuracy[cnt] = np.nan\n#                 else:\n#                     last_20_accuracy[cnt] = sum(answer_list_20[row[0]]) / len(answer_list_20[row[0]])\n                \n                # ELO Rating\n                beta_array[cnt] = beta_dict[row[3]]\n                theta_array[cnt] = theta_dict[row[0]]\n                \n            tu_part[cnt] = user_time_dict[int(row[5])][row[0]]\n\n            # count attempts for the same question, lecture attendance, and record last lecture time\n            if bundle_count == 1:\n                attempt_dict[row[0]][row[6]] += 1\n            attempt_no_array[cnt] = attempt_dict[row[0]][row[6]]\n            \n            if last_lecture_time[row[0]] == 0:\n                last_lecture_time_array[cnt] = np.nan\n                # learn_from_lecture_array[cnt] = 0\n            else:\n                last_lecture_time_array[cnt] = row[2] - last_lecture_time[row[0]]\n#                 if last_lecture_time_array[cnt] < ms_in_a_day:\n#                     learn_from_lecture_array[cnt] = (last_lecture_info[row[0]][1] in row[8])\n#                 else:\n#                     learn_from_lecture_array[cnt] = 0\n\n            # keep track of the total time a student has spent on each part and in total\n            if np.isnan(row[1]):\n                user_time_dict[int(row[5])][row[0]] += 0\n            else:\n                user_time_dict[int(row[5])][row[0]] += row[1]\n\n            prior_bundle_count[row[0]] = (row[6], bundle_count)\n\n    df['attempt_no'] = attempt_no_array\n    df['last_lecture_time'] = last_lecture_time_array\n    df['last_incorrect_time'] = last_incorrect_time_array\n    df['lag_time'] = lag_time_array\n    df['prior_question_wait_time'] = wait_time_array\n    df['theta'] = theta_array\n    df['beta'] = beta_array\n\n    user_feats_df = pd.DataFrame({'answered_correctly_sum_user': acsu, 'answered_count': cu,\n                                  'answered_correctly_sum_user_part': acsu_part, 'answered_count_part': cu_part,\n                                  'total_time_spent_user_part': tu_part,\n                                  'difficulty_correct_count': difficulty_correct_array,\n                                  'difficulty_incorrect_count': difficulty_incorrect_array\n                                  })\n\n    user_feats_df['mean_user_accuracy'] = user_feats_df['answered_correctly_sum_user'] / user_feats_df['answered_count']\n    user_feats_df['mean_user_accuracy_part'] = user_feats_df['answered_correctly_sum_user_part'] / user_feats_df[\n        'answered_count_part']\n    user_feats_df['mean_user_spent_time_part'] = user_feats_df['total_time_spent_user_part'] / user_feats_df[\n        'answered_count_part']\n    user_feats_df.loc[user_feats_df['answered_count_part'] == 0, 'mean_user_spent_time_part'] = np.nan\n    user_feats_df['difficulty_correct'] = user_feats_df['difficulty_correct_count'] / user_feats_df['answered_correctly_sum_user']\n    user_feats_df['difficulty_incorrect'] = user_feats_df['difficulty_incorrect_count'] / \\\n                                            (user_feats_df['answered_count'] - user_feats_df['answered_correctly_sum_user'])\n    user_feats_df['difficulty_diff'] = user_feats_df['difficulty_correct'] - user_feats_df['difficulty_incorrect']\n    user_feats_df.drop(columns=['difficulty_correct_count', 'difficulty_incorrect_count', 'difficulty_correct'], inplace=True)\n    \n    feats_cols = user_feats_df.columns\n    for col in feats_cols:\n        df[col] = user_feats_df[col].values\n\n    df['hmean_user_content_accuracy'] = 2 * (df['mean_user_accuracy'] * df['mean_content_accuracy_sm']) / \\\n                                        (df['mean_user_accuracy'] + df['mean_content_accuracy_sm'])\n    # floor the anomalies (?) at zero; could try other values\n    df.loc[df['prior_question_wait_time'] < 0, 'prior_question_wait_time'] = 0\n\n    return df, bundle_count, temp_values\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def update_user_feats(df):\n    bundle_count = 1\n    for row in df[['user_id', 'answered_correctly', 'content_type_id', 'timestamp',\n                   'part', 'content_id', 'answered_count', 'mean_content_accuracy_sm']].values:\n        if row[2] == 0:\n            # cumulatively add values for total & part\n            answered_correctly_sum_user_dict['total'][row[0]] += row[1]\n            answered_correctly_sum_user_dict[int(row[4])][row[0]] += row[1]\n            question_count_dict['total'][row[0]] += 1\n            question_count_dict[int(row[4])][row[0]] += 1\n            \n            theta = theta_dict[row[0]]\n            beta = beta_dict[row[5]]\n            theta_dict[row[0]] = get_new_theta(row[1], beta, left_asymptote, theta, row[6])\n\n            if row[1] == 0:\n                last_incorrect_time[row[0]] = row[3]\n                difficulty_dict[row[0]]['incorrect'] += row[7]\n            else:\n                difficulty_dict[row[0]]['correct'] += row[7]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(train_pickle, 'rb') as file:\n    df = pickle.load(file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def multi_level_dict():\n    return defaultdict(int)\n\nattempt_dict = defaultdict(multi_level_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def multi_level_float_dict():\n    return defaultdict(float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# with timer(\"Counting Attempts\"):\n#     train = count_attempts(train)\n#     del train\n#     gc.collect()\n#     valid = count_attempts(valid)\n#     del valid\n#     gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with timer(\"counting\"):\n    keys = np.sort(df['user_id'].unique())\n    total = len(keys)\n\n    # add user content attempts\n    user_bundle = df.groupby('user_id')['bundle_id'].apply(np.array).apply(np.sort).apply(np.unique)\n    user_attempts = df.groupby(['user_id', 'bundle_id'])['bundle_id'].count().astype(np.uint8).groupby('user_id').apply(np.array)\n\n    for user_id, bundle, attempt in tqdm(zip(keys, user_bundle, user_attempts), total=total):\n        attempt_dict[user_id] = defaultdict(int, zip(bundle, attempt))\n        \ndel user_bundle, user_attempts, df, bundle, attempt\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with timer(\"Loading Data...\"):\n    # answered correctly average for each content\n    # content_df = pd.read_csv('../input/lgbm-inference-db-full-data/content_df.csv', index_col = 0)\n    part_df = pd.read_csv('../input/lgbm-inference-db-full-data/part_df.csv', index_col = 0)\n\n    with open('../input/lgbm-inference-db-full-data/answered_correctly_sum_user_dict.pickle', 'rb') as file:\n        answered_correctly_sum_user_dict = pickle.load(file)\n\n    with open('../input/lgbm-inference-db-full-data/question_count_dict.pickle', 'rb') as file:\n        question_count_dict = pickle.load(file)\n\n    with open('../input/lgbm-inference-db-full-data/user_time_dict.pickle', 'rb') as file:\n        user_time_dict = pickle.load(file)\n\n    with open('../input/lgbm-inference-db-full-data/user_last_timestamp.pickle', 'rb') as file:\n        user_last_timestamp = pickle.load(file)\n\n    with open('../input/lgbm-inference-db-full-data/user_last_timestamp_traceback.pickle', 'rb') as file:\n        user_last_timestamp_traceback = pickle.load(file)\n\n#     with open('../input/lgbm-inference-db-full-data/answer_list_20.pickle', 'rb') as file:\n#         answer_list_20 = pickle.load(file)\n    \n    with open('../input/lgbm-inference-db-full-data/last_lecture_time.pickle', 'rb') as file:\n        last_lecture_time = pickle.load(file)\n        \n    with open('../input/lgbm-inference-db-full-data/last_incorrect_time.pickle', 'rb') as file:\n        last_incorrect_time = pickle.load(file)\n    \n    with open('../input/lgbm-inference-db-full-data/prior_question_lag_time.pickle', 'rb') as file:\n        prior_question_lag_time = pickle.load(file)\n        \n    with open('../input/lgbm-inference-db-full-data/prior_bundle_count.pickle', 'rb') as file:\n        prior_bundle_count = pickle.load(file)\n        \n    with open('../input/lgbm-inference-db-full-data/theta_dict.pickle', 'rb') as file:\n        theta_dict = pickle.load(file)\n        \n    with open('../input/lgbm-inference-db-full-data/beta_dict.pickle', 'rb') as file:\n        beta_dict = pickle.load(file)\n    \n    with open('../input/lgbm-inference-db-full-data/bundles.npy', 'rb') as file:\n        bundles = np.load(file)\n        \n    with open('../input/lgbm-inference-db-full-data/difficulty_dict.pickle', 'rb') as file:\n        difficulty_dict = pickle.load(file)\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# part and tags\nquestions_df = pd.read_csv(question_file, index_col='question_id')\n# questions_df['tags'].fillna('-1', inplace=True)\n# questions_df['tags'] = questions_df['tags'].apply(lambda x: np.array(x.split()).astype(int))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# feature_df = pd.read_csv(feature_file, index_col='content_id')\n# questions_df = questions_df.merge(feature_df, left_index=True, right_index=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## modeling"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"TARGET = 'answered_correctly'\nFEATS_1 = ['mean_user_accuracy',\n         # 'answered_correctly_sum_user',\n         'answered_count',\n         'mean_content_accuracy_sm',\n         'prior_question_elapsed_time',\n         # 'hmean_user_content_accuracy',\n         'last_incorrect_time', 'prior_question_wait_time',\n         # 'prior_question_had_explanation',\n         'content_freq_encoding',\n         'lag_time',\n         'attempt_no', 'last_lecture_time',\n         'mean_user_spent_time_part',\n         'answered_correctly_sum_user_part',\n         'mean_user_accuracy_part',\n         'part', 'theta', 'beta',\n         'question_avg_explanation_sm',\n         'question_avg_elapsed_time_sm',\n         'tags_lsi',\n         'difficulty_incorrect', 'difficulty_diff'\n         ]\n\ncategorical_features = ['part', 'tags_lsi']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FEATS_2 = ['mean_user_accuracy',\n         'answered_correctly_sum_user',\n         'answered_count',\n         'mean_content_accuracy_sm',\n         'prior_question_elapsed_time',\n         'hmean_user_content_accuracy',\n         'last_incorrect_time', 'prior_question_wait_time',\n         # 'prior_question_had_explanation',\n         'content_freq_encoding',\n         'lag_time',\n         'attempt_no', 'last_lecture_time',\n         'mean_user_spent_time_part',\n         'answered_correctly_sum_user_part',\n         'mean_user_accuracy_part',\n         'part', 'theta', 'beta',\n         'question_avg_explanation_sm',\n         'question_avg_elapsed_time_sm',\n         'tags_lsi',\n         'difficulty_incorrect',\n         # 'difficulty_diff'\n         ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_1 = lgb.Booster(model_file='../input/lgbm-inference-db-full-data/lightgbm_v11.5.txt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_2 = lgb.Booster(model_file='../input/lgbm-inference-db-full-data/lightgbm_v11.6.txt')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## inference"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"import riiideducation\nenv = riiideducation.make_env()\niter_test = env.iter_test()\nset_predict = env.predict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# use fast merging method for faster iterations of test\nprevious_test_df = None\nfor (test_df, sample_prediction_df) in iter_test:\n    test_df = pd.concat([test_df.reset_index(drop=True), \n                         questions_df.reindex(test_df['content_id'].values).reset_index(drop=True)], axis=1)\n    test_df = pd.concat([test_df.reset_index(drop=True), \n                         part_df.reindex(test_df['part'].values).reset_index(drop=True)], axis=1)\n    \n    \n    if previous_test_df is not None:\n        previous_test_df[TARGET] = eval(test_df[\"prior_group_answers_correct\"].iloc[0])\n        update_user_feats(previous_test_df)\n    else:\n        bundle_count = 1\n        temp_values = np.empty(10) * np.nan\n    # previous_test_df = test_df.copy()\n    \n    test_df, bundle_count, temp_values = calc_user_feats_test(test_df, bundle_count, temp_values)\n    previous_test_df = test_df.copy()\n    \n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n    test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False).astype('int8')\n    test_df['part'] = test_df.part.fillna(False).astype('int8')\n    test_df['prior_question_elapsed_time'] = test_df.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)\n    \n    for col in categorical_features:\n        test_df[col] = test_df[col].astype('category')\n    \n    test_df[TARGET] =  model_1.predict(test_df[FEATS_1]) * 0.5 + model_2.predict(test_df[FEATS_2]) * 0.5\n    # test_df[TARGET] =  model_1.predict(test_df[FEATS_1])\n    set_predict(test_df[['row_id', TARGET]])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}