{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Reading Data and Importing Libraries ##"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nfrom sklearn.metrics import roc_auc_score\nfrom collections import defaultdict\nfrom tqdm.notebook import tqdm\nimport lightgbm as lgb\n\nimport random\nimport os\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"init_state = [1, 0, 1]\nkernel = init_state[0]\ndebug = init_state[1]\nfeature_engineering = init_state[2]\n# Random seed\nSEED = 123\n\n# Function to seed everything\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nseed_everything(SEED)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_features(df, answer_correctly_u_count_dict, answer_u_count_dict, answer_uq_count_dict, answer_correct_uq_count_dict,\n                 answer_incorrect_uq_count_dict, explanation_u_count_dict, elapsed_time_u_count_dict, answer_recent_n_correct_u_avg_dict, \n                 answer_recent_u_timestamp_dict, answer_incorrect_recent_u_timestamp_dict,answer_q_count_dict,elapsed_time_q_total_dict,recent_n=5,do_update=True,make_feature=True,inference=False):\n    answer_u_count = np.zeros(len(df), dtype=np.int32)\n    answer_correctly_u_count = np.zeros(len(df), dtype=np.int32)\n    answer_correctly_u_avg = np.zeros(len(df), dtype=np.float32)\n    explanation_u_avg = np.zeros(len(df), dtype=np.float32)\n    elapsed_time_u_avg = np.zeros(len(df), dtype=np.float32)\n    answer_recent_n_correct_u_avg = np.zeros(len(df), dtype=np.float32)\n    answer_recent_u_timestamp = np.zeros(len(df), dtype=np.float32)\n    answer_incorrect_recent_u_timestamp = np.zeros(len(df), dtype=np.float32)\n    elapsed_time_q_avg=np.zeros(len(df),dtype=np.float32)\n    \n    answer_uq_count = np.zeros(len(df), dtype=np.int32)\n#     answer_correct_uq_count = np.zeros(len(df), dtype=np.int32)\n#     answer_incorrect_uq_count = np.zeros(len(df), dtype=np.int32)\n\n    if not inference:\n        df_fields=['user_id','content_id','answered_correctly','prior_question_had_explanation','prior_question_elapsed_time','timestamp']\n    else:\n        df_fields=['user_id','content_id','timestamp']\n    \n    fdict={}\n    for ind,item in enumerate(df_fields):\n        fdict[item]=ind\n    \n    for cnt, row in enumerate(tqdm(df[df_fields].values)):\n        if make_feature:\n            # record user features\n            answer_u_count[cnt] = answer_u_count_dict[row[fdict['user_id']]]\n            answer_correctly_u_count[cnt] = answer_correctly_u_count_dict[row[fdict['user_id']]]\n            answer_correctly_u_avg[cnt] = answer_correctly_u_count_dict[row[fdict['user_id']]]/answer_u_count_dict[row[fdict['user_id']]] if answer_u_count_dict[row[fdict['user_id']]] else np.nan\n            explanation_u_avg[cnt] = explanation_u_count_dict[row[fdict['user_id']]]/answer_u_count_dict[row[fdict['user_id']]] if answer_u_count_dict[row[fdict['user_id']]] else np.nan\n            elapsed_time_u_avg[cnt] = elapsed_time_u_count_dict[row[fdict['user_id']]]/answer_u_count_dict[row[fdict['user_id']]] if answer_u_count_dict[row[fdict['user_id']]] else np.nan\n            elapsed_time_q_avg[cnt]=elapsed_time_q_total_dict[row[fdict['content_id']]]/answer_q_count_dict[row[fdict['content_id']]] if answer_q_count_dict[row[fdict['content_id']]] else np.nan\n            if answer_u_count_dict[row[fdict['user_id']]] <=recent_n:\n                answer_recent_n_correct_u_avg[cnt] = answer_correctly_u_avg[cnt]\n            else:\n                answer_recent_n_correct_u_avg[cnt] = answer_recent_n_correct_u_avg_dict[row[fdict['user_id']]]\n            if len(answer_recent_u_timestamp_dict[row[fdict['user_id']]]) == 0:\n                answer_recent_u_timestamp[cnt] = np.nan\n            else:\n                answer_recent_u_timestamp[cnt] = row[fdict['timestamp']] - answer_recent_u_timestamp_dict[row[fdict['user_id']]][0]\n                \n            if len(answer_incorrect_recent_u_timestamp_dict[row[fdict['user_id']]]) == 0:\n                answer_incorrect_recent_u_timestamp[cnt] = np.nan\n            else:\n                answer_incorrect_recent_u_timestamp[cnt] = row[fdict['timestamp']] - answer_incorrect_recent_u_timestamp_dict[row[fdict['user_id']]][0]\n                \n            # record user features\n            answer_uq_count[cnt] = answer_uq_count_dict[row[fdict['user_id']]][row[fdict['content_id']]]\n#             answer_correct_uq_count[cnt] = answer_correct_uq_count_dict[row[fdict['user_id']]][row[fdict['content_id']]]\n#             answer_incorrect_uq_count[cnt] = answer_incorrect_uq_count_dict[row[fdict['user_id']]][row[fdict['content_id']]]\n\n        if do_update:\n            # update user dict\n            answer_u_count_dict[row[fdict['user_id']]]+=1\n            answer_correctly_u_count_dict[row[fdict['user_id']]]+=row[fdict['answered_correctly']]\n            explanation_u_count_dict[row[fdict['user_id']]]+=row[fdict['prior_question_had_explanation']]\n            elapsed_time_u_count_dict[row[fdict['user_id']]]+=row[fdict['prior_question_elapsed_time']]\n            answer_q_count_dict[row[fdict['content_id']]]+=1\n            elapsed_time_q_total_dict[row[fdict['content_id']]]+=row[fdict['prior_question_elapsed_time']]\n            if answer_u_count_dict[row[fdict['user_id']]] == recent_n:\n                answer_recent_n_correct_u_avg_dict[row[fdict['user_id']]] = answer_correctly_u_count_dict[row[fdict['user_id']]]/answer_u_count_dict[row[fdict['user_id']]] if answer_u_count_dict[row[fdict['user_id']]] else np.nan\n            elif answer_u_count_dict[row[fdict['user_id']]] > recent_n:\n                answer_recent_n_correct_u_avg_dict[row[fdict['user_id']]] = (answer_recent_n_correct_u_avg_dict[row[fdict['user_id']]]*(recent_n-1)+row[fdict['answered_correctly']])/recent_n\n                \n            if len(answer_recent_u_timestamp_dict[row[fdict['user_id']]]) == 0:\n                answer_recent_u_timestamp_dict[row[fdict['user_id']]].append(row[fdict['timestamp']])\n            else:\n                answer_recent_u_timestamp_dict[row[fdict['user_id']]].pop()\n                answer_recent_u_timestamp_dict[row[fdict['user_id']]].append(row[fdict['timestamp']])\n            \n            # update user_question dict\n            answer_uq_count_dict[row[fdict['user_id']]][row[fdict['content_id']]] += 1\n            if row[fdict['answered_correctly']]:\n    #             answer_correct_uq_count_dict[row[fdict['user_id']]][row[fdict['content_id']]] += 1\n                pass\n            else:\n    #             answer_incorrect_uq_count_dict[row[fdict['user_id']]][row[fdict['content_id']]] += 1\n                if len(answer_incorrect_recent_u_timestamp_dict[row[fdict['user_id']]]) == 0:\n                    answer_incorrect_recent_u_timestamp_dict[row[fdict['user_id']]].append(row[fdict['timestamp']])\n                else:\n                    answer_incorrect_recent_u_timestamp_dict[row[fdict['user_id']]].pop()\n                    answer_incorrect_recent_u_timestamp_dict[row[fdict['user_id']]].append(row[fdict['timestamp']])\n    \n    if not make_feature:\n        return None\n    \n    # concat user features\n    user_features_df = pd.DataFrame({'answer_correctly_u_count':answer_correctly_u_count, 'answer_u_count': answer_u_count,\n                                     'answer_correctly_u_avg':answer_correctly_u_avg, 'answer_recent_n_correct_u_avg': answer_recent_n_correct_u_avg,\n                                     'answer_recent_u_timestamp': answer_recent_u_timestamp, 'answer_incorrect_recent_u_timestamp': answer_incorrect_recent_u_timestamp,\n                                     'answer_uq_count':answer_uq_count, \n                                     'explanation_u_avg':explanation_u_avg,\n                                     'elapsed_time_u_avg': elapsed_time_u_avg,\n                                     'elapsed_time_q_avg': elapsed_time_q_avg\n                                    })\n    \n    #user_features_df['answer_correctly_u_avg'] = user_features_df['answer_correctly_u_count']/user_features_df['answer_u_count']\n    df = pd.concat([df, user_features_df], axis=1)\n    return df\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_and_preprocess(kernel=False, debug=True, feature_engineering=True):\n    print(\"***** Read Data\")\n    if kernel:\n        train_pickle = '../input/riiid-cross-validation-files/cv1_train.pickle'\n        valid_pickle = '../input/riiid-cross-validation-files/cv1_valid.pickle'\n        question_file = '../input/riiid-test-answer-prediction/questions.csv'\n    else:\n        train_pickle = './CVdata/cv1_train.pickle'\n        valid_pickle = './CVdata/cv1_valid.pickle'\n        question_file = './data/questions.csv'\n        \n    pickle_features = ['timestamp', 'row_id', 'user_id', 'content_id', 'content_type_id', 'answered_correctly', 'prior_question_elapsed_time', 'prior_question_had_explanation']\n    train = pd.read_pickle(train_pickle)[pickle_features]\n    valid = pd.read_pickle(valid_pickle)[pickle_features]\n\n    if debug:\n        train = train[:1000000]\n        valid = valid[:10000]\n    elif feature_engineering:\n        train = train.iloc[-40000000:]\n    \n    train = train.loc[train.content_type_id == False].reset_index(drop=True)\n    valid = valid.loc[valid.content_type_id == False].reset_index(drop=True)\n\n    questions_df = pd.read_csv(question_file)[['question_id', 'part']]\n    questions_df['part'] = questions_df['part'].astype(np.int32)\n    train = pd.merge(train, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n    valid = pd.merge(valid, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n\n    # fillna\n    prior_question_elapsed_time_mean = train.prior_question_elapsed_time.dropna().values.mean()\n    train['prior_question_elapsed_time'] = train.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)\n    valid['prior_question_elapsed_time'] = valid.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)\n\n    train['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False).astype('int8')\n    valid['prior_question_had_explanation'] = valid.prior_question_had_explanation.fillna(False).astype('int8')\n\n    # answered correctly average for each content\n    content_df = train[['content_id','answered_correctly']].groupby(['content_id']).agg(['mean']).reset_index()\n    content_df.columns = ['content_id', 'answer_correctly_q_avg']\n    train = pd.merge(train, content_df, on=['content_id'], how=\"left\")\n    valid = pd.merge(valid, content_df, on=['content_id'], how=\"left\")\n\n    # record for user    \n    answer_correctly_u_count_dict = defaultdict(int)\n    answer_u_count_dict = defaultdict(int)\n    explanation_u_count_dict = defaultdict(int)\n    elapsed_time_u_count_dict = defaultdict(int)\n    answer_recent_n_correct_u_avg_dict = defaultdict(float)\n    answer_recent_u_timestamp_dict = defaultdict(list)\n    answer_incorrect_recent_u_timestamp_dict = defaultdict(list)\n\n    # record for user_question\n    answer_uq_count_dict = defaultdict(lambda: defaultdict(int))\n    answer_correct_uq_count_dict = defaultdict(lambda: defaultdict(int))\n    answer_incorrect_uq_count_dict = defaultdict(lambda: defaultdict(int))\n    answer_q_count_dict=defaultdict(int)\n    elapsed_time_q_total_dict=defaultdict(float)\n\n    print(\"***** Add Features\")\n    train = add_features(train, answer_correctly_u_count_dict, answer_u_count_dict, answer_uq_count_dict, answer_correct_uq_count_dict,\n                         answer_incorrect_uq_count_dict,explanation_u_count_dict,elapsed_time_u_count_dict, answer_recent_n_correct_u_avg_dict,\n                         answer_recent_u_timestamp_dict, answer_incorrect_recent_u_timestamp_dict,answer_q_count_dict,elapsed_time_q_total_dict)\n    valid = add_features(valid, answer_correctly_u_count_dict, answer_u_count_dict, answer_uq_count_dict, answer_correct_uq_count_dict,\n                         answer_incorrect_uq_count_dict,explanation_u_count_dict,elapsed_time_u_count_dict, answer_recent_n_correct_u_avg_dict,\n                         answer_recent_u_timestamp_dict, answer_incorrect_recent_u_timestamp_dict,answer_q_count_dict,elapsed_time_q_total_dict)\n    gc.collect()\n    \n    features_dicts = {\n        'answer_correctly_u_count_dict': answer_correctly_u_count_dict,\n        'answer_u_count_dict': answer_u_count_dict,\n        'answer_uq_count_dict': answer_uq_count_dict,\n        'answer_correct_uq_count_dict': answer_correct_uq_count_dict,\n        'answer_incorrect_uq_count_dict': answer_incorrect_uq_count_dict,\n        'explanation_u_count_dict': explanation_u_count_dict,\n        'elapsed_time_u_count_dict': elapsed_time_u_count_dict,\n        'answer_recent_n_correct_u_avg_dict': answer_recent_n_correct_u_avg_dict,\n        'answer_recent_u_timestamp_dict': answer_recent_u_timestamp_dict,\n        'answer_incorrect_recent_u_timestamp_dict': answer_incorrect_recent_u_timestamp_dict,\n        'answer_q_count_dict': answer_q_count_dict,\n        'elapsed_time_q_total_dict': elapsed_time_q_total_dict\n    }\n    \n    return train, valid,content_df, questions_df, prior_question_elapsed_time_mean, features_dicts\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_and_evaluate(train, valid, debug=True, feature_engineering = True):\n    print(\"***** Train\")\n    TARGET = 'answered_correctly'\n    FEATS = ['answer_correctly_u_count','answer_u_count','answer_correctly_u_avg','answer_correctly_q_avg','prior_question_elapsed_time',\n             'prior_question_had_explanation', 'part']\n    new_FEATS = ['answer_uq_count', 'explanation_u_avg', 'elapsed_time_u_avg', 'answer_recent_n_correct_u_avg','answer_recent_u_timestamp',\n                 'answer_incorrect_recent_u_timestamp','elapsed_time_q_avg']\n    unused_FEATS = ['answer_incorrect_uq_count','answer_correct_uq_count']\n    FEATS = FEATS+new_FEATS\n    print('Features for train: ', FEATS)\n\n    drop_cols = list(set(train.columns)-set(FEATS))\n    print(\"Drop_cols: \", drop_cols)\n\n    if feature_engineering and not debug:\n        train = train.sample(15000000, random_state = SEED)\n    print(\"Data size: \", len(train), len(valid))\n    y_train = train[TARGET]\n    y_valid = valid[TARGET]\n    train.drop(drop_cols, axis=1, inplace=True)\n    valid.drop(drop_cols, axis=1, inplace=True)\n    _ = gc.collect()\n    \n    lgb_train = lgb.Dataset(train[FEATS], y_train)\n    lgb_valid = lgb.Dataset(valid[FEATS], y_valid)\n    del train, y_train\n    \n    model = lgb.train(\n                        {'objective': 'binary'}, \n                        lgb_train,\n                        valid_sets=[lgb_train, lgb_valid],\n                        verbose_eval=100,\n                        num_boost_round=10000,\n                        early_stopping_rounds=10\n                    )\n    print('auc:', roc_auc_score(y_valid, model.predict(valid[FEATS])))\n    _ = lgb.plot_importance(model)\n    return TARGET, FEATS, model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference(TARGET, FEATS, model,content_df, questions_df, prior_question_elapsed_time_mean, features_dicts):  \n    answer_correctly_u_count_dict = features_dicts['answer_correctly_u_count_dict']\n    answer_u_count_dict = features_dicts['answer_u_count_dict']\n    answer_uq_count_dict = features_dicts['answer_uq_count_dict']\n    answer_correct_uq_count_dict = features_dicts['answer_correct_uq_count_dict']\n    answer_incorrect_uq_count_dict = features_dicts['answer_incorrect_uq_count_dict']\n    explanation_u_count_dict = features_dicts['explanation_u_count_dict']\n    elapsed_time_u_count_dict = features_dicts['elapsed_time_u_count_dict']\n    answer_recent_n_correct_u_avg_dict = features_dicts['answer_recent_n_correct_u_avg_dict']\n    answer_recent_u_timestamp_dict = features_dicts['answer_recent_u_timestamp_dict']\n    answer_incorrect_recent_u_timestamp_dict = features_dicts['answer_incorrect_recent_u_timestamp_dict']\n    answer_q_count_dict=features_dicts['answer_q_count_dict']\n    elapsed_time_q_total_dict=features_dicts['elapsed_time_q_total_dict']\n    \n    import riiideducation\n    env = riiideducation.make_env()\n    iter_test = env.iter_test()\n    set_predict = env.predict\n\n    previous_test_df = None\n    for (test_df, sample_prediction_df) in iter_test:\n        if previous_test_df is not None:\n            previous_test_df[TARGET] = eval(test_df[\"prior_group_answers_correct\"].iloc[0])\n            previous_test_df = previous_test_df[previous_test_df['content_type_id'] == 0].reset_index(drop=True)\n            add_features(previous_test_df, answer_correctly_u_count_dict, answer_u_count_dict, answer_uq_count_dict, answer_correct_uq_count_dict,\n                         answer_incorrect_uq_count_dict,explanation_u_count_dict,elapsed_time_u_count_dict, answer_recent_n_correct_u_avg_dict,\n                         answer_recent_u_timestamp_dict, answer_incorrect_recent_u_timestamp_dict,answer_q_count_dict,elapsed_time_q_total_dict,make_feature=False)\n        test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False).astype('int8')\n        test_df['prior_question_elapsed_time_mean'] = test_df.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)\n        previous_test_df = test_df.copy()\n        test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n        test_df = add_features(test_df, answer_correctly_u_count_dict, answer_u_count_dict, answer_uq_count_dict, answer_correct_uq_count_dict,\n                         answer_incorrect_uq_count_dict,explanation_u_count_dict,elapsed_time_u_count_dict, answer_recent_n_correct_u_avg_dict,\n                         answer_recent_u_timestamp_dict, answer_incorrect_recent_u_timestamp_dict,answer_q_count_dict,elapsed_time_q_total_dict,do_update=False,inference=True)\n        test_df = pd.merge(test_df, content_df, on='content_id',  how=\"left\")\n        test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')\n        test_df[TARGET] =  model.predict(test_df[FEATS])\n\n        set_predict(test_df[['row_id', TARGET]])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train, valid,content_df, questions_df, prior_question_elapsed_time_mean, features_dicts = read_and_preprocess(kernel=kernel, debug = debug, feature_engineering = feature_engineering)\ntrain.tail(10)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TARGET, FEATURES, model = train_and_evaluate(train, valid, feature_engineering = feature_engineering, debug=debug)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if kernel:\n    inference(TARGET, FEATURES, model, content_df, questions_df, prior_question_elapsed_time_mean, features_dicts)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}