{"cells":[{"metadata":{},"cell_type":"markdown","source":"I want to convey two things in this notebook.\n## 1. Don't have to be hesitant about using Loop.\nThey say \"avoid loops!'.\nBut I think It's not bad idea to use loops for this competition.\nBecause:\n* We have to use small batch inference using Time-series API.\n* Loops have very small overhead for each batch.\n* Loops are more flexible.\n* Even loops are not so slow. 3 features are extracted within 10 minits for 100M train data, as you can see blow.\n\n## 2. Future information should not be used.\nTime-series API doesn't allow us to use information from the future.\nSo we should not use it, especially user statistics from future make things very bad."},{"metadata":{"execution":{"iopub.execute_input":"2020-10-12T18:21:01.107461Z","iopub.status.busy":"2020-10-12T18:21:01.1067Z","iopub.status.idle":"2020-10-12T18:21:02.19541Z","shell.execute_reply":"2020-10-12T18:21:02.194464Z"},"lines_to_next_cell":2,"papermill":{"duration":1.131405,"end_time":"2020-10-12T18:21:02.195556","exception":false,"start_time":"2020-10-12T18:21:01.064151","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nfrom sklearn.metrics import roc_auc_score\nfrom collections import defaultdict\nfrom tqdm.notebook import tqdm\nimport lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## setting\nCV files are generated by [this notebook](https://www.kaggle.com/its7171/cv-strategy)"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_pickle = '../input/riiid-cross-validation-files/cv1_train.pickle'\nvalid_pickle = '../input/riiid-cross-validation-files/cv1_valid.pickle'\nquestion_file = '../input/riiid-test-answer-prediction/questions.csv'\ndebug = False\nvalidaten_flg = False","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## feature engineering"},{"metadata":{"trusted":true},"cell_type":"code","source":"# funcs for user stats with loop\ndef add_user_feats(df, answered_correctly_sum_u_dict, count_u_dict):\n    acsu = np.zeros(len(df), dtype=np.int32)\n    cu = np.zeros(len(df), dtype=np.int32)\n    for cnt,row in enumerate(tqdm(df[['user_id','answered_correctly']].values)):\n        acsu[cnt] = answered_correctly_sum_u_dict[row[0]]\n        cu[cnt] = count_u_dict[row[0]]\n        answered_correctly_sum_u_dict[row[0]] += row[1]\n        count_u_dict[row[0]] += 1\n    user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})\n    user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']\n    df = pd.concat([df, user_feats_df], axis=1)\n    return df\n\ndef add_user_feats_without_update(df, answered_correctly_sum_u_dict, count_u_dict):\n    acsu = np.zeros(len(df), dtype=np.int32)\n    cu = np.zeros(len(df), dtype=np.int32)\n    for cnt,row in enumerate(df[['user_id']].values):\n        acsu[cnt] = answered_correctly_sum_u_dict[row[0]]\n        cu[cnt] = count_u_dict[row[0]]\n    user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})\n    user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']\n    df = pd.concat([df, user_feats_df], axis=1)\n    return df\n\ndef update_user_feats(df, answered_correctly_sum_u_dict, count_u_dict):\n    for row in df[['user_id','answered_correctly','content_type_id']].values:\n        if row[2] == 0:\n            answered_correctly_sum_u_dict[row[0]] += row[1]\n            count_u_dict[row[0]] += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# read data\nfeld_needed = ['row_id', 'user_id', 'content_id', 'content_type_id', 'answered_correctly', 'prior_question_elapsed_time', 'prior_question_had_explanation']\ntrain = pd.read_pickle(train_pickle)[feld_needed]\nvalid = pd.read_pickle(valid_pickle)[feld_needed]\nif debug:\n    train = train[:1000000]\n    valid = valid[:10000]\ntrain = train.loc[train.content_type_id == False].reset_index(drop=True)\nvalid = valid.loc[valid.content_type_id == False].reset_index(drop=True)\n\n# answered correctly average for each content\ncontent_df = train[['content_id','answered_correctly']].groupby(['content_id']).agg(['mean']).reset_index()\ncontent_df.columns = ['content_id', 'answered_correctly_avg_c']\ntrain = pd.merge(train, content_df, on=['content_id'], how=\"left\")\nvalid = pd.merge(valid, content_df, on=['content_id'], how=\"left\")\n\n# user stats features with loops\nanswered_correctly_sum_u_dict = defaultdict(int)\ncount_u_dict = defaultdict(int)\ntrain = add_user_feats(train, answered_correctly_sum_u_dict, count_u_dict)\nvalid = add_user_feats(valid, answered_correctly_sum_u_dict, count_u_dict)\n\n# fill with mean value for prior_question_elapsed_time\n# note that `train.prior_question_elapsed_time.mean()` dose not work!\n# please refer https://www.kaggle.com/its7171/can-we-trust-pandas-mean for detail.\nprior_question_elapsed_time_mean = train.prior_question_elapsed_time.dropna().values.mean()\ntrain['prior_question_elapsed_time_mean'] = train.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)\nvalid['prior_question_elapsed_time_mean'] = valid.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)\n\n# use only last 30M training data for limited memory on kaggle env.\n#train = train[-30000000:]\n\n# part\nquestions_df = pd.read_csv(question_file)\ntrain = pd.merge(train, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')\nvalid = pd.merge(valid, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n\n# changing dtype to avoid lightgbm error\ntrain['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False).astype('int8')\nvalid['prior_question_had_explanation'] = valid.prior_question_had_explanation.fillna(False).astype('int8')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## modeling"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"TARGET = 'answered_correctly'\nFEATS = ['answered_correctly_avg_u', 'answered_correctly_sum_u', 'count_u', 'answered_correctly_avg_c', 'part', 'prior_question_had_explanation', 'prior_question_elapsed_time']\ndro_cols = list(set(train.columns) - set(FEATS))\ny_tr = train[TARGET]\ny_va = valid[TARGET]\ntrain.drop(dro_cols, axis=1, inplace=True)\nvalid.drop(dro_cols, axis=1, inplace=True)\n_=gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_train = lgb.Dataset(train[FEATS], y_tr)\nlgb_valid = lgb.Dataset(valid[FEATS], y_va)\ndel train, y_tr\n_=gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = lgb.train(\n                    {'objective': 'binary'}, \n                    lgb_train,\n                    valid_sets=[lgb_train, lgb_valid],\n                    verbose_eval=100,\n                    num_boost_round=10000,\n                    early_stopping_rounds=10\n                )\nprint('auc:', roc_auc_score(y_va, model.predict(valid[FEATS])))\n_ = lgb.plot_importance(model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## inference"},{"metadata":{"execution":{"iopub.execute_input":"2020-10-12T18:42:48.029704Z","iopub.status.busy":"2020-10-12T18:42:48.028875Z","iopub.status.idle":"2020-10-12T18:42:48.032215Z","shell.execute_reply":"2020-10-12T18:42:48.031426Z"},"papermill":{"duration":0.07062,"end_time":"2020-10-12T18:42:48.032349","exception":false,"start_time":"2020-10-12T18:42:47.961729","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class Iter_Valid(object):\n    def __init__(self, df, max_user=1000):\n        df = df.reset_index(drop=True)\n        self.df = df\n        self.user_answer = df['user_answer'].astype(str).values\n        self.answered_correctly = df['answered_correctly'].astype(str).values\n        df['prior_group_responses'] = \"[]\"\n        df['prior_group_answers_correct'] = \"[]\"\n        self.sample_df = df[df['content_type_id'] == 0][['row_id']]\n        self.sample_df['answered_correctly'] = 0\n        self.len = len(df)\n        self.user_id = df.user_id.values\n        self.task_container_id = df.task_container_id.values\n        self.content_type_id = df.content_type_id.values\n        self.max_user = max_user\n        self.current = 0\n        self.pre_user_answer_list = []\n        self.pre_answered_correctly_list = []\n\n    def __iter__(self):\n        return self\n    \n    def fix_df(self, user_answer_list, answered_correctly_list, pre_start):\n        df= self.df[pre_start:self.current].copy()\n        sample_df = self.sample_df[pre_start:self.current].copy()\n        df.loc[pre_start,'prior_group_responses'] = '[' + \",\".join(self.pre_user_answer_list) + ']'\n        df.loc[pre_start,'prior_group_answers_correct'] = '[' + \",\".join(self.pre_answered_correctly_list) + ']'\n        self.pre_user_answer_list = user_answer_list\n        self.pre_answered_correctly_list = answered_correctly_list\n        return df, sample_df\n\n    def __next__(self):\n        added_user = set()\n        pre_start = self.current\n        pre_added_user = -1\n        pre_task_container_id = -1\n        pre_content_type_id = -1\n        user_answer_list = []\n        answered_correctly_list = []\n        while self.current < self.len:\n            crr_user_id = self.user_id[self.current]\n            crr_task_container_id = self.task_container_id[self.current]\n            crr_content_type_id = self.content_type_id[self.current]\n            if crr_user_id in added_user and (crr_user_id != pre_added_user or (crr_task_container_id != pre_task_container_id and crr_content_type_id == 0 and pre_content_type_id == 0)):\n                # known user(not prev user or (differnt task container and both question))\n                return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n            if len(added_user) == self.max_user:\n                if  crr_user_id == pre_added_user and (crr_task_container_id == pre_task_container_id or crr_content_type_id == 1):\n                    user_answer_list.append(self.user_answer[self.current])\n                    answered_correctly_list.append(self.answered_correctly[self.current])\n                    self.current += 1\n                    continue\n                else:\n                    return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n            added_user.add(crr_user_id)\n            pre_added_user = crr_user_id\n            pre_task_container_id = crr_task_container_id\n            pre_content_type_id = crr_content_type_id\n            user_answer_list.append(self.user_answer[self.current])\n            answered_correctly_list.append(self.answered_correctly[self.current])\n            self.current += 1\n        if pre_start < self.current:\n            return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n        else:\n            raise StopIteration()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# You can debug your inference code to reduce \"Submission Scoring Error\" with `validaten_flg = True`.\n# Please refer https://www.kaggle.com/its7171/time-series-api-iter-test-emulator about Time-series API (iter_test) Emulator.\n\nif validaten_flg:\n    target_df = pd.read_pickle(valid_pickle)\n    if debug:\n        target_df = target_df[:10000]\n    iter_test = Iter_Valid(target_df,max_user=1000)\n    predicted = []\n    def set_predict(df):\n        predicted.append(df)\n    # reset answered_correctly_sum_u_dict and count_u_dict\n    answered_correctly_sum_u_dict = defaultdict(int)\n    count_u_dict = defaultdict(int)\n    train = pd.read_pickle(train_pickle)[['user_id','answered_correctly','content_type_id']]\n    if debug:\n        train = train[:1000000]\n    train = train[train.content_type_id == False].reset_index(drop=True)\n    update_user_feats(train, answered_correctly_sum_u_dict, count_u_dict)\n    del train\nelse:\n    import riiideducation\n    env = riiideducation.make_env()\n    iter_test = env.iter_test()\n    set_predict = env.predict","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-12T18:42:48.174757Z","iopub.status.busy":"2020-10-12T18:42:48.173901Z","iopub.status.idle":"2020-10-12T18:42:49.112276Z","shell.execute_reply":"2020-10-12T18:42:49.111472Z"},"papermill":{"duration":1.016814,"end_time":"2020-10-12T18:42:49.112404","exception":false,"start_time":"2020-10-12T18:42:48.09559","status":"completed"},"scrolled":true,"tags":[],"trusted":true},"cell_type":"code","source":"previous_test_df = None\nfor (test_df, sample_prediction_df) in iter_test:\n    if previous_test_df is not None:\n        previous_test_df[TARGET] = eval(test_df[\"prior_group_answers_correct\"].iloc[0])\n        update_user_feats(previous_test_df, answered_correctly_sum_u_dict, count_u_dict)\n    previous_test_df = test_df.copy()\n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n    test_df = add_user_feats_without_update(test_df, answered_correctly_sum_u_dict, count_u_dict)\n    test_df = pd.merge(test_df, content_df, on='content_id',  how=\"left\")\n    test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')\n    test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False).astype('int8')\n    test_df['prior_question_elapsed_time_mean'] = test_df.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)\n    test_df[TARGET] =  model.predict(test_df[FEATS])\n    set_predict(test_df[['row_id', TARGET]])","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"if validaten_flg:\n    y_true = target_df[target_df.content_type_id == 0].answered_correctly\n    y_pred = pd.concat(predicted).answered_correctly\n    print(roc_auc_score(y_true, y_pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Have a fun with loops! :)"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}