{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"I want to convey two things in this notebook.\n## 1. Don't have to be hesitant about using Loop.\nThey say \"avoid loops!'.\nBut I think It's not bad idea to use loops for this competition.\nBecause:\n* We have to use small batch inference using Time-series API.\n* Loops have very small overhead for each batch.\n* Loops are more flexible.\n* Even loops are not so slow. 3 features are extracted within 10 minits for 100M train data, as you can see blow.\n\n## 2. Future information should not be used.\nTime-series API doesn't allow us to use information from the future.\nSo we should not use it, especially user statistics from future make things very bad.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc#廢資料回收\nfrom sklearn.metrics import roc_auc_score#計算正確率\nfrom collections import defaultdict #dict 的子類別，當值不存在 dict 中時會呼叫一個提供預設值的工廠函式\nfrom tqdm.notebook import tqdm #顯示進度條\nimport lightgbm as lgb","metadata":{"lines_to_next_cell":2,"papermill":{"duration":1.131405,"end_time":"2020-10-12T18:21:02.195556","exception":false,"start_time":"2020-10-12T18:21:01.064151","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-14T15:48:32.865915Z","iopub.execute_input":"2022-05-14T15:48:32.866687Z","iopub.status.idle":"2022-05-14T15:48:33.950173Z","shell.execute_reply.started":"2022-05-14T15:48:32.866631Z","shell.execute_reply":"2022-05-14T15:48:33.948631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## setting\nCV files are generated by [this notebook](https://www.kaggle.com/its7171/cv-strategy)","metadata":{}},{"cell_type":"code","source":"train_pickle = '../input/riiid-cross-validation-files/cv1_train.pickle'\nvalid_pickle = '../input/riiid-cross-validation-files/cv1_valid.pickle'\nquestion_file = '../input/riiid-test-answer-prediction/questions.csv'\n#dubug、validaten 除錯、驗證的開關\ndebug = True\nvalidaten_flg = False","metadata":{"execution":{"iopub.status.busy":"2022-05-14T15:48:33.952386Z","iopub.execute_input":"2022-05-14T15:48:33.952756Z","iopub.status.idle":"2022-05-14T15:48:33.958659Z","shell.execute_reply.started":"2022-05-14T15:48:33.952718Z","shell.execute_reply":"2022-05-14T15:48:33.957695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## feature engineering","metadata":{}},{"cell_type":"code","source":"# funcs for user stats with loop 利用迴圈 統計用戶數據\n#先定義好函數方便回call\ndef add_user_feats(df, answered_correctly_sum_u_dict, count_u_dict):\n    #建立一個df長度的陣列，資料型態為int\n    acsu = np.zeros(len(df), dtype=np.int32)\n    cu = np.zeros(len(df), dtype=np.int32)\n    #加總答題正確率、總答題數\n    for cnt,row in enumerate(tqdm(df[['user_id','answered_correctly']].values)):\n        acsu[cnt] = answered_correctly_sum_u_dict[row[0]]\n        cu[cnt] = count_u_dict[row[0]]\n        answered_correctly_sum_u_dict[row[0]] += row[1]\n        count_u_dict[row[0]] += 1\n    #把answered_correctly_sum_u、answered_correctly_avg_u、count_u 資料欄為接起來\n    user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})\n    user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']\n    df = pd.concat([df, user_feats_df], axis=1)\n    return df\n\n#因為官方給的API中iter_test已經有紀錄上次的答題的狀況所以函數不需再計算。\n\ndef add_user_feats_without_update(df, answered_correctly_sum_u_dict, count_u_dict):\n    acsu = np.zeros(len(df), dtype=np.int32)\n    cu = np.zeros(len(df), dtype=np.int32)\n    for cnt,row in enumerate(df[['user_id']].values):\n        acsu[cnt] = answered_correctly_sum_u_dict[row[0]]\n        cu[cnt] = count_u_dict[row[0]]\n    user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})\n    user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']\n    df = pd.concat([df, user_feats_df], axis=1)\n    return df\n\ndef update_user_feats(df, answered_correctly_sum_u_dict, count_u_dict):\n    for row in df[['user_id','answered_correctly','content_type_id']].values:\n        if row[2] == 0:\n            answered_correctly_sum_u_dict[row[0]] += row[1]\n            count_u_dict[row[0]] += 1","metadata":{"execution":{"iopub.status.busy":"2022-05-14T15:48:33.960525Z","iopub.execute_input":"2022-05-14T15:48:33.961298Z","iopub.status.idle":"2022-05-14T15:48:33.982226Z","shell.execute_reply.started":"2022-05-14T15:48:33.961246Z","shell.execute_reply":"2022-05-14T15:48:33.980979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read data 讀取資料\nfeld_needed = ['row_id', 'user_id', 'content_id', 'content_type_id', 'answered_correctly', 'prior_question_elapsed_time', 'prior_question_had_explanation']\ntrain = pd.read_pickle(train_pickle)[feld_needed]\nvalid = pd.read_pickle(valid_pickle)[feld_needed]\n#若上方debug設定為false則不執行(需除錯再打開即可)\nif debug:\n    train = train[:100]\n    valid = valid[:10]\n    \n#content_type_id (用戶作的種類 0考卷 1講座)，只留0不留1\ntrain = train.loc[train.content_type_id == False].reset_index(drop=True)\nvalid = valid.loc[valid.content_type_id == False].reset_index(drop=True)\n\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-05-14T15:48:33.983810Z","iopub.execute_input":"2022-05-14T15:48:33.984146Z","iopub.status.idle":"2022-05-14T15:49:31.931345Z","shell.execute_reply.started":"2022-05-14T15:48:33.984112Z","shell.execute_reply":"2022-05-14T15:49:31.930030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# answered correctly average for each content 每個用戶的正確率做平均\ncontent_df = train[['content_id','answered_correctly']].groupby(['content_id']).agg(['mean']).reset_index()\ncontent_df.columns = ['content_id', 'answered_correctly_avg_c']\ntrain = pd.merge(train, content_df, on=['content_id'], how=\"left\")\nvalid = pd.merge(valid, content_df, on=['content_id'], how=\"left\")\n\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-05-14T15:49:31.935669Z","iopub.execute_input":"2022-05-14T15:49:31.936079Z","iopub.status.idle":"2022-05-14T15:49:32.134983Z","shell.execute_reply.started":"2022-05-14T15:49:31.936039Z","shell.execute_reply":"2022-05-14T15:49:32.133661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# user stats features with loops 做迴圈 統計用戶數據\nanswered_correctly_sum_u_dict = defaultdict(int)\ncount_u_dict = defaultdict(int)\ntrain = add_user_feats(train, answered_correctly_sum_u_dict, count_u_dict)\nvalid = add_user_feats(valid, answered_correctly_sum_u_dict, count_u_dict)\n\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-05-14T15:49:32.137495Z","iopub.execute_input":"2022-05-14T15:49:32.138005Z","iopub.status.idle":"2022-05-14T15:49:32.253162Z","shell.execute_reply.started":"2022-05-14T15:49:32.137952Z","shell.execute_reply":"2022-05-14T15:49:32.252250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fill with mean value for prior_question_elapsed_time 填充prior_question_elapsed_time的平均值\n# note that `train.prior_question_elapsed_time.mean()` dose not work! 请注意，`train.prior_question_elapsed_time.mean()` 不起作用！\n# please refer https://www.kaggle.com/its7171/can-we-trust-pandas-mean for detail.\nprior_question_elapsed_time_mean = train.prior_question_elapsed_time.dropna().values.mean()\ntrain['prior_question_elapsed_time_mean'] = train.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)\nvalid['prior_question_elapsed_time_mean'] = valid.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)\n\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-05-14T15:49:32.254591Z","iopub.execute_input":"2022-05-14T15:49:32.254928Z","iopub.status.idle":"2022-05-14T15:49:32.295943Z","shell.execute_reply.started":"2022-05-14T15:49:32.254896Z","shell.execute_reply":"2022-05-14T15:49:32.294772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# use only last 30M training data for limited memory on kaggle env. \n#train = train[-30000000:] 減少ram的使用量\n\n# part\nquestions_df = pd.read_csv(question_file)\ntrain = pd.merge(train, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')\nvalid = pd.merge(valid, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-05-14T15:49:32.297760Z","iopub.execute_input":"2022-05-14T15:49:32.298094Z","iopub.status.idle":"2022-05-14T15:49:32.364931Z","shell.execute_reply.started":"2022-05-14T15:49:32.298063Z","shell.execute_reply":"2022-05-14T15:49:32.363489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# changing dtype to avoid lightgbm error 更改 資料型態dtype 以避免 lightgbm 錯誤\ntrain['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False).astype('int8')\nvalid['prior_question_had_explanation'] = valid.prior_question_had_explanation.fillna(False).astype('int8')\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-05-14T15:49:32.366782Z","iopub.execute_input":"2022-05-14T15:49:32.367166Z","iopub.status.idle":"2022-05-14T15:49:32.406842Z","shell.execute_reply.started":"2022-05-14T15:49:32.367090Z","shell.execute_reply":"2022-05-14T15:49:32.405412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## modeling","metadata":{}},{"cell_type":"code","source":"#目標-正確率\nTARGET = 'answered_correctly'\n#需用到的特徵\nFEATS = ['answered_correctly_avg_u', 'answered_correctly_sum_u', 'count_u', 'answered_correctly_avg_c', 'part', 'prior_question_had_explanation', 'prior_question_elapsed_time']\n#訓練集裡面 這些特徵移除 \ndro_cols = list(set(train.columns) - set(FEATS))\ny_tr = train[TARGET]\ny_va = valid[TARGET]\n# 把訓練即除了dro_cols的特徵都移除。inplace=true 不創建新的对象，直接對原始對象進行修改\ntrain.drop(dro_cols, axis=1, inplace=True)\nvalid.drop(dro_cols, axis=1, inplace=True)\n_=gc.collect()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-14T15:49:32.408656Z","iopub.execute_input":"2022-05-14T15:49:32.408993Z","iopub.status.idle":"2022-05-14T15:49:32.523317Z","shell.execute_reply.started":"2022-05-14T15:49:32.408960Z","shell.execute_reply":"2022-05-14T15:49:32.522464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_train = lgb.Dataset(train[FEATS], y_tr)\nlgb_valid = lgb.Dataset(valid[FEATS], y_va)\ndel train, y_tr\n_=gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-05-14T15:49:32.524872Z","iopub.execute_input":"2022-05-14T15:49:32.525232Z","iopub.status.idle":"2022-05-14T15:49:32.615477Z","shell.execute_reply.started":"2022-05-14T15:49:32.525198Z","shell.execute_reply":"2022-05-14T15:49:32.614050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = lgb.train(\n                    {'objective': 'binary'}, \n                    lgb_train,\n                    valid_sets=[lgb_train, lgb_valid],\n                    verbose_eval=100,#每迭代100次印出詳細資料\n                    num_boost_round=10000,#迭代次數\n                    early_stopping_rounds=10#十輪內沒最好結果即停止\n                )\nprint('auc:', roc_auc_score(y_va, model.predict(valid[FEATS])))\n_ = lgb.plot_importance(model)","metadata":{"execution":{"iopub.status.busy":"2022-05-14T15:49:32.617901Z","iopub.execute_input":"2022-05-14T15:49:32.618421Z","iopub.status.idle":"2022-05-14T15:49:32.920606Z","shell.execute_reply.started":"2022-05-14T15:49:32.618369Z","shell.execute_reply":"2022-05-14T15:49:32.919462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## inference","metadata":{}},{"cell_type":"code","source":"#時間序列API(iter_test)模擬器\nclass Iter_Valid(object):\n    def __init__(self, df, max_user=1000):\n        df = df.reset_index(drop=True)\n        self.df = df\n        self.user_answer = df['user_answer'].astype(str).values\n        self.answered_correctly = df['answered_correctly'].astype(str).values\n        df['prior_group_responses'] = \"[]\"\n        df['prior_group_answers_correct'] = \"[]\"\n        self.sample_df = df[df['content_type_id'] == 0][['row_id']]\n        self.sample_df['answered_correctly'] = 0\n        self.len = len(df)\n        self.user_id = df.user_id.values\n        self.task_container_id = df.task_container_id.values\n        self.content_type_id = df.content_type_id.values\n        self.max_user = max_user\n        self.current = 0\n        self.pre_user_answer_list = []\n        self.pre_answered_correctly_list = []\n\n    def __iter__(self):\n        return self\n    \n    def fix_df(self, user_answer_list, answered_correctly_list, pre_start):\n        df= self.df[pre_start:self.current].copy()\n        sample_df = self.sample_df[pre_start:self.current].copy()\n        df.loc[pre_start,'prior_group_responses'] = '[' + \",\".join(self.pre_user_answer_list) + ']'\n        df.loc[pre_start,'prior_group_answers_correct'] = '[' + \",\".join(self.pre_answered_correctly_list) + ']'\n        self.pre_user_answer_list = user_answer_list\n        self.pre_answered_correctly_list = answered_correctly_list\n        return df, sample_df\n\n    def __next__(self):\n        added_user = set()\n        pre_start = self.current\n        pre_added_user = -1\n        pre_task_container_id = -1\n        pre_content_type_id = -1\n        user_answer_list = []\n        answered_correctly_list = []\n        while self.current < self.len:\n            crr_user_id = self.user_id[self.current]\n            crr_task_container_id = self.task_container_id[self.current]\n            crr_content_type_id = self.content_type_id[self.current]\n            if crr_user_id in added_user and (crr_user_id != pre_added_user or (crr_task_container_id != pre_task_container_id and crr_content_type_id == 0 and pre_content_type_id == 0)):\n                # known user(not prev user or (differnt task container and both question))\n                return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n            if len(added_user) == self.max_user:\n                if  crr_user_id == pre_added_user and (crr_task_container_id == pre_task_container_id or crr_content_type_id == 1):\n                    user_answer_list.append(self.user_answer[self.current])\n                    answered_correctly_list.append(self.answered_correctly[self.current])\n                    self.current += 1\n                    continue\n                else:\n                    return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n            added_user.add(crr_user_id)\n            pre_added_user = crr_user_id\n            pre_task_container_id = crr_task_container_id\n            pre_content_type_id = crr_content_type_id\n            user_answer_list.append(self.user_answer[self.current])\n            answered_correctly_list.append(self.answered_correctly[self.current])\n            self.current += 1\n        if pre_start < self.current:\n            return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n        else:\n            raise StopIteration()","metadata":{"papermill":{"duration":0.07062,"end_time":"2020-10-12T18:42:48.032349","exception":false,"start_time":"2020-10-12T18:42:47.961729","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-14T15:49:32.922315Z","iopub.execute_input":"2022-05-14T15:49:32.922987Z","iopub.status.idle":"2022-05-14T15:49:32.951233Z","shell.execute_reply.started":"2022-05-14T15:49:32.922935Z","shell.execute_reply":"2022-05-14T15:49:32.950118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# You can debug your inference code to reduce \"Submission Scoring Error\" with `validaten_flg = True`.//需要驗證可改true\n# Please refer https://www.kaggle.com/its7171/time-series-api-iter-test-emulator about Time-series API (iter_test) Emulator.\n\nif validaten_flg:\n    target_df = pd.read_pickle(valid_pickle)\n    if debug:\n        target_df = target_df[:10000]\n    iter_test = Iter_Valid(target_df,max_user=1000)\n    predicted = []\n    def set_predict(df):\n        predicted.append(df)\n    # reset answered_correctly_sum_u_dict and count_u_dict\n    answered_correctly_sum_u_dict = defaultdict(int)\n    count_u_dict = defaultdict(int)\n    train = pd.read_pickle(train_pickle)[['user_id','answered_correctly','content_type_id']]\n    if debug:\n        train = train[:1000000]\n    train = train[train.content_type_id == False].reset_index(drop=True)\n    update_user_feats(train, answered_correctly_sum_u_dict, count_u_dict)\n    del train\nelse:\n    #設定API的配置\n    import riiideducation\n    env = riiideducation.make_env()\n    iter_test = env.iter_test() #是一個迴圈，會紀錄上一次學生答題狀況。\n    set_predict = env.predict","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-14T15:49:32.953142Z","iopub.execute_input":"2022-05-14T15:49:32.953552Z","iopub.status.idle":"2022-05-14T15:49:32.991860Z","shell.execute_reply.started":"2022-05-14T15:49:32.953500Z","shell.execute_reply":"2022-05-14T15:49:32.990677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#進行預測\nprevious_test_df = None\n#把iter_test的資料導進來，test_df紀錄上次學生答題狀況。\nfor (test_df, sample_prediction_df) in iter_test:\n    if previous_test_df is not None:\n        previous_test_df[TARGET] = eval(test_df[\"prior_group_answers_correct\"].iloc[0])\n        update_user_feats(previous_test_df, answered_correctly_sum_u_dict, count_u_dict)\n    previous_test_df = test_df.copy()\n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n    test_df = add_user_feats_without_update(test_df, answered_correctly_sum_u_dict, count_u_dict)\n    test_df = pd.merge(test_df, content_df, on='content_id',  how=\"left\")\n    test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')\n    test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False).astype('int8')\n    test_df['prior_question_elapsed_time_mean'] = test_df.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)\n    test_df[TARGET] =  model.predict(test_df[FEATS])\n    set_predict(test_df[['row_id', TARGET]])\n","metadata":{"papermill":{"duration":1.016814,"end_time":"2020-10-12T18:42:49.112404","exception":false,"start_time":"2020-10-12T18:42:48.09559","status":"completed"},"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2022-05-14T15:49:32.993810Z","iopub.execute_input":"2022-05-14T15:49:32.994225Z","iopub.status.idle":"2022-05-14T15:49:33.145811Z","shell.execute_reply.started":"2022-05-14T15:49:32.994186Z","shell.execute_reply":"2022-05-14T15:49:33.144756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if validaten_flg:\n    y_true = target_df[target_df.content_type_id == 0].answered_correctly\n    y_pred = pd.concat(predicted).answered_correctly\n    print(roc_auc_score(y_true, y_pred))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-14T15:49:33.147311Z","iopub.execute_input":"2022-05-14T15:49:33.147960Z","iopub.status.idle":"2022-05-14T15:49:33.154650Z","shell.execute_reply.started":"2022-05-14T15:49:33.147917Z","shell.execute_reply":"2022-05-14T15:49:33.153273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Have a fun with loops! :)","metadata":{}}]}