{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Riiid! LGBM Single Model Ensembling - Scoring"},{"metadata":{},"cell_type":"markdown","source":"This notebook is used as a demonstration for my thread on [Single Model Ensembling Guide | LightGBM Example](https://www.kaggle.com/c/riiid-test-answer-prediction/discussion/202344)\n\n**Main Idea**: Use different number of trees to score on test data and take the weighted average of the outputs. \n\nThis is a scoring only notebook. The Training Notebook is [available here](https://www.kaggle.com/manikanthr5/riiid-lgbm-single-model-ensembling-training/).\n\n![](https://i.imgur.com/qlQTh0b.png)\n\n**Acknowledgement:** I am using [this notebook](https://www.kaggle.com/its7171/lgbm-with-loop-feature-engineering/) as the starter to show my idea. If you like this kernel, please upvote [the actual kernel](https://www.kaggle.com/its7171/lgbm-with-loop-feature-engineering/execution/). I have removed some code which is not required for scoring purpose."},{"metadata":{"execution":{"iopub.execute_input":"2020-10-12T18:21:01.107461Z","iopub.status.busy":"2020-10-12T18:21:01.1067Z","iopub.status.idle":"2020-10-12T18:21:02.19541Z","shell.execute_reply":"2020-10-12T18:21:02.194464Z"},"lines_to_next_cell":2,"papermill":{"duration":1.131405,"end_time":"2020-10-12T18:21:02.195556","exception":false,"start_time":"2020-10-12T18:21:01.064151","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import gc\nimport joblib\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## feature engineering"},{"metadata":{"trusted":true},"cell_type":"code","source":"# funcs for user stats with loop\ndef add_user_feats_without_update(df, answered_correctly_sum_u_dict, count_u_dict):\n    acsu = np.zeros(len(df), dtype=np.int32)\n    cu = np.zeros(len(df), dtype=np.int32)\n    for cnt,row in enumerate(df[['user_id']].values):\n        acsu[cnt] = answered_correctly_sum_u_dict[row[0]]\n        cu[cnt] = count_u_dict[row[0]]\n    user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})\n    user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']\n    df = pd.concat([df, user_feats_df], axis=1)\n    return df\n\ndef update_user_feats(df, answered_correctly_sum_u_dict, count_u_dict):\n    for row in df[['user_id','answered_correctly','content_type_id']].values:\n        if row[2] == 0:\n            answered_correctly_sum_u_dict[row[0]] += row[1]\n            count_u_dict[row[0]] += 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Load Variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"answered_correctly_sum_u_dict = joblib.load(\"../input/lgbm-with-loop-feature-engineering-dataset/answered_correctly_sum_u_dict.pkl.zip\")\ncount_u_dict = joblib.load(\"../input/lgbm-with-loop-feature-engineering-dataset/count_u_dict.pkl.zip\")\n\nquestions_df = pd.read_feather('../input/lgbm-with-loop-feature-engineering-dataset/questions_df.feather')\ncontent_df = pd.read_feather('../input/lgbm-with-loop-feature-engineering-dataset/content_df.feather')\n\nprior_question_elapsed_time_mean = joblib.load(\"../input/lgbm-with-loop-feature-engineering-dataset/prior_question_elapsed_time_mean.pkl.zip\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## modeling"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"TARGET = 'answered_correctly'\nFEATS = ['answered_correctly_avg_u', 'answered_correctly_sum_u', 'count_u', \n         'answered_correctly_avg_c', 'part', 'prior_question_had_explanation', \n         'prior_question_elapsed_time'\n        ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = lgb.Booster(model_file=\"../input/lgbm-with-loop-feature-engineering-dataset/fold0_lgb_model.txt\")\nmodel.best_iteration = joblib.load(\"../input/lgbm-with-loop-feature-engineering-dataset/fold0_lgb_model_best_iteration.pkl.zip\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimized_weights = joblib.load(\"../input/lgbm-with-loop-feature-engineering-dataset/optimized_weights.pkl.zip\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## inference"},{"metadata":{"execution":{"iopub.execute_input":"2020-10-12T18:42:48.029704Z","iopub.status.busy":"2020-10-12T18:42:48.028875Z","iopub.status.idle":"2020-10-12T18:42:48.032215Z","shell.execute_reply":"2020-10-12T18:42:48.031426Z"},"papermill":{"duration":0.07062,"end_time":"2020-10-12T18:42:48.032349","exception":false,"start_time":"2020-10-12T18:42:47.961729","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class Iter_Valid(object):\n    def __init__(self, df, max_user=1000):\n        df = df.reset_index(drop=True)\n        self.df = df\n        self.user_answer = df['user_answer'].astype(str).values\n        self.answered_correctly = df['answered_correctly'].astype(str).values\n        df['prior_group_responses'] = \"[]\"\n        df['prior_group_answers_correct'] = \"[]\"\n        self.sample_df = df[df['content_type_id'] == 0][['row_id']]\n        self.sample_df['answered_correctly'] = 0\n        self.len = len(df)\n        self.user_id = df.user_id.values\n        self.task_container_id = df.task_container_id.values\n        self.content_type_id = df.content_type_id.values\n        self.max_user = max_user\n        self.current = 0\n        self.pre_user_answer_list = []\n        self.pre_answered_correctly_list = []\n\n    def __iter__(self):\n        return self\n    \n    def fix_df(self, user_answer_list, answered_correctly_list, pre_start):\n        df= self.df[pre_start:self.current].copy()\n        sample_df = self.sample_df[pre_start:self.current].copy()\n        df.loc[pre_start,'prior_group_responses'] = '[' + \",\".join(self.pre_user_answer_list) + ']'\n        df.loc[pre_start,'prior_group_answers_correct'] = '[' + \",\".join(self.pre_answered_correctly_list) + ']'\n        self.pre_user_answer_list = user_answer_list\n        self.pre_answered_correctly_list = answered_correctly_list\n        return df, sample_df\n\n    def __next__(self):\n        added_user = set()\n        pre_start = self.current\n        pre_added_user = -1\n        pre_task_container_id = -1\n        pre_content_type_id = -1\n        user_answer_list = []\n        answered_correctly_list = []\n        while self.current < self.len:\n            crr_user_id = self.user_id[self.current]\n            crr_task_container_id = self.task_container_id[self.current]\n            crr_content_type_id = self.content_type_id[self.current]\n            if crr_user_id in added_user and (crr_user_id != pre_added_user or (crr_task_container_id != pre_task_container_id and crr_content_type_id == 0 and pre_content_type_id == 0)):\n                # known user(not prev user or (differnt task container and both question))\n                return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n            if len(added_user) == self.max_user:\n                if  crr_user_id == pre_added_user and (crr_task_container_id == pre_task_container_id or crr_content_type_id == 1):\n                    user_answer_list.append(self.user_answer[self.current])\n                    answered_correctly_list.append(self.answered_correctly[self.current])\n                    self.current += 1\n                    continue\n                else:\n                    return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n            added_user.add(crr_user_id)\n            pre_added_user = crr_user_id\n            pre_task_container_id = crr_task_container_id\n            pre_content_type_id = crr_content_type_id\n            user_answer_list.append(self.user_answer[self.current])\n            answered_correctly_list.append(self.answered_correctly[self.current])\n            self.current += 1\n        if pre_start < self.current:\n            return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n        else:\n            raise StopIteration()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"import riiideducation\nenv = riiideducation.make_env()\niter_test = env.iter_test()\nset_predict = env.predict","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-12T18:42:48.174757Z","iopub.status.busy":"2020-10-12T18:42:48.173901Z","iopub.status.idle":"2020-10-12T18:42:49.112276Z","shell.execute_reply":"2020-10-12T18:42:49.111472Z"},"papermill":{"duration":1.016814,"end_time":"2020-10-12T18:42:49.112404","exception":false,"start_time":"2020-10-12T18:42:48.09559","status":"completed"},"scrolled":true,"tags":[],"trusted":true},"cell_type":"code","source":"previous_test_df = None\nfor (test_df, sample_prediction_df) in iter_test:\n    if previous_test_df is not None:\n        previous_test_df[TARGET] = eval(test_df[\"prior_group_answers_correct\"].iloc[0])\n        update_user_feats(previous_test_df, answered_correctly_sum_u_dict, count_u_dict)\n    previous_test_df = test_df.copy()\n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n    test_df = add_user_feats_without_update(test_df, answered_correctly_sum_u_dict, count_u_dict)\n    test_df = pd.merge(test_df, content_df, on='content_id',  how=\"left\")\n    test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')\n    test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False).astype('int8')\n    test_df['prior_question_elapsed_time_mean'] = test_df.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)\n    preds = optimized_weights[0] * model.predict(test_df[FEATS], num_iteration=400)\n    preds += optimized_weights[1] * model.predict(test_df[FEATS], num_iteration=700)\n    preds += optimized_weights[2] * model.predict(test_df[FEATS], num_iteration=model.best_iteration)\n    test_df[TARGET] = preds\n    set_predict(test_df[['row_id', TARGET]])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" "},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}