{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.metrics import roc_auc_score\nimport lightgbm as lgb\nimport pickle\n\nimport riiideducation\n\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class user:\n    def __init__(self):\n        self.questions_answered = 0\n        self.answered_correctly = 0\n        self.user_prof = 0.\n        self.user_prof_inv = 1.\n        \n        self.questions_per_part = {1:0,2:0,3:0,4:0,5:0,6:0,7:0}\n        self.correct_per_part = {1:0,2:0,3:0,4:0,5:0,6:0,7:0}\n        self.user_part_prof = {1:0,2:0,3:0,4:0,5:0,6:0,7:0}\n        self.user_part_prof_inv = {1:0,2:0,3:0,4:0,5:0,6:0,7:0}\n\n        self.had_explanation = 0\n        self.no_explanation = 0\n        self.mean_user_p_e = 0.\n\n        \n    def train_update(self, data):\n        self.questions_answered = self.questions_answered + 1\n        self.questions_per_part[data['part']] = self.questions_per_part[data['part']] + 1\n\n        if data['answered_correctly']==1:\n            self.answered_correctly = self.answered_correctly + 1\n            self.correct_per_part[data['part']] = self.correct_per_part[data['part']] + 1\n\n        if data['prior_question_had_explanation']==1:\n            self.had_explanation = self.had_explanation + 1\n        else:\n            self.no_explanation = self.no_explanation + 1\n\n    def test_update(self, data):\n        self.questions_answered = self.questions_answered + 1\n        self.questions_per_part[data['part']] = self.questions_per_part[data['part']] + 1\n\n        if data['prior_question_had_explanation']==1:\n            self.had_explanation = self.had_explanation + 1\n        else:\n            self.no_explanation = self.no_explanation + 1        \n    \n    def update_answers(self,data):\n        self.answered_correctly = self.answered_correctly + 1\n        self.correct_per_part[data['part']] = self.correct_per_part[data['part']] + 1\n        self.do_the_monster_math()\n        \n    def do_the_monster_math(self):\n        self.user_prof = self.answered_correctly / self.questions_answered\n        self.user_prof_inv = 1 - self.user_prof\n        \n        for part in self.questions_per_part:\n            if self.questions_per_part[part]==0:\n                continue\n            self.user_part_prof[part] = self.correct_per_part[part] / self.questions_per_part[part]\n            self.user_part_prof_inv[part] = 1 - self.user_part_prof[part]\n        \n        self.mean_user_p_e = self.had_explanation / (self.had_explanation + self.no_explanation)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\nquestions_df = pd.read_csv('../input/riiid-test-answer-prediction/questions.csv',\n                          usecols=['question_id','part'],\n                          dtype={'question_id':'int32','part':'int8'})\n\nwith open('../input/riiid-model/final.pickle', 'rb') as handle:\n    train_dict = pickle.load(handle)\n    \nmodel = pd.read_pickle('../input/riiid-model/lgbm_no_comm.pkl')\n# print('Load model:', model)\n\n\nenv = riiideducation.make_env()\niter_test = env.iter_test()\nfirst_pass = True\n\nfor (test_df, sample_prediction_df) in iter_test:\n    test_df = test_df.merge(questions_df,left_on='content_id',right_on='question_id',how='left').drop('question_id',axis=1)\n    test_df.loc[:,'prior_question_had_explanation'].fillna(False,inplace=True)\n    \n    # Add Features\n    if first_pass == False:\n        try:\n            old_df['answered_correctly'] = eval(test_df.iat[0,-3])\n            for _,row in old_df.iterrows():\n                if row['content_type_id']==0:\n                    if row['answered_correctly']==1:\n                        train_dict[row['user_id']].update_answers(row)\n                    else:\n                        train_dict[row['user_id']].do_the_monster_math()\n        except Exception as e:\n            print('first: ',e)\n\n    old_df = test_df[['row_id','user_id','part','content_type_id']]\n    \n    cols = ['user_part_prof','user_part_prof_inv','user_prof','user_prof_inv','mean_user_p_e','num_qs_ans']\n    test_df.loc[:,cols] = np.nan\n    test_df[cols] = test_df[cols].astype('float16')\n    \n    for index,row in test_df.iterrows():\n        try:\n            if row['user_id'] not in train_dict:\n                train_dict[row['user_id']] = user()\n            if row['content_type_id']==0:\n                train_dict[row['user_id']].test_update(row)\n            test_df.iloc[index,-6:] = [train_dict[row['user_id']].user_part_prof[row['part']],\n                                       train_dict[row['user_id']].user_part_prof_inv[row['part']],\n                                       train_dict[row['user_id']].user_prof,\n                                       train_dict[row['user_id']].user_prof_inv,\n                                       train_dict[row['user_id']].mean_user_p_e,\n                                       train_dict[row['user_id']].questions_answered]\n\n        except Exception as e:\n            print('last: ',e)\n            pass\n    \n    # Correct Data\n    test_df[['user_id','content_id']] = test_df[['user_id','content_id']].astype('category')\n    first_pass = False\n    \n    # Predictions\n    test_df['answered_correctly'] = model.predict(test_df[['user_id','content_id','user_part_prof','user_part_prof_inv','user_prof','user_prof_inv','mean_user_p_e','num_qs_ans']], num_iteration=model.best_iteration)\n    env.predict(test_df.loc[test_df['content_type_id']==0,['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}