{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport pickle\nimport lightgbm as lgb\nfrom collections import OrderedDict, defaultdict\nfrom tqdm import tqdm\nimport gc\nimport os\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append('../input/riiid-ym-train/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import psutil\nimport os\nimport time\nimport sys\nimport math\nfrom contextlib import contextmanager\n\n@contextmanager\ndef trace(title):\n    t0 = time.time()\n    p = psutil.Process(os.getpid())\n    m0 = p.memory_info()[0] / 2. ** 30\n    yield\n    m1 = p.memory_info()[0] / 2. ** 30\n    delta = m1 - m0\n    sign = '+' if delta >= 0 else '-'\n    delta = math.fabs(delta)\n    print(f\"[{m1:.1f}GB({sign}{delta:.1f}GB):{time.time() - t0:.1f}sec] {title} \", file=sys.stderr)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"EX = 61\n#EX_CAT = 54","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with trace(\"riiid_tag_feature\"):\n    with open(f'../input/riiid-ym/EX{EX}_baseline/model.pkl','rb') as file:\n        clf = pickle.load(file)\n    \n#with open(f'../input/riiid-ym/EX{EX_CAT}_baseline/model.pkl','rb') as file:\n#    clf_cat = pickle.load(file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"content_df_new = pd.read_csv('../input/riiid-ym-train/content_df_new.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ealsed_fill = 25440.594220232804","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from RiiidUtil.RiiidStateFeature import RiiidStateFeature\nfrom RiiidUtil.RiiidStateSharedFeature import RiiidStateSharedFeature\nfrom RiiidUtil.RiiidStateTagFeature import RiiidStateTagFeature\nfrom RiiidUtil.RiiidStateSub import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loop_type = True\n\nuser_target_count = RiiidStateCount(['user_id'], 'answered_correctly', loop_type)\nuser_target_sum = RiiidStateSum(['user_id'], 'answered_correctly', loop_type)\n\nuser_part_target_count = RiiidStateCount2SourceSeries(['user_id','part'], 'answered_correctly', loop_type)\nuser_part_target_sum = RiiidStateSum2SourceSeries(['user_id','part'], 'answered_correctly', loop_type)\n\nuser_prior_time_currentsum = RiiidStateCurrentSum(['user_id'], 'prior_question_elapsed_time', loop_type)\nuser_prior_hadexp_currentsum = RiiidStateCurrentSum(['user_id'], 'prior_question_had_explanation', loop_type)\n\nuser_content_id_first = RiiidStateFirst(['user_id'], 'content_id')\nuser_content_id_last = RiiidStateLast(['user_id'], 'content_id')\n\nuser_timestamp_last = RiiidStateLast(['user_id'], 'timestamp')\nuser_timestamp_last2 = RiiidStateLag(['user_id'], 'timestamp', 2)\nuser_timestamp_last3 = RiiidStateLag(['user_id'], 'timestamp', 3)\nuser_timestamp_last10 = RiiidStateLag(['user_id'], 'timestamp', 10)\n\nuser_answered_incorrectly_sum = RiiidStateLagSum(['user_id'], 'answered_correctly', 10)\nuser_answered_correctly_avg_c_sum = RiiidStateSum(['user_id'], 'answered_correctly_avg_c', loop_type)\nuser_answered_correctly_last = RiiidStateLast(['user_id'], 'answered_correctly')\nuser_answered_correctly_last2 = RiiidStateLag(['user_id'], 'answered_correctly', 2)\nuser_answered_correctly_last3 = RiiidStateLag(['user_id'], 'answered_correctly', 3)\n\n\nuser_content_id_count_currentsum = RiiidStateCurrentSum(['user_id'], 'content_id_count', loop_type)\nuser_content_id_ptp_sum = RiiidStateSum(['user_id'], 'content_id_ptp', loop_type)\n\nuser_part_answered_correctly_timestamp = RiiidStateLast2SourceSeries(['user_id','part'], 'timestamp', loop_type)\n\nuser_cluster_answered_correctly_sum = RiiidStateSum2SourceSeries(['user_id','cluster'],'answered_correctly', loop_type)\nuser_cluster_answered_correctly_count = RiiidStateCount2SourceSeries(['user_id','cluster'],'answered_correctly', loop_type)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"riiid_state_feature = RiiidStateFeature('../input/riiid-ym-train')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"riiid_state_feature.append(user_target_count)\nriiid_state_feature.append(user_target_sum)\n\nriiid_state_feature.append(user_part_target_count)\nriiid_state_feature.append(user_part_target_sum)\n\n#riiid_state_feature.append(user_content_target_count)\n#riiid_state_feature.append(user_content_target_sum)\n\nriiid_state_feature.append(user_prior_time_currentsum)\nriiid_state_feature.append(user_prior_hadexp_currentsum)\n\nriiid_state_feature.append(user_content_id_first)\nriiid_state_feature.append(user_content_id_last)\n#riiid_state_feature.append(user_content_id_last2)\n\nriiid_state_feature.append(user_timestamp_last)\nriiid_state_feature.append(user_timestamp_last2)\nriiid_state_feature.append(user_timestamp_last3)\nriiid_state_feature.append(user_timestamp_last10)\n\nriiid_state_feature.append(user_answered_incorrectly_sum)\nriiid_state_feature.append(user_answered_correctly_avg_c_sum)\nriiid_state_feature.append(user_answered_correctly_last)\nriiid_state_feature.append(user_answered_correctly_last2)\nriiid_state_feature.append(user_answered_correctly_last3)\n\n#riiid_state_feature.append(user_tags1_answered_correctly_sum)\n#riiid_state_feature.append(user_tags1_answered_correctly_count)\nriiid_state_feature.append(user_content_id_count_currentsum)\nriiid_state_feature.append(user_content_id_ptp_sum)\n#riiid_state_feature.append(user_content_id_answered_correctly_timestamp)\n#riiid_state_feature.append(user_content_id_answered_correctly_last)\nriiid_state_feature.append(user_part_answered_correctly_timestamp)\n#riiid_state_feature.append(user_part_answered_correctly_last)\n#riiid_state_feature.append(user_tags1_timestamp_last)\n\nriiid_state_feature.append(user_cluster_answered_correctly_sum)\nriiid_state_feature.append(user_cluster_answered_correctly_count)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#user_target_count.load('../input/riiid-ym-train')\nwith trace(\"riiid_state_feature.load()\"):\n    riiid_state_feature.load()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with trace(\"riiid_state_feature.load()\"):\n    user_content_state_feature = RiiidStateSharedFeature(['user_id','content_id'], \n                                                         ['answered_correctly','answered_correctly','timestamp'], \n                                                         ['count','sum','last'], \n                                                         [loop_type,loop_type,loop_type], '../input/riiid-ym-train')\n\n    user_content_state_feature.load()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with trace(\"riiid_tag_feature\"):\n    riiid_tag_feature = RiiidStateTagFeature('../input/riiid-ym-train')\n    riiid_tag_feature.load()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_columns = clf.feature_name().copy()\ntarget = 'answered_correctly'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\nenv = riiideducation.make_env()\niter_test = env.iter_test()\nset_predict = env.predict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"previous_test_df = None\nfor (test_df, sample_prediction_df) in iter_test:\n    try:\n        if previous_test_df is not None:\n            previous_test_df[target] = eval(test_df[\"prior_group_answers_correct\"].iloc[0])\n            previous_test_df = previous_test_df[previous_test_df['content_type_id']==0].reset_index(drop=True)\n            #previous_test_df = pd.merge(previous_test_df, content_df, on='content_id',  how=\"left\")\n            #previous_test_df = pd.merge(previous_test_df, questions , left_on='content_id', right_on='question_id', how='left')\n            previous_test_df = pd.concat([previous_test_df, content_df_new.reindex(previous_test_df['content_id'].values).reset_index(drop=True)], axis=1)\n            previous_test_df['prior_question_had_explanation'] = previous_test_df['prior_question_had_explanation'].fillna(False).astype('int8')\n            previous_test_df['prior_question_elapsed_time'] = previous_test_df['prior_question_elapsed_time'].fillna(ealsed_fill)\n            \n            #user_target_count.make_run(previous_test_df, True)\n            #with trace(\"riiid_state_feature.only_update(previous_test_df)\"):\n            riiid_state_feature.only_update(previous_test_df)\n            #previous_test_df = user_content_state_feature.run(previous_test_df, True)\n            #with trace(\"user_content_state_feature.total_loop_only_update(previous_test_df)\"):\n            user_content_state_feature.total_loop_only_update(previous_test_df)\n                \n            riiid_tag_feature.total_loop_only_update(previous_test_df)\n    \n        previous_test_df = test_df.copy()\n        test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n        #test_df = pd.merge(test_df, content_df, on='content_id',  how=\"left\")\n        #test_df = pd.merge(test_df,questions , left_on='content_id', right_on='question_id', how='left')\n        test_df = pd.concat([test_df, content_df_new.reindex(test_df['content_id'].values).reset_index(drop=True)], axis=1)\n        test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(False).astype('int8')\n        test_df['prior_question_elapsed_time'] = test_df['prior_question_elapsed_time'].fillna(ealsed_fill)\n        #test_df['content_time_mean'] = test_df['content_id'].map(content_time_mean_dict)\n        \n        test_df[target] = np.nan\n        #user_target_count.make_run(test_df, False)\n        #with trace(\"riiid_state_feature.run(test_df, False)\"):\n        test_df = riiid_state_feature.run(test_df, False)\n        #test_df = user_content_state_feature.run(test_df, False)\n        #with trace(\"user_content_state_feature.total_loop_inference(test_df)\"):\n        test_df = user_content_state_feature.total_loop_inference(test_df)\n        test_df = riiid_tag_feature.total_loop_inference(test_df)\n        \n        test_df['user_id_answered_correctly_mean'] = test_df['user_id_answered_correctly_sum']/test_df['user_id_answered_correctly_count']\n        test_df['user_id_content_id_answered_correctly_mean'] = test_df['user_id_content_id_answered_correctly_sum']/test_df['user_id_content_id_answered_correctly_count']\n        test_df['user_id_part_answered_correctly_mean'] = test_df['user_id_part_answered_correctly_sum']/test_df['user_id_part_answered_correctly_count']\n        test_df['upm_div_cm'] = test_df['user_id_part_answered_correctly_mean']/test_df['answered_correctly_avg_c']\n        #test_df.loc[np.isinf(test_df['upm_div_cm']),'upm_div_cm'] = 1\n        test_df['elapsed_time_mean'] = test_df['user_id_prior_question_elapsed_time_currentsum']/(test_df['user_id_answered_correctly_count']+1)\n        test_df['ptime_div_em'] = test_df['prior_question_elapsed_time']/test_df['elapsed_time_mean']\n        test_df['same_content'] = test_df['content_id'] == test_df['user_id_content_id_last']\n        test_df['same_content'] = test_df['same_content'].astype('int8')\n        test_df['timediff'] = test_df['timestamp'] - test_df['user_id_timestamp_last']\n        test_df['timediff2'] = test_df['timestamp'] - test_df['user_id_timestamp_last2']\n        test_df['timediff3'] = test_df['timestamp'] - test_df['user_id_timestamp_last3']\n        test_df['timediff10'] = test_df['timestamp'] - test_df['user_id_timestamp_last10']\n        test_df.loc[test_df['timediff']==0,'timediff'] = np.nan\n        test_df['timediff'] = test_df['timediff'].fillna(method='ffill')\n        test_df.loc[test_df['user_id_answered_correctly_count']==0,'timediff'] = np.nan\n        test_df.loc[test_df['timediff2']==0,'timediff2'] = np.nan\n        test_df['lag_diff_elapsed'] = test_df['prior_question_elapsed_time'] - test_df['timediff']\n        test_df['lag_diff_elapsed2'] = test_df['prior_question_elapsed_time'] - test_df['timediff2']\n        \n        \n        test_df['user_id_answered_correctly_avg_c_sum_mean'] = test_df['user_id_answered_correctly_avg_c_sum']/test_df['user_id_answered_correctly_count']\n        test_df['user_id_prior_question_had_explanation_mean'] = test_df['user_id_prior_question_had_explanation_currentsum']/(test_df['user_id_answered_correctly_count']+1)\n        test_df['user_id_content_id_ptp_mean'] = test_df['user_id_content_id_ptp_sum']/test_df['user_id_answered_correctly_count']\n        test_df['user_id_content_id_timestamp_diff'] = test_df['timestamp'] - test_df['user_id_content_id_timestamp_last']\n        #test_df['user_id_content_id_user_id_answered_correctly_count_last_diff'] = test_df['user_id_answered_correctly_count'] - test_df['user_id_content_id_user_id_answered_correctly_count_last']\n        test_df['user_id_part_timestamp_diff'] = test_df['timestamp'] - test_df['user_id_part_timestamp_last']\n        #test_df['user_id_part_user_id_answered_correctly_count_last_diff'] = test_df['user_id_answered_correctly_count'] - test_df['user_id_part_user_id_answered_correctly_count_last']\n        \n        test_df['answered_correctly_avg_c_div_b'] = test_df['answered_correctly_avg_c']/test_df['answered_correctly_avg_b']\n        test_df['user_id_cluster_answered_correctly_mean'] = test_df['user_id_cluster_answered_correctly_sum'] / test_df['user_id_cluster_answered_correctly_count']  \n        \n        test_df['user_id_tags1_answer_mean'] = test_df['user_id_tags1_answer_sum']/test_df['user_id_tags1_answer_count']\n        test_df['user_id_tags2_answer_mean'] = test_df['user_id_tags2_answer_sum']/test_df['user_id_tags2_answer_count']\n        test_df['user_id_tags3_answer_mean'] = test_df['user_id_tags3_answer_sum']/test_df['user_id_tags3_answer_count']\n        test_df['user_id_tags4_answer_mean'] = test_df['user_id_tags4_answer_sum']/test_df['user_id_tags4_answer_count']\n        test_df['user_id_tags1_answer_diff'] = test_df['timestamp'] - test_df['user_id_tags1_answer_last']\n        test_df['user_id_tags2_answer_diff'] = test_df['timestamp'] - test_df['user_id_tags2_answer_last']\n        test_df['user_id_tags3_answer_diff'] = test_df['timestamp'] - test_df['user_id_tags3_answer_last']\n        test_df['user_id_tags4_answer_diff'] = test_df['timestamp'] - test_df['user_id_tags4_answer_last']\n\n        #lgb_pred = clf.predict(test_df[train_columns])\n        #cat_pred =  clf_cat.predict_proba(test_df[train_columns])[:,1]\n        test_df[target] = clf.predict(test_df[train_columns]) #lgb_pred*0.5 + cat_pred*0.5\n        set_predict(test_df[['row_id', target]])\n        #del lgb_pred\n        #del cat_pred\n        #gc.collect()\n    except BaseException as e:\n        print(e)\n        continue","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}