{"cells":[{"metadata":{},"cell_type":"markdown","source":"This will likely be the last public notebook I release for a while. I wanted to clean up my prior LGBM model and give myself a solid baseline before using other models.\n\nI really need to find better ways of getting training and validation data. Grouping by user_id and sampling task_container_id?\n\nThere is a lot more work to do in generating better student/user features.\n\nI generated the question2 file using bigquery to aggregate question percentage correct and count of questions from the train data with the original question data."},{"metadata":{"trusted":true},"cell_type":"code","source":"#task_container_id should be monotonically increasing by user, but it isn't, should it be fixed?\n#train['task_container_id'] = (\n#    train\n#    .groupby('user_id')['task_container_id']\n#    .transform(lambda x: pd.factorize(x)[0])\n#    .astype('int16')\n#)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Reading Data and Importing Libraries ##"},{"metadata":{"trusted":true},"cell_type":"code","source":"#!pip install ../input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n#import datatable as dt\nimport pandas as pd\nimport numpy as np\nfrom sklearn.metrics import roc_auc_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train = dt.fread('/kaggle/input/riiid-test-answer-prediction/train.csv')\n#del train[:, ('row_id', 'user_answer', 'prior_question_had_explanation')]\n#train = train.sort(['user_id', 'timestamp'])\n#train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train = train[dt.f.content_type_id== 0, :]\n#del(train[:,'content_type_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train = train.to_pandas()\n#train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',\n                   usecols=[1, 2, 3],\n                   dtype={'timestamp': 'int64',\n                          'user_id': 'int32',\n                          'content_id': 'int16',\n                          'content_type_id': 'int8',\n                          'task_container_id': 'int16',\n                          'answered_correctly':'int8',\n                          'prior_question_elapsed_time': 'float32'}\n                   )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train[train.content_type_id == 0]\ntrain.drop('content_type_id', axis = 1, inplace = True)\ntrain = train.sort_values(['user_id','timestamp'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Affirmatives (True) for content_type_id are only for those with a different type of content (lectures). These are not real questions."},{"metadata":{"trusted":true},"cell_type":"code","source":"#saving value to fillna\nelapsed_mean = train.prior_question_elapsed_time.mean()\n\ntrain.prior_question_elapsed_time.fillna(elapsed_mean, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Saving Grouped Features ##"},{"metadata":{"trusted":true},"cell_type":"code","source":"group1 = train[['task_container_id', 'user_id']].groupby(['task_container_id']).agg(['count'])\ngroup1.columns = ['avg_questions']\ngroup2 = train[['task_container_id', 'user_id']].groupby(['task_container_id']).agg(['nunique'])\ngroup2.columns = ['avg_questions']\ngroup3 = group1 / group2\ndel(group1, group2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#supposed to be monotonically increasing by user, but not quite, close enough hopefully\ngroup3['avg_questions_seen'] = group3.avg_questions.cumsum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pickle as pkl\n\ngroup3.to_pickle('./group3.pkl', compression='infer')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results_u_final = train[['user_id','answered_correctly']].groupby(['user_id']).agg(['mean'])\nresults_u_final.columns = ['answered_correctly_user']\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results_u_final.to_pickle('./results_u_final.pkl', compression='infer')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Exploration ##"},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.answered_correctly.mean()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Timestamp Features ##"},{"metadata":{"trusted":true},"cell_type":"code","source":"train['time_since_last_action'] = train['timestamp'] - train.timestamp.shift(1)\n\ntrain.time_since_last_action.replace(to_replace=0, method = 'ffill', inplace = True)\n\ntrain.loc[train.time_since_last_action < 0, 'time_since_last_action'] = np.nan\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"time_since_median = train.time_since_last_action.median()\n\ntrain.time_since_last_action.fillna(time_since_median, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Extracting Training Data ##"},{"metadata":{"trusted":true},"cell_type":"code","source":"X = train.sample(frac=.15, random_state = 100)\ntrain = train[~train.index.isin(X.index)]\nlen(X) + len(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.answered_correctly.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.answered_correctly.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.task_container_id.value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Creating Validation Set ##"},{"metadata":{"trusted":true},"cell_type":"code","source":"validation = train.sample(frac=.03, random_state = 100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train[~train.index.isin(validation.index)]\nlen(train) + len(validation) + len(X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validation.answered_correctly.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.answered_correctly.mean()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Does it make sense to use last questions as validation? Why is the rate of correct answers so low?\nI am convinced there is a better way to match the test data."},{"metadata":{},"cell_type":"markdown","source":"## Merging Data ##"},{"metadata":{"trusted":true},"cell_type":"code","source":"#clearing memory\ndel(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.read_pickle(\"./.pkl\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = pd.merge(X, group3, left_on=['task_container_id'], right_index= True, how=\"left\")\nX = pd.merge(X, results_u_final, on=['user_id'], how=\"left\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validation = pd.merge(validation, group3, left_on=['task_container_id'], right_index= True, how=\"left\")\nvalidation = pd.merge(validation, results_u_final, on=['user_id'], how=\"left\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#reading in question df\nquestion2 = pd.read_csv('/kaggle/input/question2/question2.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"content_mean = question2.quest_pct.mean()\n\nquestion2.quest_pct.mean()\n#there are a lot of high percentage questions, should use median instead?","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#filling questions with no info with a new value\nquestion2.quest_pct = question2.quest_pct.mask((question2['count'] < 4), .65)\n\n\n#filling very hard new questions with a more reasonable value\nquestion2.quest_pct = question2.quest_pct.mask((question2.quest_pct < .2) & (question2['count'] < 21), .2)\n\n#filling very easy new questions with a more reasonable value\nquestion2.quest_pct = question2.quest_pct.mask((question2.quest_pct > .95) & (question2['count'] < 21), .95)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question2.to_pickle('./question2.pkl', compression='infer')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = pd.merge(X, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')\nvalidation = pd.merge(validation, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')\nX.part = X.part - 1\nvalidation.part = validation.part - 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = X['answered_correctly']\nX = X.drop(['answered_correctly'], axis=1)\nX.head()\n\ny_val = validation['answered_correctly']\nX_val = validation.drop(['answered_correctly'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#try to concat this to test if it works later\n#time_df = validation[['user_id', 'content_id', 'timestamp', 'time_since_last_action']].drop_duplicates('user_id', keep = 'last')\n#not working on test data yet.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = X[['answered_correctly_user', 'quest_pct', 'avg_questions_seen', 'prior_question_elapsed_time', 'time_since_last_action', 'part', 'count']]\nX_val = X_val[['answered_correctly_user', 'quest_pct', 'avg_questions_seen', 'prior_question_elapsed_time', 'time_since_last_action', 'part', 'count']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X['answered_correctly_user'].fillna(0.65,  inplace=True)\nX['quest_pct'].fillna(content_mean, inplace=True)\n\nX['part'].fillna(4, inplace = True)\nX['avg_questions_seen'].fillna(1, inplace = True)\nX['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_val['answered_correctly_user'].fillna(0.65,  inplace=True)\nX_val['quest_pct'].fillna(content_mean,  inplace=True)\n\nX_val['part'].fillna(4, inplace = True)\nX['avg_questions_seen'].fillna(1, inplace = True)\nX_val['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(X), len(X_val))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Modeling ##"},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\n\nparams = {\n    'objective': 'binary',\n    'boosting' : 'gbdt',\n    'metric': 'auc'\n    'max_bin': 1500,\n    'learning_rate': 0.0175,\n    'num_leaves': 80\n}\n\nlgb_train = lgb.Dataset(X, y, categorical_feature = ['part'])\nlgb_eval = lgb.Dataset(X_val, y_val, categorical_feature = ['part'], reference=lgb_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del(X, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = lgb.train(\n    params, lgb_train,\n    valid_sets=[lgb_train, lgb_eval],\n    verbose_eval=50,\n    num_boost_round=1000,\n    early_stopping_rounds=8\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = model.predict(X_val)\ny_true = np.array(y_val)\nroc_auc_score(y_true, y_pred)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Examining Feature Importance ##"},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#displaying the most important features by split\nlgb.plot_importance(model)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#displaying the most important features by gain\nlgb.plot_importance(model, importance_type = 'gain')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# save model to file\nmodel.save_model('./lgbm_model.txt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bst = lgb.Booster(model_file='./lgbm_model.txt')\n#y_pred = bst.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_val.iloc[0].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_val.iloc[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Making Predictions for New Data ##"},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    test_df['task_container_id'] = test_df.task_container_id.mask(test_df.task_container_id > 9999, 9999)\n    test_df = pd.merge(test_df, group3, left_on=['task_container_id'], right_index= True, how=\"left\")\n    test_df = pd.merge(test_df, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')\n    test_df = pd.merge(test_df, results_u_final, on=['user_id'],  how=\"left\")\n    test_df['answered_correctly_user'].fillna(0.65,  inplace=True)\n    test_df['quest_pct'].fillna(content_mean,  inplace=True)\n    test_df['part'] = test_df.part - 1\n\n    test_df['part'].fillna(4, inplace = True)\n    test_df['avg_questions_seen'].fillna(1, inplace = True)\n    test_df['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)\n    test_df['prior_question_had_explanation'].fillna(False, inplace=True)\n    \n    test_df['answered_correctly'] =  model.predict(test_df[['answered_correctly_user', 'quest_pct', 'avg_questions_seen', \n                                                            'prior_question_elapsed_time', 'time_since_last_action', 'part', 'count']])\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"time_since_median","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#students don't appear in every task container ID what can I do about this, can't always follow sequentially?","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#secret code for making calculation work\n'''time_df = pd.concat([time_df, test_df[['user_id', 'content_id', 'timestamp']]], axis=0, ignore_index = True)\n\ntime_df = time_df.sort_values(['user_id','timestamp'])\n\ntime_df['prev_timestamp'] = time_df.timestamp.shift(1)\n\ntime_df['time_since_last_action'] = time_df['timestamp'] - time_df['prev_timestamp']\n\ntime_df.time_since_last_action.replace(to_replace=0, method = 'ffill', inplace = True)\n\ntime_df.loc[time_df.time_since_last_action < 0, 'time_since_last_action'] = np.nan\n\ntime_df.time_since_last_action.fillna(time_since_median, inplace = True)\n\ntest_df = pd.merge(test_df, time_df, on=['user_id', 'content_id', 'timestamp'], how = \"left\")\n\ntest_df.time_since_last_action.fillna(time_since_median, inplace = True)'''\n\n#use map instead if you only have a single column\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}