{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Reading Data and Importing Libraries ##","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.metrics import roc_auc_score","metadata":{"execution":{"iopub.status.busy":"2021-07-12T07:29:03.890345Z","iopub.execute_input":"2021-07-12T07:29:03.890673Z","iopub.status.idle":"2021-07-12T07:29:03.895044Z","shell.execute_reply.started":"2021-07-12T07:29:03.890643Z","shell.execute_reply":"2021-07-12T07:29:03.893937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',\n                   usecols=[1, 2, 3, 4, 5, 7, 8, 9],\n                   dtype={'timestamp': 'int64',\n                          'user_id': 'int32',\n                          'content_id': 'int16',\n                          'content_type_id': 'int8',\n                          'task_container_id': 'int16',\n                          'answered_correctly':'int8',\n                          'prior_question_elapsed_time': 'float32',\n                          'prior_question_had_explanation': 'boolean'}\n                   )","metadata":{"execution":{"iopub.status.busy":"2021-07-12T07:29:07.431674Z","iopub.execute_input":"2021-07-12T07:29:07.432015Z","iopub.status.idle":"2021-07-12T07:32:34.378146Z","shell.execute_reply.started":"2021-07-12T07:29:07.431984Z","shell.execute_reply":"2021-07-12T07:32:34.377318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#reading in question df\nquestions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',                         \n                            usecols=[0, 3],\n                            dtype={'question_id': 'int16',\n                              'part': 'int8'}\n                          )","metadata":{"execution":{"iopub.status.busy":"2021-07-12T07:32:47.762002Z","iopub.execute_input":"2021-07-12T07:32:47.762336Z","iopub.status.idle":"2021-07-12T07:32:47.783949Z","shell.execute_reply.started":"2021-07-12T07:32:47.762306Z","shell.execute_reply":"2021-07-12T07:32:47.783158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#reading in lecture df\nlectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv')","metadata":{"execution":{"iopub.status.busy":"2021-07-12T07:32:49.681454Z","iopub.execute_input":"2021-07-12T07:32:49.681785Z","iopub.status.idle":"2021-07-12T07:32:49.694714Z","shell.execute_reply.started":"2021-07-12T07:32:49.681754Z","shell.execute_reply":"2021-07-12T07:32:49.693961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')\n\nlectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])\n\npart_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]\n\ntypes_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]","metadata":{"execution":{"iopub.status.busy":"2021-07-12T07:32:51.262582Z","iopub.execute_input":"2021-07-12T07:32:51.262912Z","iopub.status.idle":"2021-07-12T07:32:51.284309Z","shell.execute_reply.started":"2021-07-12T07:32:51.262863Z","shell.execute_reply":"2021-07-12T07:32:51.283086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"part_lectures_columns","metadata":{"execution":{"iopub.status.busy":"2021-07-12T07:32:59.849534Z","iopub.execute_input":"2021-07-12T07:32:59.849848Z","iopub.status.idle":"2021-07-12T07:32:59.856324Z","shell.execute_reply.started":"2021-07-12T07:32:59.849818Z","shell.execute_reply":"2021-07-12T07:32:59.855489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"types_of_lectures_columns","metadata":{"execution":{"iopub.status.busy":"2021-07-12T07:33:01.881638Z","iopub.execute_input":"2021-07-12T07:33:01.881963Z","iopub.status.idle":"2021-07-12T07:33:01.886664Z","shell.execute_reply.started":"2021-07-12T07:33:01.881932Z","shell.execute_reply":"2021-07-12T07:33:01.885848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lectures_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-12T07:33:09.245581Z","iopub.execute_input":"2021-07-12T07:33:09.245925Z","iopub.status.idle":"2021-07-12T07:33:09.262212Z","shell.execute_reply.started":"2021-07-12T07:33:09.245887Z","shell.execute_reply":"2021-07-12T07:33:09.26109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# merge lecture features to train dataset\ntrain_lectures = train[train.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left')","metadata":{"execution":{"iopub.status.busy":"2021-07-12T07:33:13.768924Z","iopub.execute_input":"2021-07-12T07:33:13.76925Z","iopub.status.idle":"2021-07-12T07:33:14.5194Z","shell.execute_reply.started":"2021-07-12T07:33:13.76922Z","shell.execute_reply":"2021-07-12T07:33:14.518623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_lectures.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-12T07:33:16.349166Z","iopub.execute_input":"2021-07-12T07:33:16.349471Z","iopub.status.idle":"2021-07-12T07:33:16.376233Z","shell.execute_reply.started":"2021-07-12T07:33:16.349443Z","shell.execute_reply":"2021-07-12T07:33:16.375273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# collect per user stats\nuser_lecture_stats_part = train_lectures.groupby('user_id')[part_lectures_columns + types_of_lectures_columns].sum()","metadata":{"execution":{"iopub.status.busy":"2021-07-12T07:33:26.31675Z","iopub.execute_input":"2021-07-12T07:33:26.317092Z","iopub.status.idle":"2021-07-12T07:33:26.521144Z","shell.execute_reply.started":"2021-07-12T07:33:26.317062Z","shell.execute_reply":"2021-07-12T07:33:26.520386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_lecture_stats_part.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-12T07:33:31.767796Z","iopub.execute_input":"2021-07-12T07:33:31.768164Z","iopub.status.idle":"2021-07-12T07:33:31.783718Z","shell.execute_reply.started":"2021-07-12T07:33:31.768133Z","shell.execute_reply":"2021-07-12T07:33:31.782486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# add boolean features\nfor column in user_lecture_stats_part.columns:\n    bool_column = column + '_boolean'\n    user_lecture_stats_part[bool_column] = (user_lecture_stats_part[column] > 0).astype(int)","metadata":{"execution":{"iopub.status.busy":"2021-07-12T07:33:37.04569Z","iopub.execute_input":"2021-07-12T07:33:37.046087Z","iopub.status.idle":"2021-07-12T07:33:37.06801Z","shell.execute_reply.started":"2021-07-12T07:33:37.04605Z","shell.execute_reply":"2021-07-12T07:33:37.067197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_lecture_stats_part.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-12T07:33:43.336322Z","iopub.execute_input":"2021-07-12T07:33:43.336633Z","iopub.status.idle":"2021-07-12T07:33:43.357255Z","shell.execute_reply.started":"2021-07-12T07:33:43.336603Z","shell.execute_reply":"2021-07-12T07:33:43.356362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#clearing memory\ndel(train_lectures)","metadata":{"execution":{"iopub.status.busy":"2021-07-12T07:33:52.565702Z","iopub.execute_input":"2021-07-12T07:33:52.566052Z","iopub.status.idle":"2021-07-12T07:33:52.569723Z","shell.execute_reply.started":"2021-07-12T07:33:52.566021Z","shell.execute_reply":"2021-07-12T07:33:52.568865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Affirmatives (True) for content_type_id are only for those with a different type of content (lectures). These are not real questions.","metadata":{}},{"cell_type":"code","source":"#removing True or 1 for content_type_id\n\ntrain = train[train.content_type_id == False].sort_values('timestamp').reset_index(drop = True)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T17:27:20.530273Z","iopub.execute_input":"2021-06-25T17:27:20.530671Z","iopub.status.idle":"2021-06-25T17:27:48.845776Z","shell.execute_reply.started":"2021-06-25T17:27:20.530635Z","shell.execute_reply":"2021-06-25T17:27:48.842859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[(train.task_container_id == 9999)].tail()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T17:28:54.109376Z","iopub.execute_input":"2021-06-25T17:28:54.109685Z","iopub.status.idle":"2021-06-25T17:28:54.180513Z","shell.execute_reply.started":"2021-06-25T17:28:54.109655Z","shell.execute_reply":"2021-06-25T17:28:54.179775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[(train.content_type_id == False)].task_container_id.nunique()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T17:29:45.854566Z","iopub.execute_input":"2021-06-25T17:29:45.854952Z","iopub.status.idle":"2021-06-25T17:29:50.585898Z","shell.execute_reply.started":"2021-06-25T17:29:45.854919Z","shell.execute_reply":"2021-06-25T17:29:50.584494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#saving value to fillna\nelapsed_mean = train.prior_question_elapsed_time.mean()\n","metadata":{"execution":{"iopub.status.busy":"2021-06-25T17:29:52.795931Z","iopub.execute_input":"2021-06-25T17:29:52.796264Z","iopub.status.idle":"2021-06-25T17:29:52.934564Z","shell.execute_reply.started":"2021-06-25T17:29:52.796232Z","shell.execute_reply":"2021-06-25T17:29:52.933775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"group1 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id']).agg(['count'])\ngroup1.columns = ['avg_questions']\ngroup2 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id']).agg(['nunique'])\ngroup2.columns = ['avg_questions']\ngroup3 = group1 / group2","metadata":{"execution":{"iopub.status.busy":"2021-06-25T17:30:17.480224Z","iopub.execute_input":"2021-06-25T17:30:17.480554Z","iopub.status.idle":"2021-06-25T17:31:33.361193Z","shell.execute_reply.started":"2021-06-25T17:30:17.480522Z","shell.execute_reply":"2021-06-25T17:31:33.360381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"group3['avg_questions_seen'] = group3.avg_questions.cumsum()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T17:54:49.184271Z","iopub.execute_input":"2021-06-25T17:54:49.18462Z","iopub.status.idle":"2021-06-25T17:54:49.193447Z","shell.execute_reply.started":"2021-06-25T17:54:49.184588Z","shell.execute_reply":"2021-06-25T17:54:49.192204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"group3.iloc[0].avg_questions_seen","metadata":{"execution":{"iopub.status.busy":"2021-06-25T17:54:57.418114Z","iopub.execute_input":"2021-06-25T17:54:57.418431Z","iopub.status.idle":"2021-06-25T17:54:57.427376Z","shell.execute_reply.started":"2021-06-25T17:54:57.418398Z","shell.execute_reply":"2021-06-25T17:54:57.423973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_u_final = train.loc[train.content_type_id == False, ['user_id','answered_correctly']].groupby(['user_id']).agg(['mean'])\nresults_u_final.columns = ['answered_correctly_user']\n\nresults_u2_final = train.loc[train.content_type_id == False, ['user_id','prior_question_had_explanation']].groupby(['user_id']).agg(['mean'])\nresults_u2_final.columns = ['explanation_mean_user']","metadata":{"execution":{"iopub.status.busy":"2021-06-25T17:55:34.194821Z","iopub.execute_input":"2021-06-25T17:55:34.195151Z","iopub.status.idle":"2021-06-25T17:56:07.080945Z","shell.execute_reply.started":"2021-06-25T17:55:34.195121Z","shell.execute_reply":"2021-06-25T17:56:07.080123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_u2_final.explanation_mean_user.describe()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:49:06.128953Z","iopub.execute_input":"2021-06-25T15:49:06.129335Z","iopub.status.idle":"2021-06-25T15:49:06.157422Z","shell.execute_reply.started":"2021-06-25T15:49:06.129303Z","shell.execute_reply":"2021-06-25T15:49:06.156174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.merge(train, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left')","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:49:19.7344Z","iopub.execute_input":"2021-06-25T15:49:19.734781Z","iopub.status.idle":"2021-06-25T15:49:47.710543Z","shell.execute_reply.started":"2021-06-25T15:49:19.734749Z","shell.execute_reply":"2021-06-25T15:49:47.709203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_q_final = train.loc[train.content_type_id == False, ['question_id','answered_correctly']].groupby(['question_id']).agg(['mean'])\nresults_q_final.columns = ['quest_pct']","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:49:47.712667Z","iopub.execute_input":"2021-06-25T15:49:47.713215Z","iopub.status.idle":"2021-06-25T15:50:12.080543Z","shell.execute_reply.started":"2021-06-25T15:49:47.71314Z","shell.execute_reply":"2021-06-25T15:50:12.079178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_q2_final = train.loc[train.content_type_id == False, ['question_id','part']].groupby(['question_id']).agg(['count'])\nresults_q2_final.columns = ['count']","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:50:38.33204Z","iopub.execute_input":"2021-06-25T15:50:38.332393Z","iopub.status.idle":"2021-06-25T15:51:01.640664Z","shell.execute_reply.started":"2021-06-25T15:50:38.332361Z","shell.execute_reply":"2021-06-25T15:51:01.63955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"question2 = pd.merge(questions_df, results_q_final, left_on = 'question_id', right_on = 'question_id', how = 'left')","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:51:05.326734Z","iopub.execute_input":"2021-06-25T15:51:05.32713Z","iopub.status.idle":"2021-06-25T15:51:05.343906Z","shell.execute_reply.started":"2021-06-25T15:51:05.327097Z","shell.execute_reply":"2021-06-25T15:51:05.342577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"question2 = pd.merge(question2, results_q2_final, left_on = 'question_id', right_on = 'question_id', how = 'left')","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:51:07.942036Z","iopub.execute_input":"2021-06-25T15:51:07.942416Z","iopub.status.idle":"2021-06-25T15:51:07.958229Z","shell.execute_reply.started":"2021-06-25T15:51:07.942382Z","shell.execute_reply":"2021-06-25T15:51:07.957178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"question2.quest_pct = round(question2.quest_pct,5)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:51:10.664806Z","iopub.execute_input":"2021-06-25T15:51:10.665197Z","iopub.status.idle":"2021-06-25T15:51:10.671978Z","shell.execute_reply.started":"2021-06-25T15:51:10.665164Z","shell.execute_reply":"2021-06-25T15:51:10.670635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(question2.head(), question2.tail())","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:51:13.537443Z","iopub.execute_input":"2021-06-25T15:51:13.537832Z","iopub.status.idle":"2021-06-25T15:51:13.563665Z","shell.execute_reply.started":"2021-06-25T15:51:13.5378Z","shell.execute_reply":"2021-06-25T15:51:13.56172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:51:42.620023Z","iopub.execute_input":"2021-06-25T15:51:42.620382Z","iopub.status.idle":"2021-06-25T15:51:42.637374Z","shell.execute_reply.started":"2021-06-25T15:51:42.620349Z","shell.execute_reply":"2021-06-25T15:51:42.635973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:51:44.238544Z","iopub.execute_input":"2021-06-25T15:51:44.238976Z","iopub.status.idle":"2021-06-25T15:51:44.245943Z","shell.execute_reply.started":"2021-06-25T15:51:44.238944Z","shell.execute_reply":"2021-06-25T15:51:44.244652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Exploration ##","metadata":{}},{"cell_type":"code","source":"len(train)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:51:54.130543Z","iopub.execute_input":"2021-06-25T15:51:54.130957Z","iopub.status.idle":"2021-06-25T15:51:54.140014Z","shell.execute_reply.started":"2021-06-25T15:51:54.130924Z","shell.execute_reply":"2021-06-25T15:51:54.138765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.answered_correctly.mean()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:51:55.610826Z","iopub.execute_input":"2021-06-25T15:51:55.611194Z","iopub.status.idle":"2021-06-25T15:51:55.717773Z","shell.execute_reply.started":"2021-06-25T15:51:55.611164Z","shell.execute_reply":"2021-06-25T15:51:55.716607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prior_mean_user = results_u2_final.explanation_mean_user.mean()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:51:58.500316Z","iopub.execute_input":"2021-06-25T15:51:58.500768Z","iopub.status.idle":"2021-06-25T15:51:58.50727Z","shell.execute_reply.started":"2021-06-25T15:51:58.500735Z","shell.execute_reply":"2021-06-25T15:51:58.505883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.loc[(train.timestamp == 0)].answered_correctly.mean()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:52:02.525261Z","iopub.execute_input":"2021-06-25T15:52:02.525676Z","iopub.status.idle":"2021-06-25T15:52:02.682134Z","shell.execute_reply.started":"2021-06-25T15:52:02.525643Z","shell.execute_reply":"2021-06-25T15:52:02.681239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.loc[(train.timestamp != 0)].answered_correctly.mean()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:52:07.335778Z","iopub.execute_input":"2021-06-25T15:52:07.336129Z","iopub.status.idle":"2021-06-25T15:52:12.406553Z","shell.execute_reply.started":"2021-06-25T15:52:07.336098Z","shell.execute_reply":"2021-06-25T15:52:12.405618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop(['timestamp', 'content_type_id', 'question_id', 'part'], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:52:18.294357Z","iopub.execute_input":"2021-06-25T15:52:18.2948Z","iopub.status.idle":"2021-06-25T15:52:19.127971Z","shell.execute_reply.started":"2021-06-25T15:52:18.294766Z","shell.execute_reply":"2021-06-25T15:52:19.126889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:52:20.754114Z","iopub.execute_input":"2021-06-25T15:52:20.754494Z","iopub.status.idle":"2021-06-25T15:52:20.761653Z","shell.execute_reply.started":"2021-06-25T15:52:20.754462Z","shell.execute_reply":"2021-06-25T15:52:20.760276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating Validation Set (Most Recent Answers by User) ##","metadata":{}},{"cell_type":"code","source":"validation = train.groupby('user_id').tail(5)\ntrain = train[~train.index.isin(validation.index)]\nlen(train) + len(validation)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:52:30.183881Z","iopub.execute_input":"2021-06-25T15:52:30.184279Z","iopub.status.idle":"2021-06-25T15:52:53.636685Z","shell.execute_reply.started":"2021-06-25T15:52:30.184226Z","shell.execute_reply":"2021-06-25T15:52:53.635458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation.answered_correctly.mean()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:52:55.989176Z","iopub.execute_input":"2021-06-25T15:52:55.989558Z","iopub.status.idle":"2021-06-25T15:52:56.00123Z","shell.execute_reply.started":"2021-06-25T15:52:55.989508Z","shell.execute_reply":"2021-06-25T15:52:55.999642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.answered_correctly.mean()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:52:57.776697Z","iopub.execute_input":"2021-06-25T15:52:57.777101Z","iopub.status.idle":"2021-06-25T15:52:57.881128Z","shell.execute_reply.started":"2021-06-25T15:52:57.777067Z","shell.execute_reply":"2021-06-25T15:52:57.879736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_u_val = train[['user_id','answered_correctly']].groupby(['user_id']).agg(['mean'])\nresults_u_val.columns = ['answered_correctly_user']\n\nresults_u2_val = train[['user_id','prior_question_had_explanation']].groupby(['user_id']).agg(['mean'])\nresults_u2_val.columns = ['explanation_mean_user']","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:53:00.822205Z","iopub.execute_input":"2021-06-25T15:53:00.822617Z","iopub.status.idle":"2021-06-25T15:53:24.564359Z","shell.execute_reply.started":"2021-06-25T15:53:00.822585Z","shell.execute_reply":"2021-06-25T15:53:24.563163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Does it make sense to use last questions as validation? Why is the rate of correct answers so low?\nI am convinced there is a better way to match the test data.","metadata":{}},{"cell_type":"markdown","source":"## Extracting Training Data ##","metadata":{}},{"cell_type":"code","source":"X = train.groupby('user_id').tail(18)\ntrain = train[~train.index.isin(X.index)]\nlen(X) + len(train) + len(validation)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:53:39.427802Z","iopub.execute_input":"2021-06-25T15:53:39.428215Z","iopub.status.idle":"2021-06-25T15:54:06.097048Z","shell.execute_reply.started":"2021-06-25T15:53:39.428183Z","shell.execute_reply":"2021-06-25T15:54:06.095686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.answered_correctly.mean()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:56:51.824132Z","iopub.execute_input":"2021-06-25T15:56:51.824547Z","iopub.status.idle":"2021-06-25T15:56:51.841134Z","shell.execute_reply.started":"2021-06-25T15:56:51.824497Z","shell.execute_reply":"2021-06-25T15:56:51.839834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.answered_correctly.mean()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:57:06.585335Z","iopub.execute_input":"2021-06-25T15:57:06.585713Z","iopub.status.idle":"2021-06-25T15:57:06.67939Z","shell.execute_reply.started":"2021-06-25T15:57:06.58568Z","shell.execute_reply":"2021-06-25T15:57:06.678078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_u_X = train[['user_id','answered_correctly']].groupby(['user_id']).agg(['mean'])\nresults_u_X.columns = ['answered_correctly_user']\n\nresults_u2_X = train[['user_id','prior_question_had_explanation']].groupby(['user_id']).agg(['mean'])\nresults_u2_X.columns = ['explanation_mean_user']","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:57:16.67882Z","iopub.execute_input":"2021-06-25T15:57:16.679233Z","iopub.status.idle":"2021-06-25T15:57:38.410437Z","shell.execute_reply.started":"2021-06-25T15:57:16.679198Z","shell.execute_reply":"2021-06-25T15:57:38.409156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Merging Data ##","metadata":{}},{"cell_type":"code","source":"#clearing memory\ndel(train)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:57:43.317465Z","iopub.execute_input":"2021-06-25T15:57:43.317975Z","iopub.status.idle":"2021-06-25T15:57:43.429329Z","shell.execute_reply.started":"2021-06-25T15:57:43.317928Z","shell.execute_reply":"2021-06-25T15:57:43.427968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = pd.merge(X, group3, left_on=['task_container_id'], right_index= True, how=\"left\")\nX = pd.merge(X, results_u_X, on=['user_id'], how=\"left\")\nX = pd.merge(X, results_u2_X, on=['user_id'], how=\"left\")\n\nX = pd.merge(X, user_lecture_stats_part, on=['user_id'], how=\"left\")","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:57:53.309488Z","iopub.execute_input":"2021-06-25T15:57:53.309917Z","iopub.status.idle":"2021-06-25T15:57:58.68491Z","shell.execute_reply.started":"2021-06-25T15:57:53.309882Z","shell.execute_reply":"2021-06-25T15:57:58.683987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation = pd.merge(validation, group3, left_on=['task_container_id'], right_index= True, how=\"left\")\nvalidation = pd.merge(validation, results_u_val, on=['user_id'], how=\"left\")\nvalidation = pd.merge(validation, results_u2_val, on=['user_id'], how=\"left\")\n\nvalidation = pd.merge(validation, user_lecture_stats_part, on=['user_id'], how=\"left\")","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:58:02.564808Z","iopub.execute_input":"2021-06-25T15:58:02.565196Z","iopub.status.idle":"2021-06-25T15:58:04.001686Z","shell.execute_reply.started":"2021-06-25T15:58:02.565164Z","shell.execute_reply":"2021-06-25T15:58:04.000751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nlb_make = LabelEncoder()\n\nX.prior_question_had_explanation.fillna(False, inplace = True)\nvalidation.prior_question_had_explanation.fillna(False, inplace = True)\n\nvalidation[\"prior_question_had_explanation_enc\"] = lb_make.fit_transform(validation[\"prior_question_had_explanation\"])\nX[\"prior_question_had_explanation_enc\"] = lb_make.fit_transform(X[\"prior_question_had_explanation\"])","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:58:25.750512Z","iopub.execute_input":"2021-06-25T15:58:25.750917Z","iopub.status.idle":"2021-06-25T15:58:28.53608Z","shell.execute_reply.started":"2021-06-25T15:58:25.750884Z","shell.execute_reply":"2021-06-25T15:58:28.534934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#reading in question df\n#question2 = pd.read_csv('/kaggle/input/question2/question2.csv)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:58:30.431973Z","iopub.execute_input":"2021-06-25T15:58:30.432493Z","iopub.status.idle":"2021-06-25T15:58:30.440791Z","shell.execute_reply.started":"2021-06-25T15:58:30.432459Z","shell.execute_reply":"2021-06-25T15:58:30.439364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"content_mean = question2.quest_pct.mean()\n\nquestion2.quest_pct.mean()\n#there are a lot of high percentage questions, should use median instead?","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:58:32.761478Z","iopub.execute_input":"2021-06-25T15:58:32.761949Z","iopub.status.idle":"2021-06-25T15:58:32.773057Z","shell.execute_reply.started":"2021-06-25T15:58:32.761915Z","shell.execute_reply":"2021-06-25T15:58:32.771905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#filling questions with no info with a new value\nquestion2.quest_pct = question2.quest_pct.mask((question2['count'] < 3), .65)\n\n\n#filling very hard new questions with a more reasonable value\nquestion2.quest_pct = question2.quest_pct.mask((question2.quest_pct < .2) & (question2['count'] < 21), .2)\n\n#filling very easy new questions with a more reasonable value\nquestion2.quest_pct = question2.quest_pct.mask((question2.quest_pct > .95) & (question2['count'] < 21), .95)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:58:43.283649Z","iopub.execute_input":"2021-06-25T15:58:43.284023Z","iopub.status.idle":"2021-06-25T15:58:43.301257Z","shell.execute_reply.started":"2021-06-25T15:58:43.28399Z","shell.execute_reply":"2021-06-25T15:58:43.300253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = pd.merge(X, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')\nvalidation = pd.merge(validation, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')\nX.part = X.part - 1\nvalidation.part = validation.part - 1","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:58:48.406771Z","iopub.execute_input":"2021-06-25T15:58:48.407191Z","iopub.status.idle":"2021-06-25T15:58:52.154392Z","shell.execute_reply.started":"2021-06-25T15:58:48.407158Z","shell.execute_reply":"2021-06-25T15:58:52.152971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:58:55.943857Z","iopub.execute_input":"2021-06-25T15:58:55.944316Z","iopub.status.idle":"2021-06-25T15:58:55.987288Z","shell.execute_reply.started":"2021-06-25T15:58:55.944284Z","shell.execute_reply":"2021-06-25T15:58:55.985947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = X['answered_correctly']\nX = X.drop(['answered_correctly'], axis=1)\nX.head()\n\ny_val = validation['answered_correctly']\nX_val = validation.drop(['answered_correctly'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:58:59.174449Z","iopub.execute_input":"2021-06-25T15:58:59.174928Z","iopub.status.idle":"2021-06-25T15:59:01.562491Z","shell.execute_reply.started":"2021-06-25T15:58:59.174894Z","shell.execute_reply":"2021-06-25T15:59:01.561172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = X[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',\n       'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',\n       'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',\n       'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',\n       'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',\n       'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']]\nX_val = X_val[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',\n               'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',\n               'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',\n               'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',\n               'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',\n               'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']]","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:59:11.37789Z","iopub.execute_input":"2021-06-25T15:59:11.378311Z","iopub.status.idle":"2021-06-25T15:59:12.125077Z","shell.execute_reply.started":"2021-06-25T15:59:11.378279Z","shell.execute_reply":"2021-06-25T15:59:12.124128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Filling with 0.5 for simplicity; there could likely be a better value\nX['answered_correctly_user'].fillna(0.65,  inplace=True)\nX['explanation_mean_user'].fillna(prior_mean_user,  inplace=True)\nX['quest_pct'].fillna(content_mean, inplace=True)\n\nX['part'].fillna(4, inplace = True)\nX['avg_questions_seen'].fillna(1, inplace = True)\nX['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)\nX['prior_question_had_explanation_enc'].fillna(0, inplace = True)\n\nX['part_1'].fillna(0, inplace = True)\nX['part_2'].fillna(0, inplace = True)\nX['part_3'].fillna(0, inplace = True)\nX['part_4'].fillna(0, inplace = True)\nX['part_5'].fillna(0, inplace = True)\nX['part_6'].fillna(0, inplace = True)\nX['part_7'].fillna(0, inplace = True)\nX['type_of_concept'].fillna(0, inplace = True)\nX['type_of_intention'].fillna(0, inplace = True)\nX['type_of_solving_question'].fillna(0, inplace = True)\nX['type_of_starter'].fillna(0, inplace = True)\nX['part_1_boolean'].fillna(0, inplace = True)\nX['part_2_boolean'].fillna(0, inplace = True)\nX['part_3_boolean'].fillna(0, inplace = True)\nX['part_4_boolean'].fillna(0, inplace = True)\nX['part_5_boolean'].fillna(0, inplace = True)\nX['part_6_boolean'].fillna(0, inplace = True)\nX['part_7_boolean'].fillna(0, inplace = True)\nX['type_of_concept_boolean'].fillna(0, inplace = True)\nX['type_of_intention_boolean'].fillna(0, inplace = True)\nX['type_of_solving_question_boolean'].fillna(0, inplace = True)\nX['type_of_starter_boolean'].fillna(0, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:59:25.980235Z","iopub.execute_input":"2021-06-25T15:59:25.980664Z","iopub.status.idle":"2021-06-25T15:59:26.626866Z","shell.execute_reply.started":"2021-06-25T15:59:25.980623Z","shell.execute_reply":"2021-06-25T15:59:26.625745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_val['answered_correctly_user'].fillna(0.65,  inplace=True)\nX_val['explanation_mean_user'].fillna(prior_mean_user,  inplace=True)\nX_val['quest_pct'].fillna(content_mean,  inplace=True)\n\nX_val['part'].fillna(4, inplace = True)\nX_val['avg_questions_seen'].fillna(1, inplace = True)\nX_val['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)\nX_val['prior_question_had_explanation_enc'].fillna(0, inplace = True)\n\nX_val['part_1'].fillna(0, inplace = True)\nX_val['part_2'].fillna(0, inplace = True)\nX_val['part_3'].fillna(0, inplace = True)\nX_val['part_4'].fillna(0, inplace = True)\nX_val['part_5'].fillna(0, inplace = True)\nX_val['part_6'].fillna(0, inplace = True)\nX_val['part_7'].fillna(0, inplace = True)\nX_val['type_of_concept'].fillna(0, inplace = True)\nX_val['type_of_intention'].fillna(0, inplace = True)\nX_val['type_of_solving_question'].fillna(0, inplace = True)\nX_val['type_of_starter'].fillna(0, inplace = True)\nX_val['part_1_boolean'].fillna(0, inplace = True)\nX_val['part_2_boolean'].fillna(0, inplace = True)\nX_val['part_3_boolean'].fillna(0, inplace = True)\nX_val['part_4_boolean'].fillna(0, inplace = True)\nX_val['part_5_boolean'].fillna(0, inplace = True)\nX_val['part_6_boolean'].fillna(0, inplace = True)\nX_val['part_7_boolean'].fillna(0, inplace = True)\nX_val['type_of_concept_boolean'].fillna(0, inplace = True)\nX_val['type_of_intention_boolean'].fillna(0, inplace = True)\nX_val['type_of_solving_question_boolean'].fillna(0, inplace = True)\nX_val['type_of_starter_boolean'].fillna(0, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:59:28.143722Z","iopub.execute_input":"2021-06-25T15:59:28.14415Z","iopub.status.idle":"2021-06-25T15:59:28.349477Z","shell.execute_reply.started":"2021-06-25T15:59:28.144118Z","shell.execute_reply":"2021-06-25T15:59:28.348455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modeling ##","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\nscaler = MinMaxScaler(feature_range=(0, 1))\nX = scaler.fit_transform(X)\nX_val = scaler.transform(X_val)\nX_train = X.reshape(X.shape[0], X.shape[1], 1)\nX_test = X_val.reshape(X_val.shape[0], X_val.shape[1], 1)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:59:51.817818Z","iopub.execute_input":"2021-06-25T15:59:51.818192Z","iopub.status.idle":"2021-06-25T15:59:55.260088Z","shell.execute_reply.started":"2021-06-25T15:59:51.81816Z","shell.execute_reply":"2021-06-25T15:59:55.258921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import Sequential\nfrom tensorflow.keras.layers import BatchNormalization,Dropout,Dense,Flatten,Conv1D\nfrom tensorflow.keras.optimizers import Adam\nfrom keras.metrics import BinaryAccuracy\nfrom keras import backend as K\nfrom tensorflow import keras\nfrom tensorflow.keras import layers","metadata":{"execution":{"iopub.status.busy":"2021-06-25T16:00:31.715839Z","iopub.execute_input":"2021-06-25T16:00:31.716265Z","iopub.status.idle":"2021-06-25T16:00:37.90451Z","shell.execute_reply.started":"2021-06-25T16:00:31.716234Z","shell.execute_reply":"2021-06-25T16:00:37.903476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MultiHeadSelfAttention(layers.Layer):\n    def __init__(self, embed_dim, num_heads=8):\n        super(MultiHeadSelfAttention, self).__init__()\n        self.embed_dim = embed_dim\n        self.num_heads = num_heads\n        if embed_dim % num_heads != 0:\n            raise ValueError(\n                f\"embedding dimension = {embed_dim} should be divisible by number of heads = {num_heads}\"\n            )\n        self.projection_dim = embed_dim // num_heads\n        self.query_dense = layers.Dense(embed_dim)\n        self.key_dense = layers.Dense(embed_dim)\n        self.value_dense = layers.Dense(embed_dim)\n        self.combine_heads = layers.Dense(embed_dim)\n\n    def attention(self, query, key, value):\n        score = tf.matmul(query, key, transpose_b=True)\n        dim_key = tf.cast(tf.shape(key)[-1], tf.float32)\n        scaled_score = score / tf.math.sqrt(dim_key)\n        weights = tf.nn.softmax(scaled_score, axis=-1)\n        output = tf.matmul(weights, value)\n        return output, weights\n\n    def separate_heads(self, x, batch_size):\n        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.projection_dim))\n        return tf.transpose(x, perm=[0, 2, 1, 3])\n\n    def call(self, inputs):\n        # x.shape = [batch_size, seq_len, embedding_dim]\n        batch_size = tf.shape(inputs)[0]\n        query = self.query_dense(inputs)  # (batch_size, seq_len, embed_dim)\n        key = self.key_dense(inputs)  # (batch_size, seq_len, embed_dim)\n        value = self.value_dense(inputs)  # (batch_size, seq_len, embed_dim)\n        query = self.separate_heads(\n            query, batch_size\n        )  # (batch_size, num_heads, seq_len, projection_dim)\n        key = self.separate_heads(\n            key, batch_size\n        )  # (batch_size, num_heads, seq_len, projection_dim)\n        value = self.separate_heads(\n            value, batch_size\n        )  # (batch_size, num_heads, seq_len, projection_dim)\n        attention, weights = self.attention(query, key, value)\n        attention = tf.transpose(\n            attention, perm=[0, 2, 1, 3]\n        )  # (batch_size, seq_len, num_heads, projection_dim)\n        concat_attention = tf.reshape(\n            attention, (batch_size, -1, self.embed_dim)\n        )  # (batch_size, seq_len, embed_dim)\n        output = self.combine_heads(\n            concat_attention\n        )  # (batch_size, seq_len, embed_dim)\n        return output","metadata":{"execution":{"iopub.status.busy":"2021-06-25T16:01:35.761535Z","iopub.execute_input":"2021-06-25T16:01:35.762028Z","iopub.status.idle":"2021-06-25T16:01:35.783924Z","shell.execute_reply.started":"2021-06-25T16:01:35.761993Z","shell.execute_reply":"2021-06-25T16:01:35.782375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TransformerBlock(layers.Layer):\n    def __init__(self, embed_dim, num_heads, ff_dim, rate=0.2):\n        super(TransformerBlock, self).__init__()\n        self.att = MultiHeadSelfAttention(embed_dim, num_heads)\n        self.ffn = keras.Sequential(\n            [layers.Dense(ff_dim, activation=\"relu\"), layers.Dense(embed_dim),]\n        )\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.dropout1 = layers.Dropout(rate)\n        self.dropout2 = layers.Dropout(rate)\n\n    def call(self, inputs, training):\n        attn_output = self.att(inputs)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(inputs + attn_output)\n        ffn_output = self.ffn(out1)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        return self.layernorm2(out1 + ffn_output)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T16:01:43.118457Z","iopub.execute_input":"2021-06-25T16:01:43.118867Z","iopub.status.idle":"2021-06-25T16:01:43.13146Z","shell.execute_reply.started":"2021-06-25T16:01:43.118817Z","shell.execute_reply":"2021-06-25T16:01:43.130024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TokenAndPositionEmbedding(layers.Layer):\n    def __init__(self, maxlen, vocab_size, embed_dim):\n        super(TokenAndPositionEmbedding, self).__init__()\n        self.token_emb = layers.Embedding(input_dim=vocab_size, output_dim=embed_dim)\n        self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=embed_dim)\n\n    def call(self, x):\n        maxlen = tf.shape(x)[-1]\n        positions = tf.range(start=0, limit=maxlen, delta=1)\n        positions = self.pos_emb(positions)\n        x = self.token_emb(x)\n        return x + positions","metadata":{"execution":{"iopub.status.busy":"2021-06-25T16:01:50.096479Z","iopub.execute_input":"2021-06-25T16:01:50.097003Z","iopub.status.idle":"2021-06-25T16:01:50.110305Z","shell.execute_reply.started":"2021-06-25T16:01:50.096965Z","shell.execute_reply":"2021-06-25T16:01:50.109007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"maxlen = 32 \nX_train = keras.preprocessing.sequence.pad_sequences(X_train, maxlen=maxlen)\nX_test = keras.preprocessing.sequence.pad_sequences(X_test, maxlen=maxlen)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T16:02:01.939526Z","iopub.execute_input":"2021-06-25T16:02:01.939947Z","iopub.status.idle":"2021-06-25T16:02:55.266914Z","shell.execute_reply.started":"2021-06-25T16:02:01.939913Z","shell.execute_reply":"2021-06-25T16:02:55.265756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embed_dim = 32  # Embedding size for each token\nnum_heads = 8  # Number of attention heads\nff_dim = 64  # Hidden layer size in feed forward network inside transformer\nvocab_size = 50000\ninputs = layers.Input(shape=(maxlen,))\nembedding_layer = TokenAndPositionEmbedding(maxlen, vocab_size, embed_dim)\nx = embedding_layer(inputs)\ntransformer_block = TransformerBlock(embed_dim, num_heads, ff_dim)\nx = transformer_block(x)\nx = layers.GlobalAveragePooling1D()(x)\nx = layers.Dropout(0.2)(x)\nx = layers.Dense(20, activation=\"relu\")(x)\nx = layers.Dropout(0.2)(x)\noutputs = layers.Dense(1, activation=\"sigmoid\")(x)\n\nmodel = keras.Model(inputs=inputs, outputs=outputs)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T16:02:55.269325Z","iopub.execute_input":"2021-06-25T16:02:55.269878Z","iopub.status.idle":"2021-06-25T16:02:58.458976Z","shell.execute_reply.started":"2021-06-25T16:02:55.269827Z","shell.execute_reply":"2021-06-25T16:02:58.457782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.callbacks import EarlyStopping\nes = EarlyStopping(monitor='val_auc', mode='max',patience=10)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T16:03:01.85766Z","iopub.execute_input":"2021-06-25T16:03:01.858056Z","iopub.status.idle":"2021-06-25T16:03:01.866413Z","shell.execute_reply.started":"2021-06-25T16:03:01.858022Z","shell.execute_reply":"2021-06-25T16:03:01.865294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(\"adam\", \"binary_crossentropy\", metrics=[tf.keras.metrics.AUC()])\n\nhistory = model.fit(\n    X_train, y, epochs=25,verbose=1,validation_split=0.1,batch_size=65536,callbacks=[es]\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T16:03:04.125003Z","iopub.execute_input":"2021-06-25T16:03:04.149676Z","iopub.status.idle":"2021-06-25T16:26:51.692472Z","shell.execute_reply.started":"2021-06-25T16:03:04.149604Z","shell.execute_reply":"2021-06-25T16:26:51.691523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(X_test)\ny_true = np.array(y_val)\nroc_auc_score(y_true, y_pred)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T17:11:38.288027Z","iopub.execute_input":"2021-06-25T17:11:38.288369Z","iopub.status.idle":"2021-06-25T17:11:38.319678Z","shell.execute_reply.started":"2021-06-25T17:11:38.288338Z","shell.execute_reply":"2021-06-25T17:11:38.31781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Making Predictions for New Data ##","metadata":{}},{"cell_type":"code","source":"import riiideducation\nenv = riiideducation.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2021-05-31T19:48:08.321718Z","iopub.execute_input":"2021-05-31T19:48:08.322075Z","iopub.status.idle":"2021-05-31T19:48:08.36183Z","shell.execute_reply.started":"2021-05-31T19:48:08.322041Z","shell.execute_reply":"2021-05-31T19:48:08.361098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    test_df['task_container_id'] = test_df.task_container_id.mask(test_df.task_container_id > 9999, 9999)\n    test_df = pd.merge(test_df, group3, left_on=['task_container_id'], right_index= True, how=\"left\")\n    test_df = pd.merge(test_df, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')\n    test_df = pd.merge(test_df, results_u_final, on=['user_id'],  how=\"left\")\n    test_df = pd.merge(test_df, results_u2_final, on=['user_id'],  how=\"left\")\n    \n    test_df = pd.merge(test_df, user_lecture_stats_part, on=['user_id'], how=\"left\")\n    test_df['part_1'].fillna(0, inplace = True)\n    test_df['part_2'].fillna(0, inplace = True)\n    test_df['part_3'].fillna(0, inplace = True)\n    test_df['part_4'].fillna(0, inplace = True)\n    test_df['part_5'].fillna(0, inplace = True)\n    test_df['part_6'].fillna(0, inplace = True)\n    test_df['part_7'].fillna(0, inplace = True)\n    test_df['type_of_concept'].fillna(0, inplace = True)\n    test_df['type_of_intention'].fillna(0, inplace = True)\n    test_df['type_of_solving_question'].fillna(0, inplace = True)\n    test_df['type_of_starter'].fillna(0, inplace = True)\n    test_df['part_1_boolean'].fillna(0, inplace = True)\n    test_df['part_2_boolean'].fillna(0, inplace = True)\n    test_df['part_3_boolean'].fillna(0, inplace = True)\n    test_df['part_4_boolean'].fillna(0, inplace = True)\n    test_df['part_5_boolean'].fillna(0, inplace = True)\n    test_df['part_6_boolean'].fillna(0, inplace = True)\n    test_df['part_7_boolean'].fillna(0, inplace = True)\n    test_df['type_of_concept_boolean'].fillna(0, inplace = True)\n    test_df['type_of_intention_boolean'].fillna(0, inplace = True)\n    test_df['type_of_solving_question_boolean'].fillna(0, inplace = True)\n    test_df['type_of_starter_boolean'].fillna(0, inplace = True)\n    \n    test_df['answered_correctly_user'].fillna(0.65,  inplace=True)\n    test_df['explanation_mean_user'].fillna(prior_mean_user,  inplace=True)\n    test_df['quest_pct'].fillna(content_mean,  inplace=True)\n    test_df['part'] = test_df.part - 1\n\n    test_df['part'].fillna(4, inplace = True)\n    test_df['avg_questions_seen'].fillna(1, inplace = True)\n    test_df['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)\n    test_df['prior_question_had_explanation'].fillna(False, inplace=True)\n    test_df[\"prior_question_had_explanation_enc\"] = lb_make.fit_transform(test_df[\"prior_question_had_explanation\"])\n    X = test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',\n                                                            'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',\n                                                            'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',\n                                                            'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',\n                                                            'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',\n                                                            'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']]\n    X=scaler.transform(X)\n    X = X.reshape(X.shape[0], X.shape[1], 1)\n    X = keras.preprocessing.sequence.pad_sequences(X, maxlen=maxlen)\n    test_df['answered_correctly'] =  model.predict(X)\n\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","metadata":{"execution":{"iopub.status.busy":"2021-05-31T19:48:13.339326Z","iopub.execute_input":"2021-05-31T19:48:13.339652Z","iopub.status.idle":"2021-05-31T19:48:14.297016Z","shell.execute_reply.started":"2021-05-31T19:48:13.339621Z","shell.execute_reply":"2021-05-31T19:48:14.296236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}