{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**This notebook lb is 0.772,valid 0.770,training on 12M data.\nI meet a question: after adding incorrect count group by user,improve valid to 0.785,but lb down to 0.769. anyone can explain this? is it overfitting?need more training data?**\n\nThe code is as follows：\ncum = train_df.groupby('user_id')[target].agg(['cumsum', 'cumcount'])\ncum['cumcount']=cum['cumcount']+1\ntrain_df['user_correctness'] = cum['cumsum'] / cum['cumcount']\ntrain_df['user_correct_count'] = cum['cumsum']\ntrain_df['user_incorrect_count'] = cum['cumcount']-cum['cumsum']","metadata":{}},{"cell_type":"markdown","source":"## 1. [Preprocess](#Preprocess)\n## 2. [FeatureEngineering](#FeatureEngineering)\n## 3. [Train](#Train)\n## 4. [Inference](#Inference)","metadata":{}},{"cell_type":"code","source":"!pip install ../input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-22T05:56:12.537487Z","iopub.execute_input":"2022-08-22T05:56:12.537919Z","iopub.status.idle":"2022-08-22T05:56:40.268114Z","shell.execute_reply.started":"2022-08-22T05:56:12.537877Z","shell.execute_reply":"2022-08-22T05:56:40.267165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom collections import defaultdict\nimport datatable as dt\nimport lightgbm as lgb\nfrom matplotlib import pyplot as plt\nimport riiideducation\nimport random\nfrom sklearn.metrics import roc_auc_score\nimport gc\n\n_ = np.seterr(divide='ignore', invalid='ignore')","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2022-08-22T05:56:40.270321Z","iopub.execute_input":"2022-08-22T05:56:40.270701Z","iopub.status.idle":"2022-08-22T05:56:41.136046Z","shell.execute_reply.started":"2022-08-22T05:56:40.270652Z","shell.execute_reply":"2022-08-22T05:56:41.135002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='Preprocess'></a>\n# Preprocess","metadata":{}},{"cell_type":"code","source":"data_types_dict = {\n    'timestamp': 'int64',\n    'user_id': 'int32', \n    'content_id': 'int16', \n    'content_type_id':'int8', \n    'task_container_id': 'int16',\n    #'user_answer': 'int8',\n    'answered_correctly': 'int8', \n    'prior_question_elapsed_time': 'float32', \n    'prior_question_had_explanation': 'bool'\n}\ntarget = 'answered_correctly'","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:56:41.139308Z","iopub.execute_input":"2022-08-22T05:56:41.139603Z","iopub.status.idle":"2022-08-22T05:56:41.143691Z","shell.execute_reply.started":"2022-08-22T05:56:41.139573Z","shell.execute_reply":"2022-08-22T05:56:41.143051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = dt.fread('../input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:56:41.144689Z","iopub.execute_input":"2022-08-22T05:56:41.144930Z","iopub.status.idle":"2022-08-22T05:58:10.666088Z","shell.execute_reply.started":"2022-08-22T05:56:41.144904Z","shell.execute_reply":"2022-08-22T05:58:10.664485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#reading in lecture df\nlectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:58:10.670294Z","iopub.execute_input":"2022-08-22T05:58:10.670696Z","iopub.status.idle":"2022-08-22T05:58:10.687959Z","shell.execute_reply.started":"2022-08-22T05:58:10.670647Z","shell.execute_reply":"2022-08-22T05:58:10.687104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')\n\nlectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])\n\npart_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]\n\ntypes_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:58:10.690241Z","iopub.execute_input":"2022-08-22T05:58:10.690501Z","iopub.status.idle":"2022-08-22T05:58:10.719093Z","shell.execute_reply.started":"2022-08-22T05:58:10.690473Z","shell.execute_reply":"2022-08-22T05:58:10.718220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_lectures = train_df[train_df.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left')","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:58:10.720117Z","iopub.execute_input":"2022-08-22T05:58:10.720585Z","iopub.status.idle":"2022-08-22T05:58:11.621678Z","shell.execute_reply.started":"2022-08-22T05:58:10.720521Z","shell.execute_reply":"2022-08-22T05:58:11.620978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_lecture_stats_part = train_lectures.groupby('user_id',as_index = False)[part_lectures_columns + types_of_lectures_columns].sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:58:11.622745Z","iopub.execute_input":"2022-08-22T05:58:11.623081Z","iopub.status.idle":"2022-08-22T05:58:12.080029Z","shell.execute_reply.started":"2022-08-22T05:58:11.623048Z","shell.execute_reply":"2022-08-22T05:58:12.079332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='FeatureEngineering'></a>\n# FeatureEngineering","metadata":{}},{"cell_type":"code","source":"lecturedata_types_dict = {   \n    'user_id': 'int32', \n    'part_1': 'int8',\n    'part_2': 'int8',\n    'part_3': 'int8',\n    'part_4': 'int8',\n    'part_5': 'int8',\n    'part_6': 'int8',\n    'part_7': 'int8',\n    'type_of_concept': 'int8',\n    'type_of_intention': 'int8',\n    'type_of_solving_question': 'int8',\n    'type_of_starter': 'int8'\n}\nuser_lecture_stats_part = user_lecture_stats_part.astype(lecturedata_types_dict)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:58:12.081234Z","iopub.execute_input":"2022-08-22T05:58:12.081700Z","iopub.status.idle":"2022-08-22T05:58:12.093200Z","shell.execute_reply.started":"2022-08-22T05:58:12.081664Z","shell.execute_reply":"2022-08-22T05:58:12.092163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for column in user_lecture_stats_part.columns:\n    #bool_column = column + '_boolean'\n    if(column !='user_id'):\n        user_lecture_stats_part[column] = (user_lecture_stats_part[column] > 0).astype('int8')","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:58:12.095229Z","iopub.execute_input":"2022-08-22T05:58:12.095869Z","iopub.status.idle":"2022-08-22T05:58:12.110575Z","shell.execute_reply.started":"2022-08-22T05:58:12.095831Z","shell.execute_reply":"2022-08-22T05:58:12.109596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_lectures[train_lectures.user_id==5382]","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:58:12.111866Z","iopub.execute_input":"2022-08-22T05:58:12.112273Z","iopub.status.idle":"2022-08-22T05:58:12.141070Z","shell.execute_reply.started":"2022-08-22T05:58:12.112239Z","shell.execute_reply":"2022-08-22T05:58:12.140255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_lecture_stats_part[user_lecture_stats_part.user_id==5382]","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:58:12.142260Z","iopub.execute_input":"2022-08-22T05:58:12.142595Z","iopub.status.idle":"2022-08-22T05:58:12.155814Z","shell.execute_reply.started":"2022-08-22T05:58:12.142537Z","shell.execute_reply":"2022-08-22T05:58:12.154849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_lecture_stats_part.tail()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:58:12.157180Z","iopub.execute_input":"2022-08-22T05:58:12.157679Z","iopub.status.idle":"2022-08-22T05:58:12.171740Z","shell.execute_reply.started":"2022-08-22T05:58:12.157629Z","shell.execute_reply":"2022-08-22T05:58:12.170832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_lecture_stats_part.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:58:12.172941Z","iopub.execute_input":"2022-08-22T05:58:12.173266Z","iopub.status.idle":"2022-08-22T05:58:12.185969Z","shell.execute_reply.started":"2022-08-22T05:58:12.173233Z","shell.execute_reply":"2022-08-22T05:58:12.185266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#clearing memory\ndel(train_lectures)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:58:12.187252Z","iopub.execute_input":"2022-08-22T05:58:12.187794Z","iopub.status.idle":"2022-08-22T05:58:12.199954Z","shell.execute_reply.started":"2022-08-22T05:58:12.187754Z","shell.execute_reply":"2022-08-22T05:58:12.198966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cum = train_df.groupby('user_id')['content_type_id'].agg(['cumsum', 'cumcount'])\ntrain_df['user_lecture_cumsum'] = cum['cumsum'] \ntrain_df['user_lecture_lv'] = cum['cumsum'] / cum['cumcount']\n\ntrain_df.user_lecture_lv=train_df.user_lecture_lv.astype('float16')\ntrain_df.user_lecture_cumsum=train_df.user_lecture_cumsum.astype('int8')\nuser_lecture_agg = train_df.groupby('user_id')['content_type_id'].agg(['sum', 'count'])","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:58:12.201537Z","iopub.execute_input":"2022-08-22T05:58:12.201890Z","iopub.status.idle":"2022-08-22T05:58:26.097128Z","shell.execute_reply.started":"2022-08-22T05:58:12.201855Z","shell.execute_reply":"2022-08-22T05:58:26.096197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['prior_question_had_explanation'].fillna(False, inplace=True)\ntrain_df = train_df.astype(data_types_dict)\ntrain_df = train_df[train_df[target] != -1].reset_index(drop=True)\nprior_question_elapsed_time_mean=train_df['prior_question_elapsed_time'].mean()\ntrain_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:58:26.098459Z","iopub.execute_input":"2022-08-22T05:58:26.098828Z","iopub.status.idle":"2022-08-22T05:58:42.411533Z","shell.execute_reply.started":"2022-08-22T05:58:26.098794Z","shell.execute_reply":"2022-08-22T05:58:42.410703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_timestamp_u = train_df[['user_id','timestamp']].groupby(['user_id']).agg(['max']).reset_index()\nmax_timestamp_u.columns = ['user_id', 'max_time_stamp']","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:58:42.412757Z","iopub.execute_input":"2022-08-22T05:58:42.413073Z","iopub.status.idle":"2022-08-22T05:58:45.885198Z","shell.execute_reply.started":"2022-08-22T05:58:42.413036Z","shell.execute_reply":"2022-08-22T05:58:45.884289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain_df['lagtime'] = train_df.groupby('user_id')['timestamp'].shift()\ntrain_df['lagtime']=train_df['timestamp']-train_df['lagtime']\ntrain_df['lagtime'].fillna(0, inplace=True)\ntrain_df.lagtime=train_df.lagtime.astype('int32')","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:58:45.886279Z","iopub.execute_input":"2022-08-22T05:58:45.886499Z","iopub.status.idle":"2022-08-22T05:58:49.636675Z","shell.execute_reply.started":"2022-08-22T05:58:45.886473Z","shell.execute_reply":"2022-08-22T05:58:49.635941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lagtime_agg = train_df.groupby('user_id')['lagtime'].agg(['mean'])\ntrain_df['lagtime_mean'] = train_df['user_id'].map(lagtime_agg['mean'])\ntrain_df.lagtime_mean=train_df.lagtime_mean.astype('int32')","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:58:49.637796Z","iopub.execute_input":"2022-08-22T05:58:49.638188Z","iopub.status.idle":"2022-08-22T05:58:53.231439Z","shell.execute_reply.started":"2022-08-22T05:58:49.638158Z","shell.execute_reply":"2022-08-22T05:58:53.230324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_prior_question_elapsed_time = train_df[['user_id','prior_question_elapsed_time']].groupby(['user_id']).tail(1)\nuser_prior_question_elapsed_time.columns = ['user_id', 'prior_question_elapsed_time']","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:58:53.232689Z","iopub.execute_input":"2022-08-22T05:58:53.232955Z","iopub.status.idle":"2022-08-22T05:58:59.117145Z","shell.execute_reply.started":"2022-08-22T05:58:53.232927Z","shell.execute_reply":"2022-08-22T05:58:59.115955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain_df['delta_prior_question_elapsed_time'] = train_df.groupby('user_id')['prior_question_elapsed_time'].shift()\ntrain_df['delta_prior_question_elapsed_time']=train_df['prior_question_elapsed_time']-train_df['delta_prior_question_elapsed_time']\ntrain_df['delta_prior_question_elapsed_time'].fillna(0, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:58:59.118851Z","iopub.execute_input":"2022-08-22T05:58:59.119265Z","iopub.status.idle":"2022-08-22T05:59:02.312103Z","shell.execute_reply.started":"2022-08-22T05:58:59.119224Z","shell.execute_reply":"2022-08-22T05:59:02.311094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.delta_prior_question_elapsed_time=train_df.delta_prior_question_elapsed_time.astype('int32')","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:59:02.313402Z","iopub.execute_input":"2022-08-22T05:59:02.313741Z","iopub.status.idle":"2022-08-22T05:59:02.501141Z","shell.execute_reply.started":"2022-08-22T05:59:02.313709Z","shell.execute_reply":"2022-08-22T05:59:02.500353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['timestamp']=train_df['timestamp']/(1000*3600)\ntrain_df.timestamp=train_df.timestamp.astype('int16')","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:59:02.502702Z","iopub.execute_input":"2022-08-22T05:59:02.503007Z","iopub.status.idle":"2022-08-22T05:59:02.954732Z","shell.execute_reply.started":"2022-08-22T05:59:02.502979Z","shell.execute_reply":"2022-08-22T05:59:02.953653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain_df['lag'] = train_df.groupby('user_id')[target].shift()\n\ncum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])\ntrain_df['user_correctness'] = cum['cumsum'] / cum['cumcount']\ntrain_df['user_correct_cumsum'] = cum['cumsum']\ntrain_df['user_correct_cumcount'] = cum['cumcount']\ntrain_df.drop(columns=['lag'], inplace=True)\n\n# train_df['user_correctness'].fillna(1, inplace=True)\ntrain_df['user_correct_cumsum'].fillna(0, inplace=True)\n#train_df['user_correct_cumcount'].fillna(0, inplace=True)\ntrain_df.user_correctness=train_df.user_correctness.astype('float16')\ntrain_df.user_correct_cumcount=train_df.user_correct_cumcount.astype('int16')\ntrain_df.user_correct_cumsum=train_df.user_correct_cumsum.astype('int16')","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:59:02.955856Z","iopub.execute_input":"2022-08-22T05:59:02.956100Z","iopub.status.idle":"2022-08-22T05:59:21.566753Z","shell.execute_reply.started":"2022-08-22T05:59:02.956076Z","shell.execute_reply":"2022-08-22T05:59:21.565782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.prior_question_had_explanation=train_df.prior_question_had_explanation.astype('int8')\n\ntrain_df['lag'] = train_df.groupby('user_id')['prior_question_had_explanation'].shift()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:59:21.567837Z","iopub.execute_input":"2022-08-22T05:59:21.568066Z","iopub.status.idle":"2022-08-22T05:59:24.764898Z","shell.execute_reply.started":"2022-08-22T05:59:21.568042Z","shell.execute_reply":"2022-08-22T05:59:24.763904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ncum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])\ntrain_df['explanation_mean'] = cum['cumsum'] / cum['cumcount']\ntrain_df['explanation_cumsum'] = cum['cumsum'] \ntrain_df.drop(columns=['lag'], inplace=True)\n\ntrain_df['explanation_mean'].fillna(0, inplace=True)\ntrain_df['explanation_cumsum'].fillna(0, inplace=True)\ntrain_df.explanation_mean=train_df.explanation_mean.astype('float16')\ntrain_df.explanation_cumsum=train_df.explanation_cumsum.astype('int16')","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:59:24.766451Z","iopub.execute_input":"2022-08-22T05:59:24.766762Z","iopub.status.idle":"2022-08-22T05:59:36.005830Z","shell.execute_reply.started":"2022-08-22T05:59:24.766732Z","shell.execute_reply":"2022-08-22T05:59:36.004452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del cum\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:59:36.007055Z","iopub.execute_input":"2022-08-22T05:59:36.007350Z","iopub.status.idle":"2022-08-22T05:59:36.102827Z","shell.execute_reply.started":"2022-08-22T05:59:36.007321Z","shell.execute_reply":"2022-08-22T05:59:36.101623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain_df[\"attempt_no\"] = 1\ntrain_df.attempt_no=train_df.attempt_no.astype('int8')\ntrain_df[\"attempt_no\"] = train_df[[\"user_id\",\"content_id\",'attempt_no']].groupby([\"user_id\",\"content_id\"])[\"attempt_no\"].cumsum()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T05:59:36.104037Z","iopub.execute_input":"2022-08-22T05:59:36.104327Z","iopub.status.idle":"2022-08-22T06:00:11.149104Z","shell.execute_reply.started":"2022-08-22T05:59:36.104301Z","shell.execute_reply":"2022-08-22T06:00:11.147931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:00:11.153055Z","iopub.execute_input":"2022-08-22T06:00:11.154251Z","iopub.status.idle":"2022-08-22T06:00:11.174132Z","shell.execute_reply.started":"2022-08-22T06:00:11.154203Z","shell.execute_reply":"2022-08-22T06:00:11.173198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:00:11.175459Z","iopub.execute_input":"2022-08-22T06:00:11.175925Z","iopub.status.idle":"2022-08-22T06:00:11.189236Z","shell.execute_reply.started":"2022-08-22T06:00:11.175891Z","shell.execute_reply":"2022-08-22T06:00:11.188630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"explanation_agg = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['sum', 'count'])\nexplanation_agg=explanation_agg.astype('int16')\n#train_df.drop(columns=['prior_question_had_explanation'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:00:11.194100Z","iopub.execute_input":"2022-08-22T06:00:11.194528Z","iopub.status.idle":"2022-08-22T06:00:14.548766Z","shell.execute_reply.started":"2022-08-22T06:00:11.194496Z","shell.execute_reply":"2022-08-22T06:00:14.547639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_agg = train_df.groupby('user_id')[target].agg(['sum', 'count'])\ncontent_agg = train_df.groupby('content_id')[target].agg(['sum', 'count','var'])\ntask_container_agg = train_df.groupby('task_container_id')[target].agg(['sum', 'count','var'])\n\n#prior_question_elapsed_time_agg = train_df.groupby('user_id')['prior_question_elapsed_time'].agg(['sum', 'count'])","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:00:14.550105Z","iopub.execute_input":"2022-08-22T06:00:14.550343Z","iopub.status.idle":"2022-08-22T06:00:29.365398Z","shell.execute_reply.started":"2022-08-22T06:00:14.550320Z","shell.execute_reply":"2022-08-22T06:00:29.364252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_agg=user_agg.astype('int16')\ncontent_agg=content_agg.astype('float32')\ntask_container_agg=task_container_agg.astype('float32')","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:00:29.366733Z","iopub.execute_input":"2022-08-22T06:00:29.367023Z","iopub.status.idle":"2022-08-22T06:00:29.373831Z","shell.execute_reply.started":"2022-08-22T06:00:29.366994Z","shell.execute_reply":"2022-08-22T06:00:29.372519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"attempt_no_agg=train_df.groupby([\"user_id\",\"content_id\"])[\"attempt_no\"].agg(['sum'])\nattempt_no_agg=attempt_no_agg.astype('int8')\n#attempt_series = train_df[['user_id', 'content_id','attempt_no']].groupby(['user_id','content_id'])['attempt_no'].max()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:00:29.375426Z","iopub.execute_input":"2022-08-22T06:00:29.375928Z","iopub.status.idle":"2022-08-22T06:01:14.325786Z","shell.execute_reply.started":"2022-08-22T06:00:29.375888Z","shell.execute_reply":"2022-08-22T06:01:14.324860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['content_count'] = train_df['content_id'].map(content_agg['count']).astype('int32')\ntrain_df['content_sum'] = train_df['content_id'].map(content_agg['sum']).astype('int32')\ntrain_df['content_correctness'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'])\ntrain_df.content_correctness=train_df.content_correctness.astype('float16')\ntrain_df['task_container_sum'] = train_df['task_container_id'].map(task_container_agg['sum']).astype('int32')\ntrain_df['task_container_std'] = train_df['task_container_id'].map(task_container_agg['var']).astype('float16')\ntrain_df['task_container_correctness'] = train_df['task_container_id'].map(task_container_agg['sum'] / task_container_agg['count'])\ntrain_df.task_container_correctness=train_df.task_container_correctness.astype('float16')","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:01:14.328449Z","iopub.execute_input":"2022-08-22T06:01:14.329031Z","iopub.status.idle":"2022-08-22T06:01:20.763052Z","shell.execute_reply.started":"2022-08-22T06:01:14.328992Z","shell.execute_reply":"2022-08-22T06:01:20.762051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"questions_df = pd.read_csv(\n    '../input/riiid-test-answer-prediction/questions.csv', \n    usecols=[0, 1,3,4],\n    dtype={'question_id': 'int16','bundle_id': 'int16', 'part': 'int8','tags': 'str'}\n)\nquestions_df['part_bundle_id']=questions_df['part']*100000+questions_df['bundle_id']\nquestions_df.part_bundle_id=questions_df.part_bundle_id.astype('int32')\ntag = questions_df[\"tags\"].str.split(\" \", n = 10, expand = True)\ntag.columns = ['tags1','tags2','tags3','tags4','tags5','tags6']\n#\n\ntag.fillna(0, inplace=True)\ntag = tag.astype('int16')\nquestions_df =  pd.concat([questions_df,tag],axis=1).drop(['tags'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:01:20.765162Z","iopub.execute_input":"2022-08-22T06:01:20.765700Z","iopub.status.idle":"2022-08-22T06:01:20.838309Z","shell.execute_reply.started":"2022-08-22T06:01:20.765656Z","shell.execute_reply":"2022-08-22T06:01:20.837454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"questions_df.rename(columns={'question_id':'content_id'}, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:01:20.839235Z","iopub.execute_input":"2022-08-22T06:01:20.839475Z","iopub.status.idle":"2022-08-22T06:01:20.843973Z","shell.execute_reply.started":"2022-08-22T06:01:20.839448Z","shell.execute_reply":"2022-08-22T06:01:20.842967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"questions_df['content_correctness'] = questions_df['content_id'].map(content_agg['sum'] / content_agg['count'])\nquestions_df.content_correctness=questions_df.content_correctness.astype('float16')\nquestions_df['content_correctness_std'] = questions_df['content_id'].map(content_agg['var'])\nquestions_df.content_correctness_std=questions_df.content_correctness_std.astype('float16')","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:01:20.845038Z","iopub.execute_input":"2022-08-22T06:01:20.845263Z","iopub.status.idle":"2022-08-22T06:01:20.860602Z","shell.execute_reply.started":"2022-08-22T06:01:20.845241Z","shell.execute_reply":"2022-08-22T06:01:20.859468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"part_agg = questions_df.groupby('part')['content_correctness'].agg(['mean', 'var'])\nquestions_df['part_correctness_mean'] = questions_df['part'].map(part_agg['mean'])\nquestions_df['part_correctness_std'] = questions_df['part'].map(part_agg['var'])\nquestions_df.part_correctness_mean=questions_df.part_correctness_mean.astype('float16')\nquestions_df.part_correctness_std=questions_df.part_correctness_std.astype('float16')","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:01:20.862233Z","iopub.execute_input":"2022-08-22T06:01:20.862880Z","iopub.status.idle":"2022-08-22T06:01:20.876162Z","shell.execute_reply.started":"2022-08-22T06:01:20.862812Z","shell.execute_reply":"2022-08-22T06:01:20.874884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bundle_agg = questions_df.groupby('bundle_id')['content_correctness'].agg(['mean'])\nquestions_df['bundle_correctness'] = questions_df['bundle_id'].map(bundle_agg['mean'])\nquestions_df.bundle_correctness=questions_df.bundle_correctness.astype('float16')","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:01:20.877513Z","iopub.execute_input":"2022-08-22T06:01:20.878186Z","iopub.status.idle":"2022-08-22T06:01:20.891095Z","shell.execute_reply.started":"2022-08-22T06:01:20.878145Z","shell.execute_reply":"2022-08-22T06:01:20.889929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tags1_agg = questions_df.groupby('tags1')['content_correctness'].agg(['mean', 'var'])\nquestions_df['tags1_correctness_mean'] = questions_df['tags1'].map(tags1_agg['mean'])\nquestions_df['tags1_correctness_std'] = questions_df['tags1'].map(tags1_agg['var'])\nquestions_df.tags1_correctness_mean=questions_df.tags1_correctness_mean.astype('float16')\nquestions_df.tags1_correctness_std=questions_df.tags1_correctness_std.astype('float16')","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:01:20.892404Z","iopub.execute_input":"2022-08-22T06:01:20.892811Z","iopub.status.idle":"2022-08-22T06:01:20.906963Z","shell.execute_reply.started":"2022-08-22T06:01:20.892772Z","shell.execute_reply":"2022-08-22T06:01:20.905616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"questions_df.drop(columns=['content_correctness'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:01:20.908421Z","iopub.execute_input":"2022-08-22T06:01:20.908746Z","iopub.status.idle":"2022-08-22T06:01:20.914535Z","shell.execute_reply.started":"2022-08-22T06:01:20.908717Z","shell.execute_reply":"2022-08-22T06:01:20.913860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"questions_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:01:20.915617Z","iopub.execute_input":"2022-08-22T06:01:20.916000Z","iopub.status.idle":"2022-08-22T06:01:20.926768Z","shell.execute_reply.started":"2022-08-22T06:01:20.915972Z","shell.execute_reply":"2022-08-22T06:01:20.925993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del bundle_agg\ndel part_agg\ndel tags1_agg\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:01:20.928415Z","iopub.execute_input":"2022-08-22T06:01:20.929103Z","iopub.status.idle":"2022-08-22T06:01:21.017377Z","shell.execute_reply.started":"2022-08-22T06:01:20.929063Z","shell.execute_reply":"2022-08-22T06:01:21.016648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:01:21.037353Z","iopub.execute_input":"2022-08-22T06:01:21.037648Z","iopub.status.idle":"2022-08-22T06:01:21.049102Z","shell.execute_reply.started":"2022-08-22T06:01:21.037611Z","shell.execute_reply":"2022-08-22T06:01:21.048433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['user_correctness'].fillna( 1, inplace=True)\ntrain_df['attempt_no'].fillna(1, inplace=True)\n#\ntrain_df.fillna(0, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:01:21.050255Z","iopub.execute_input":"2022-08-22T06:01:21.050624Z","iopub.status.idle":"2022-08-22T06:01:23.097032Z","shell.execute_reply.started":"2022-08-22T06:01:21.050590Z","shell.execute_reply":"2022-08-22T06:01:23.096206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:01:23.098019Z","iopub.execute_input":"2022-08-22T06:01:23.098266Z","iopub.status.idle":"2022-08-22T06:01:23.126600Z","shell.execute_reply.started":"2022-08-22T06:01:23.098242Z","shell.execute_reply":"2022-08-22T06:01:23.125387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:01:23.136947Z","iopub.execute_input":"2022-08-22T06:01:23.137265Z","iopub.status.idle":"2022-08-22T06:01:23.151531Z","shell.execute_reply.started":"2022-08-22T06:01:23.137239Z","shell.execute_reply":"2022-08-22T06:01:23.150040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='Train'></a>\n# Train","metadata":{}},{"cell_type":"code","source":"features = [\n    #'user_id',\n    'timestamp',\n    'lagtime',\n    'lagtime_mean',\n    'content_id',\n    'task_container_id',\n    'user_lecture_cumsum',\n    'user_lecture_lv',\n    'prior_question_elapsed_time',\n    'delta_prior_question_elapsed_time',\n    'user_correctness',\n    'user_correct_cumcount',\n    'user_correct_cumsum',\n    'content_correctness',\n    'content_correctness_std',\n    'content_count',\n    'content_sum',\n    'task_container_correctness',\n    'task_container_std',\n    'task_container_sum',\n    'bundle_correctness',\n    'attempt_no',\n    'part',\n    'part_correctness_mean',\n    'part_correctness_std',\n    'tags1',\n    'tags1_correctness_mean',\n    'tags1_correctness_std',\n    'tags2',\n    'tags3',\n    'tags4',\n    'tags5',\n    'tags6',\n    'bundle_id',\n    'part_bundle_id',\n    'explanation_mean', \n    'explanation_cumsum',\n    'prior_question_had_explanation',\n#     'part_1',\n#     'part_2',\n#     'part_3',\n#     'part_4',\n#     'part_5',\n#     'part_6',\n#     'part_7',\n#     'type_of_concept',\n#     'type_of_intention',\n#     'type_of_solving_question',\n#     'type_of_starter'\n]\ncategorical_columns= [\n    #'user_id',\n    'content_id',\n    'task_container_id',\n    'part',        \n    'tags1',\n    'tags2',\n    'tags3',\n    'tags4',\n    'tags5',\n    'tags6',\n    'bundle_id',\n    'part_bundle_id',\n    'prior_question_had_explanation',\n#     'part_1',\n#     'part_2',\n#     'part_3',\n#     'part_4',\n#     'part_5',\n#     'part_6',\n#     'part_7',\n#     'type_of_concept',\n#     'type_of_intention',\n#     'type_of_solving_question',\n#     'type_of_starter'\n]\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:01:23.152839Z","iopub.execute_input":"2022-08-22T06:01:23.153137Z","iopub.status.idle":"2022-08-22T06:01:23.160483Z","shell.execute_reply.started":"2022-08-22T06:01:23.153109Z","shell.execute_reply":"2022-08-22T06:01:23.159757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"flag_lgbm=True\nclfs = list()\nparams = {\n'num_leaves': 350,\n'max_bin':700,\n'min_child_weight': 0.03454472573214212,\n'feature_fraction': 0.58,\n'bagging_fraction': 0.58,\n#'min_data_in_leaf': 106,\n'objective': 'binary',\n'max_depth': -1,\n'learning_rate': 0.05,\n\"boosting_type\": \"gbdt\",\n\"bagging_seed\": 11,\n\"metric\": 'auc',\n\"verbosity\": -1,\n'reg_alpha': 0.3899927210061127,\n'reg_lambda': 0.6485237330340494,\n'random_state': 47\n}\ntrains=list()\nvalids=list()\nnum=1\nfor i in range(0,num):\n  \n    #train_df=train_df.reset_index(drop=True)\n    train_df_clf=train_df.sample(n=1200*10000)\n    print('sample end')\n    #train_df.drop(train_df_clf.index, inplace=True)\n    #print('train_df drop end')\n    \n   \n    del train_df\n    \n    users=train_df_clf['user_id'].drop_duplicates()#去重\n    \n    users=users.sample(frac=0.025)\n    users_df=pd.DataFrame()\n    users_df['user_id']=users.values\n  \n  \n    valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)\n    del users_df\n    del users\n    gc.collect()\n    #\n    train_df_clf.drop(valid_df_newuser.index, inplace=True)\n   \n    #-----------\n    #train_df_clf=train_df_clf.sample(frac=0.2)\n    #train_df_clf.drop(valid_df_newuser.index, inplace=True)\n    train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)#\n    valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)#\n    \n#     train_df_clf = pd.merge(train_df_clf, user_lecture_stats_part, on='user_id', how=\"left\",right_index=True)\n#     valid_df_newuser = pd.merge(valid_df_newuser, user_lecture_stats_part, on='user_id', how=\"left\",right_index=True)\n\n    valid_df=train_df_clf.sample(frac=0.15)\n    train_df_clf.drop(valid_df.index, inplace=True)\n   \n    valid_df = valid_df.append(valid_df_newuser)\n    del valid_df_newuser\n    gc.collect()\n    #\n\n    trains.append(train_df_clf)\n    valids.append(valid_df)\n    print('valid_df length：',len(valid_df))\n    #train_df=train_df.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:01:23.161671Z","iopub.execute_input":"2022-08-22T06:01:23.162031Z","iopub.status.idle":"2022-08-22T06:01:46.992974Z","shell.execute_reply.started":"2022-08-22T06:01:23.162005Z","shell.execute_reply":"2022-08-22T06:01:46.992090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#del train_df\ndel train_df_clf\ndel valid_df\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:01:46.994017Z","iopub.execute_input":"2022-08-22T06:01:46.994256Z","iopub.status.idle":"2022-08-22T06:01:47.078999Z","shell.execute_reply.started":"2022-08-22T06:01:46.994230Z","shell.execute_reply":"2022-08-22T06:01:47.078039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfor i in range(0,num):\n\n#     \n    tr_data = lgb.Dataset(trains[i][features], label=trains[i][target])\n    va_data = lgb.Dataset(valids[i][features], label=valids[i][target])\n    \n#     del train_df_clf\n#     del valid_df\n#     gc.collect()\n    del trains\n    del valids\n    gc.collect()\n\n    model = lgb.train(\n        params, \n        tr_data,\n#         train_df[features],\n#         train_df[target],\n        num_boost_round=5000,\n        #valid_sets=[(train_df[features],train_df[target]), (valid_df[features],valid_df[target])], \n        valid_sets=[tr_data, va_data],\n        early_stopping_rounds=50,\n        feature_name=features,\n        categorical_feature=categorical_columns,\n        verbose_eval=50\n    )\n    clfs.append(model)\n    #print('auc:', roc_auc_score(valid_df[target], model.predict(valid_df[features])))\n    #model.save_model(f'model.txt')\n    lgb.plot_importance(model, importance_type='gain')\n    plt.show()\n\n    del tr_data\n    del va_data\n    gc.collect()\n#    \n# del trains\n# del valids\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:01:47.080330Z","iopub.execute_input":"2022-08-22T06:01:47.080773Z","iopub.status.idle":"2022-08-22T06:18:27.063735Z","shell.execute_reply.started":"2022-08-22T06:01:47.080736Z","shell.execute_reply":"2022-08-22T06:18:27.062846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='Inference'></a>\n# Inference","metadata":{}},{"cell_type":"code","source":"user_sum_dict = user_agg['sum'].astype('int16').to_dict(defaultdict(int))\nuser_count_dict = user_agg['count'].astype('int16').to_dict(defaultdict(int))\ncontent_sum_dict = content_agg['sum'].astype('int32').to_dict(defaultdict(int))\ncontent_count_dict = content_agg['count'].astype('int32').to_dict(defaultdict(int))\n\ndel user_agg\ndel content_agg\ngc.collect()\n\ntask_container_sum_dict = task_container_agg['sum'].astype('int32').to_dict(defaultdict(int))\ntask_container_count_dict = task_container_agg['count'].astype('int32').to_dict(defaultdict(int))\ntask_container_std_dict = task_container_agg['var'].astype('float16').to_dict(defaultdict(int))\n\nexplanation_sum_dict = explanation_agg['sum'].astype('int16').to_dict(defaultdict(int))\nexplanation_count_dict = explanation_agg['count'].astype('int16').to_dict(defaultdict(int))\ndel task_container_agg\ndel explanation_agg\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:18:27.094537Z","iopub.execute_input":"2022-08-22T06:18:27.094804Z","iopub.status.idle":"2022-08-22T06:18:27.868837Z","shell.execute_reply.started":"2022-08-22T06:18:27.094778Z","shell.execute_reply":"2022-08-22T06:18:27.868209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_lecture_sum_dict = user_lecture_agg['sum'].astype('int16').to_dict(defaultdict(int))\nuser_lecture_count_dict = user_lecture_agg['count'].astype('int16').to_dict(defaultdict(int))\n\nlagtime_mean_dict = lagtime_agg['mean'].astype('int32').to_dict(defaultdict(int))\n#del prior_question_elapsed_time_agg\ndel user_lecture_agg\ndel lagtime_agg\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:18:27.870120Z","iopub.execute_input":"2022-08-22T06:18:27.870466Z","iopub.status.idle":"2022-08-22T06:18:28.382473Z","shell.execute_reply.started":"2022-08-22T06:18:27.870430Z","shell.execute_reply":"2022-08-22T06:18:28.381508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"attempt_no_agg=attempt_no_agg[attempt_no_agg['sum'] >1]\nattempt_no_sum_dict = attempt_no_agg['sum'].to_dict(defaultdict(int))\n\ndel attempt_no_agg\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:18:28.383764Z","iopub.execute_input":"2022-08-22T06:18:28.384193Z","iopub.status.idle":"2022-08-22T06:18:39.963689Z","shell.execute_reply.started":"2022-08-22T06:18:28.384158Z","shell.execute_reply":"2022-08-22T06:18:39.962750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_timestamp_u_dict=max_timestamp_u.set_index('user_id').to_dict()\nuser_prior_question_elapsed_time_dict=user_prior_question_elapsed_time.set_index('user_id').to_dict()\n#del question_elapsed_time_agg\ndel max_timestamp_u\ndel user_prior_question_elapsed_time\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:18:39.964938Z","iopub.execute_input":"2022-08-22T06:18:39.965205Z","iopub.status.idle":"2022-08-22T06:18:40.531106Z","shell.execute_reply.started":"2022-08-22T06:18:39.965176Z","shell.execute_reply":"2022-08-22T06:18:40.530504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(max_timestamp_u_dict['max_time_stamp'])","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:18:40.532267Z","iopub.execute_input":"2022-08-22T06:18:40.532498Z","iopub.status.idle":"2022-08-22T06:18:40.536901Z","shell.execute_reply.started":"2022-08-22T06:18:40.532474Z","shell.execute_reply":"2022-08-22T06:18:40.536407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_max_attempt(user_id,content_id):\n    k = (user_id,content_id)\n\n    if k in attempt_no_sum_dict.keys():\n        attempt_no_sum_dict[k]+=1\n        return attempt_no_sum_dict[k]\n\n    attempt_no_sum_dict[k] = 1\n    return attempt_no_sum_dict[k]","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:18:40.537657Z","iopub.execute_input":"2022-08-22T06:18:40.537950Z","iopub.status.idle":"2022-08-22T06:18:40.549233Z","shell.execute_reply.started":"2022-08-22T06:18:40.537927Z","shell.execute_reply":"2022-08-22T06:18:40.548323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = lgb.Booster(model_file='../input/riiid-lgbm-starter/model.txt')\nenv = riiideducation.make_env()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:18:40.550330Z","iopub.execute_input":"2022-08-22T06:18:40.550589Z","iopub.status.idle":"2022-08-22T06:18:40.559569Z","shell.execute_reply.started":"2022-08-22T06:18:40.550559Z","shell.execute_reply":"2022-08-22T06:18:40.558780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iter_test = env.iter_test()\nprior_test_df = None","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:18:40.560828Z","iopub.execute_input":"2022-08-22T06:18:40.561227Z","iopub.status.idle":"2022-08-22T06:18:40.572236Z","shell.execute_reply.started":"2022-08-22T06:18:40.561190Z","shell.execute_reply":"2022-08-22T06:18:40.571619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfor (test_df, sample_prediction_df) in iter_test:    \n    if prior_test_df is not None:\n        prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])\n        prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)       \n        prior_test_df['prior_question_had_explanation'].fillna(False, inplace=True)       \n        prior_test_df.prior_question_had_explanation=prior_test_df.prior_question_had_explanation.astype('int8')\n    \n        user_ids = prior_test_df['user_id'].values\n        content_ids = prior_test_df['content_id'].values\n        task_container_ids = prior_test_df['task_container_id'].values\n        prior_question_had_explanations = prior_test_df['prior_question_had_explanation'].values\n        targets = prior_test_df[target].values\n       \n        for user_id, content_id,prior_question_had_explanation,task_container_id,answered_correctly in zip(user_ids, content_ids, prior_question_had_explanations,task_container_ids,targets):\n            user_sum_dict[user_id] += answered_correctly\n            user_count_dict[user_id] += 1         \n            explanation_sum_dict[user_id] += prior_question_had_explanation\n            explanation_count_dict[user_id] += 1\n            \n\n    prior_test_df = test_df.copy()\n    lecture_test_df = test_df[test_df['content_type_id'] == 1].reset_index(drop=True)\n    for i, (user_id,content_type_id, content_id) in enumerate(zip(lecture_test_df['user_id'].values,lecture_test_df['content_type_id'].values,lecture_test_df['content_id'].values)):\n      \n        user_lecture_sum_dict[user_id] += content_type_id\n        user_lecture_count_dict[user_id] += 1\n        #\n        if(len(user_lecture_stats_part[user_lecture_stats_part.user_id==user_id])==0):\n            user_lecture_stats_part = user_lecture_stats_part.append([{'user_id':user_id}], ignore_index=True)\n            user_lecture_stats_part.fillna(0, inplace=True)\n            user_lecture_stats_part.loc[user_lecture_stats_part.user_id==user_id,part_lectures_columns + types_of_lectures_columns]+=lectures_df[lectures_df.lecture_id==content_id][part_lectures_columns + types_of_lectures_columns].values\n        else:\n            user_lecture_stats_part.loc[user_lecture_stats_part.user_id==user_id,part_lectures_columns + types_of_lectures_columns]+=lectures_df[lectures_df.lecture_id==content_id][part_lectures_columns + types_of_lectures_columns].values\n  \n        \n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n   \n    test_df['prior_question_had_explanation'].fillna(False, inplace=True)\n    test_df.prior_question_had_explanation=test_df.prior_question_had_explanation.astype('int8')\n    test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)\n    \n\n    user_lecture_sum = np.zeros(len(test_df), dtype=np.int16)\n    user_lecture_count = np.zeros(len(test_df), dtype=np.int16) \n    \n    user_sum = np.zeros(len(test_df), dtype=np.int16)\n    user_count = np.zeros(len(test_df), dtype=np.int16)\n    content_sum = np.zeros(len(test_df), dtype=np.int32)\n    content_count = np.zeros(len(test_df), dtype=np.int32)\n    task_container_sum = np.zeros(len(test_df), dtype=np.int32)\n    task_container_count = np.zeros(len(test_df), dtype=np.int32)\n    task_container_std = np.zeros(len(test_df), dtype=np.float16)\n    content_task_mean = np.zeros(len(test_df), dtype=np.float16)\n    explanation_sum = np.zeros(len(test_df), dtype=np.int32)\n    explanation_count = np.zeros(len(test_df), dtype=np.int32)\n    delta_prior_question_elapsed_time = np.zeros(len(test_df), dtype=np.int32)\n\n    attempt_no_count = np.zeros(len(test_df), dtype=np.int16)\n    lagtime = np.zeros(len(test_df), dtype=np.int32)\n    lagtime_mean = np.zeros(len(test_df), dtype=np.int32)\n   \n    \n    for i, (user_id,prior_question_had_explanation,content_type_id,prior_question_elapsed_time,timestamp, content_id,task_container_id) in enumerate(zip(test_df['user_id'].values,test_df['prior_question_had_explanation'].values,test_df['content_type_id'].values,test_df['prior_question_elapsed_time'].values,test_df['timestamp'].values, test_df['content_id'].values, test_df['task_container_id'].values)):\n         \n        user_lecture_sum_dict[user_id] += content_type_id\n        user_lecture_count_dict[user_id] += 1\n        \n        user_lecture_sum[i] = user_lecture_sum_dict[user_id]\n        user_lecture_count[i] = user_lecture_count_dict[user_id]\n        \n        user_sum[i] = user_sum_dict[user_id]\n        user_count[i] = user_count_dict[user_id]\n        content_sum[i] = content_sum_dict[content_id]\n        content_count[i] = content_count_dict[content_id]\n        task_container_sum[i] = task_container_sum_dict[task_container_id]\n        task_container_count[i] = task_container_count_dict[task_container_id]\n        task_container_std[i]=task_container_std_dict[task_container_id]\n      \n        explanation_sum[i] = explanation_sum_dict[user_id]\n        explanation_count[i] = explanation_count_dict[user_id]\n  \n        if user_id in max_timestamp_u_dict['max_time_stamp'].keys():\n            lagtime[i]=timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]\n            max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp\n            lagtime_mean[i]=(lagtime_mean_dict[user_id]+lagtime[i])/2           \n        else:\n            lagtime[i]=0\n            max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})\n            lagtime_mean_dict.update({user_id:timestamp})\n            lagtime_mean[i]=(lagtime_mean_dict[user_id]+lagtime[i])/2\n            \n        if user_id in user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].keys():            \n            delta_prior_question_elapsed_time[i]=prior_question_elapsed_time-user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]\n            user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]=prior_question_elapsed_time\n        else:           \n            delta_prior_question_elapsed_time[i]=0    \n            user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].update({user_id:prior_question_elapsed_time})\n           \n        \n        \n    \n    #\n    #test_df = pd.merge(test_df, questions_df, on='content_id', how='left',right_index=True)    \n    #test_df = pd.concat([test_df.reset_index(drop=True), questions_df.reindex(test_df['content_id'].values).reset_index(drop=True)], axis=1)\n    test_df=test_df.merge(questions_df.loc[questions_df.index.isin(test_df['content_id'])],\n                  how='left', on='content_id', right_index=True)\n    \n    #test_df = pd.merge(test_df, user_lecture_stats_part, on=['user_id'], how=\"left\",right_index=True)\n    #test_df = pd.concat([test_df.reset_index(drop=True), user_lecture_stats_part.reindex(test_df['user_id'].values).reset_index(drop=True)], axis=1)\n#     test_df=test_df.merge(user_lecture_stats_part.loc[user_lecture_stats_part.index.isin(test_df['user_id'])],\n#                   how='left', on='user_id', right_index=True)\n \n    test_df['user_lecture_lv'] = user_lecture_sum / user_lecture_count\n    test_df['user_lecture_cumsum'] = user_lecture_sum\n    test_df['user_correctness'] = user_sum / user_count\n    test_df['user_correct_cumcount'] =user_count\n    test_df['user_correct_cumsum'] =user_sum\n    #\n    test_df['content_correctness'] = content_sum / content_count\n    test_df['content_count'] = content_count\n    test_df['content_sum'] = content_sum\n    \n    test_df['task_container_correctness'] = task_container_sum / task_container_count\n    test_df['task_container_sum'] = task_container_sum \n    test_df['task_container_std'] = task_container_std \n    #test_df['content_task_mean'] = content_task_mean \n    \n    test_df['explanation_mean'] = explanation_sum / explanation_count\n    test_df['explanation_cumsum'] = explanation_sum \n    \n    #\n    test_df['delta_prior_question_elapsed_time'] = delta_prior_question_elapsed_time \n    \n  \n \n    test_df[\"attempt_no\"] = test_df[[\"user_id\", \"content_id\"]].apply(lambda row: get_max_attempt(row[\"user_id\"], row[\"content_id\"]), axis=1)\n    test_df[\"lagtime\"]=lagtime\n    test_df[\"lagtime_mean\"]=lagtime_mean\n\n    test_df['user_correctness'].fillna( 1, inplace=True)\n    test_df['attempt_no'].fillna(1, inplace=True)\n    #\n    test_df.fillna(0, inplace=True)\n    \n\n    test_df['timestamp']=test_df['timestamp']/(1000*3600)\n    test_df.timestamp=test_df.timestamp.astype('int16')\n\n\n    sub_preds = np.zeros(test_df.shape[0])\n    for i, model in enumerate(clfs, 1):\n        test_preds  = model.predict(test_df[features])\n        sub_preds += test_preds\n    test_df[target]=sub_preds / len(clfs)\n    \n#     if(flag_lgbm):\n#         test_df[target] = model.predict(test_df[features])\n#     else:\n#         test_df[target] = model.predict(test_df[features].values)\n    env.predict(test_df[['row_id', target]])","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:18:40.573289Z","iopub.execute_input":"2022-08-22T06:18:40.573678Z","iopub.status.idle":"2022-08-22T06:18:41.235676Z","shell.execute_reply.started":"2022-08-22T06:18:40.573650Z","shell.execute_reply":"2022-08-22T06:18:41.234594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T06:46:19.870153Z","iopub.execute_input":"2022-08-22T06:46:19.870485Z","iopub.status.idle":"2022-08-22T06:46:19.895082Z","shell.execute_reply.started":"2022-08-22T06:46:19.870457Z","shell.execute_reply":"2022-08-22T06:46:19.894214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}