{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Combining of Simple and Speedy SAKT with Riiid LGBM bagging2\n\nSource Kernels\n1. SAKT with Randomization & State Updates LB0.771 https://www.kaggle.com/leadbest/sakt-with-randomization-state-updates\n2. Riiid! LGBM bagging2 LB0.772 https://www.kaggle.com/zephyrwang666/riiid-lgbm-bagging2"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install ../input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport psutil\n\nfrom collections import defaultdict\nimport datatable as dt\nimport lightgbm as lgb\nfrom matplotlib import pyplot as plt\nimport riiideducation\nimport random\nfrom sklearn.metrics import roc_auc_score\nimport gc\n\n_ = np.seterr(divide='ignore', invalid='ignore')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocess"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_types_dict = {\n    'timestamp': 'int64',\n    'user_id': 'int32', \n    'content_id': 'int16', \n    'content_type_id':'int8', \n    'task_container_id': 'int16',\n    #'user_answer': 'int8',\n    'answered_correctly': 'int8', \n    'prior_question_elapsed_time': 'float32', \n    'prior_question_had_explanation': 'bool'\n}\ntarget = 'answered_correctly'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = dt.fread('../input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(psutil.virtual_memory().percent)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#reading in lecture df\nlectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')\n\nlectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])\n\npart_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]\n\ntypes_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_lectures = train_df[train_df.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_lecture_stats_part = train_lectures.groupby('user_id',as_index = False)[part_lectures_columns + types_of_lectures_columns].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lecturedata_types_dict = {   \n    'user_id': 'int32', \n    'part_1': 'int8',\n    'part_2': 'int8',\n    'part_3': 'int8',\n    'part_4': 'int8',\n    'part_5': 'int8',\n    'part_6': 'int8',\n    'part_7': 'int8',\n    'type_of_concept': 'int8',\n    'type_of_intention': 'int8',\n    'type_of_solving_question': 'int8',\n    'type_of_starter': 'int8'\n}\nuser_lecture_stats_part = user_lecture_stats_part.astype(lecturedata_types_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for column in user_lecture_stats_part.columns:\n    #bool_column = column + '_boolean'\n    if(column !='user_id'):\n        user_lecture_stats_part[column] = (user_lecture_stats_part[column] > 0).astype('int8')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_lectures[train_lectures.user_id==5382]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_lecture_stats_part[user_lecture_stats_part.user_id==5382]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_lecture_stats_part.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_lecture_stats_part.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#clearing memory\ndel(train_lectures)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(psutil.virtual_memory().percent)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cum = train_df.groupby('user_id')['content_type_id'].agg(['cumsum', 'cumcount'])\ntrain_df['user_lecture_cumsum'] = cum['cumsum'] \ntrain_df['user_lecture_lv'] = cum['cumsum'] / cum['cumcount']\n\n\ntrain_df.user_lecture_lv=train_df.user_lecture_lv.astype('float16')\ntrain_df.user_lecture_cumsum=train_df.user_lecture_cumsum.astype('int8')\nuser_lecture_agg = train_df.groupby('user_id')['content_type_id'].agg(['sum', 'count'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['prior_question_had_explanation'].fillna(False, inplace=True)\ntrain_df = train_df.astype(data_types_dict)\ntrain_df = train_df[train_df[target] != -1].reset_index(drop=True)\nprior_question_elapsed_time_mean=train_df['prior_question_elapsed_time'].mean()\ntrain_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"max_timestamp_u = train_df[['user_id','timestamp']].groupby(['user_id']).agg(['max']).reset_index()\n#max_timestamp_u = train_df[['user_id','timestamp']].groupby(['user_id']).agg(['max'])\nmax_timestamp_u.columns = ['user_id', 'max_time_stamp']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain_df['lagtime'] = train_df.groupby('user_id')['timestamp'].shift()\ntrain_df['lagtime']=train_df['timestamp']-train_df['lagtime']\ntrain_df['lagtime'].fillna(0, inplace=True)\ntrain_df.lagtime=train_df.lagtime.astype('int32')\n#train_df.drop(columns=['timestamp'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lagtime_agg = train_df.groupby('user_id')['lagtime'].agg(['mean'])\ntrain_df['lagtime_mean'] = train_df['user_id'].map(lagtime_agg['mean'])\ntrain_df.lagtime_mean=train_df.lagtime_mean.astype('int32')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_prior_question_elapsed_time = train_df[['user_id','prior_question_elapsed_time']].groupby(['user_id']).tail(1)\n#max_timestamp_u = train_df[['user_id','timestamp']].groupby(['user_id']).agg(['max'])\nuser_prior_question_elapsed_time.columns = ['user_id', 'prior_question_elapsed_time']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain_df['delta_prior_question_elapsed_time'] = train_df.groupby('user_id')['prior_question_elapsed_time'].shift()\ntrain_df['delta_prior_question_elapsed_time']=train_df['prior_question_elapsed_time']-train_df['delta_prior_question_elapsed_time']\ntrain_df['delta_prior_question_elapsed_time'].fillna(0, inplace=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.delta_prior_question_elapsed_time=train_df.delta_prior_question_elapsed_time.astype('int32')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['timestamp']=train_df['timestamp']/(1000*3600)\ntrain_df.timestamp=train_df.timestamp.astype('int16')\n#","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain_df['lag'] = train_df.groupby('user_id')[target].shift()\n\ncum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])\ntrain_df['user_correctness'] = cum['cumsum'] / cum['cumcount']\ntrain_df['user_correct_cumsum'] = cum['cumsum']\ntrain_df['user_correct_cumcount'] = cum['cumcount']\ntrain_df.drop(columns=['lag'], inplace=True)\n\n# train_df['user_correctness'].fillna(1, inplace=True)\ntrain_df['user_correct_cumsum'].fillna(0, inplace=True)\n#train_df['user_correct_cumcount'].fillna(0, inplace=True)\ntrain_df.user_correctness=train_df.user_correctness.astype('float16')\ntrain_df.user_correct_cumcount=train_df.user_correct_cumcount.astype('int16')\ntrain_df.user_correct_cumsum=train_df.user_correct_cumsum.astype('int16')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.prior_question_had_explanation=train_df.prior_question_had_explanation.astype('int8')\n\ntrain_df['lag'] = train_df.groupby('user_id')['prior_question_had_explanation'].shift()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ncum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])\ntrain_df['explanation_mean'] = cum['cumsum'] / cum['cumcount']\ntrain_df['explanation_cumsum'] = cum['cumsum'] \ntrain_df.drop(columns=['lag'], inplace=True)\n\ntrain_df['explanation_mean'].fillna(0, inplace=True)\ntrain_df['explanation_cumsum'].fillna(0, inplace=True)\ntrain_df.explanation_mean=train_df.explanation_mean.astype('float16')\ntrain_df.explanation_cumsum=train_df.explanation_cumsum.astype('int16')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del cum\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain_df[\"attempt_no\"] = 1\ntrain_df.attempt_no=train_df.attempt_no.astype('int8')\ntrain_df[\"attempt_no\"] = train_df[[\"user_id\",\"content_id\",'attempt_no']].groupby([\"user_id\",\"content_id\"])[\"attempt_no\"].cumsum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"explanation_agg = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['sum', 'count'])\nexplanation_agg=explanation_agg.astype('int16')\n#train_df.drop(columns=['prior_question_had_explanation'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_agg = train_df.groupby('user_id')[target].agg(['sum', 'count'])\ncontent_agg = train_df.groupby('content_id')[target].agg(['sum', 'count','var'])\ntask_container_agg = train_df.groupby('task_container_id')[target].agg(['sum', 'count','var'])\n\n#prior_question_elapsed_time_agg = train_df.groupby('user_id')['prior_question_elapsed_time'].agg(['sum', 'count'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_agg=user_agg.astype('int16')\ncontent_agg=content_agg.astype('float32')\ntask_container_agg=task_container_agg.astype('float32')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"attempt_no_agg=train_df.groupby([\"user_id\",\"content_id\"])[\"attempt_no\"].agg(['sum'])\nattempt_no_agg=attempt_no_agg.astype('int8')\n#attempt_series = train_df[['user_id', 'content_id','attempt_no']].groupby(['user_id','content_id'])['attempt_no'].max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['content_count'] = train_df['content_id'].map(content_agg['count']).astype('int32')\ntrain_df['content_sum'] = train_df['content_id'].map(content_agg['sum']).astype('int32')\ntrain_df['content_correctness'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'])\ntrain_df.content_correctness=train_df.content_correctness.astype('float16')\ntrain_df['task_container_sum'] = train_df['task_container_id'].map(task_container_agg['sum']).astype('int32')\ntrain_df['task_container_std'] = train_df['task_container_id'].map(task_container_agg['var']).astype('float16')\ntrain_df['task_container_correctness'] = train_df['task_container_id'].map(task_container_agg['sum'] / task_container_agg['count'])\ntrain_df.task_container_correctness=train_df.task_container_correctness.astype('float16')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df = pd.read_csv(\n    '../input/riiid-test-answer-prediction/questions.csv', \n    usecols=[0, 1,3,4],\n    dtype={'question_id': 'int16','bundle_id': 'int16', 'part': 'int8','tags': 'str'}\n)\nquestions_df['part_bundle_id']=questions_df['part']*100000+questions_df['bundle_id']\nquestions_df.part_bundle_id=questions_df.part_bundle_id.astype('int32')\ntag = questions_df[\"tags\"].str.split(\" \", n = 10, expand = True)\ntag.columns = ['tags1','tags2','tags3','tags4','tags5','tags6']\n#\n\ntag.fillna(0, inplace=True)\ntag = tag.astype('int16')\nquestions_df =  pd.concat([questions_df,tag],axis=1).drop(['tags'],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df.rename(columns={'question_id':'content_id'}, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df['content_correctness'] = questions_df['content_id'].map(content_agg['sum'] / content_agg['count'])\nquestions_df.content_correctness=questions_df.content_correctness.astype('float16')\nquestions_df['content_correctness_std'] = questions_df['content_id'].map(content_agg['var'])\nquestions_df.content_correctness_std=questions_df.content_correctness_std.astype('float16')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"part_agg = questions_df.groupby('part')['content_correctness'].agg(['mean', 'var'])\nquestions_df['part_correctness_mean'] = questions_df['part'].map(part_agg['mean'])\nquestions_df['part_correctness_std'] = questions_df['part'].map(part_agg['var'])\nquestions_df.part_correctness_mean=questions_df.part_correctness_mean.astype('float16')\nquestions_df.part_correctness_std=questions_df.part_correctness_std.astype('float16')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bundle_agg = questions_df.groupby('bundle_id')['content_correctness'].agg(['mean'])\nquestions_df['bundle_correctness'] = questions_df['bundle_id'].map(bundle_agg['mean'])\nquestions_df.bundle_correctness=questions_df.bundle_correctness.astype('float16')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tags1_agg = questions_df.groupby('tags1')['content_correctness'].agg(['mean', 'var'])\nquestions_df['tags1_correctness_mean'] = questions_df['tags1'].map(tags1_agg['mean'])\nquestions_df['tags1_correctness_std'] = questions_df['tags1'].map(tags1_agg['var'])\nquestions_df.tags1_correctness_mean=questions_df.tags1_correctness_mean.astype('float16')\nquestions_df.tags1_correctness_std=questions_df.tags1_correctness_std.astype('float16')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df.drop(columns=['content_correctness'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del bundle_agg\ndel part_agg\ndel tags1_agg\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#pd.set_option(\"display.max_columns\",500)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#questions_df.drop(columns=['tags4','tags5','tags6'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['user_correctness'].fillna( 1, inplace=True)\ntrain_df['attempt_no'].fillna(1, inplace=True)\n#\ntrain_df.fillna(0, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_df.drop(columns=['content_type_id'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.dtypes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# SAKT Part I"},{"metadata":{"trusted":true},"cell_type":"code","source":"#HDKIM \n\nMAX_SEQ = 160\n\nskills = train_df[\"content_id\"].unique()\nn_skill = len(skills)\nprint(\"number skills\", len(skills))\n\ngroup = train_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id').apply(lambda r: (\n            r['content_id'].values,\n            r['answered_correctly'].values))\n\nfor user_id in group.index:\n    q, qa = group[user_id]\n    if len(q)>MAX_SEQ:\n        group[user_id] = (q[-MAX_SEQ:],qa[-MAX_SEQ:])\n        \nimport pickle\npickle.dump(group, open(\"group.pkl\", \"wb\"))\ndel group\ngc.collect()\n\n#HDKIMHDKIM\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"features = [\n#   'user_id',\n#HDKIM    'timestamp',\n    'lagtime',\n    'lagtime_mean',\n    'content_id',\n    'task_container_id',\n    'user_lecture_cumsum',\n    'user_lecture_lv',\n    'prior_question_elapsed_time',\n    'delta_prior_question_elapsed_time',\n    'user_correctness',\n    'user_correct_cumcount',\n    'user_correct_cumsum',\n    'content_correctness',\n    'content_correctness_std',\n    'content_count',\n    'content_sum',\n    'task_container_correctness',\n    'task_container_std',\n    'task_container_sum',\n    'bundle_correctness',\n    'attempt_no',\n    'part',\n    'part_correctness_mean',\n    'part_correctness_std',\n    'tags1',\n    'tags1_correctness_mean',\n    'tags1_correctness_std',\n#HDKIM    'tags2',\n#HDKIM    'tags3',\n#HDKIM    'tags4',\n#HDKIM    'tags5',\n#HDKIM    'tags6',\n    'bundle_id',\n    'part_bundle_id',\n    'explanation_mean', \n    'explanation_cumsum',\n    'prior_question_had_explanation',\n#     'part_1',\n#     'part_2',\n#     'part_3',\n#     'part_4',\n#     'part_5',\n#     'part_6',\n#     'part_7',\n#     'type_of_concept',\n#     'type_of_intention',\n#     'type_of_solving_question',\n#     'type_of_starter'\n]\ncategorical_columns= [\n#   'user_id',\n    'content_id',\n    'task_container_id',\n    'part',        \n    'tags1',\n#HDKIM    'tags2',\n#HDKIM    'tags3',\n#HDKIM    'tags4',\n#HDKIM    'tags5',\n#HDKIM    'tags6',\n    'bundle_id',\n    'part_bundle_id',\n    'prior_question_had_explanation',\n#     'part_1',\n#     'part_2',\n#     'part_3',\n#     'part_4',\n#     'part_5',\n#     'part_6',\n#     'part_7',\n#     'type_of_concept',\n#     'type_of_intention',\n#     'type_of_solving_question',\n#     'type_of_starter'\n]\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"flag_lgbm=True\nclfs = list()\nparams = {\n'num_leaves': 350,\n'max_bin':700,\n'min_child_weight': 0.03454472573214212,\n'feature_fraction': 0.58,\n'bagging_fraction': 0.58,\n#'min_data_in_leaf': 106,\n'objective': 'binary',\n'max_depth': -1,\n'learning_rate': 0.05,\n\"boosting_type\": \"gbdt\",\n\"bagging_seed\": 11,\n\"metric\": 'auc',\n\"verbosity\": -1,\n'reg_alpha': 0.3899927210061127,\n'reg_lambda': 0.6485237330340494,\n'random_state': 47\n}\ntrains=list()\nvalids=list()\nnum=1\nfor i in range(0,num):\n  \n    #train_df=train_df.reset_index(drop=True)\n    train_df_clf=train_df.sample(n=1200*10000)\n    print('sample end')\n    #train_df.drop(train_df_clf.index, inplace=True)\n    #print('train_df drop end')\n    \n   \n    del train_df\n    \n    users=train_df_clf['user_id'].drop_duplicates()#去重\n    \n    users=users.sample(frac=0.025)\n    users_df=pd.DataFrame()\n    users_df['user_id']=users.values\n  \n  \n    valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)\n    del users_df\n    del users\n    gc.collect()\n    #\n    train_df_clf.drop(valid_df_newuser.index, inplace=True)\n   \n    #-----------\n    #train_df_clf=train_df_clf.sample(frac=0.2)\n    #train_df_clf.drop(valid_df_newuser.index, inplace=True)\n    train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)#\n    valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)#\n    \n#     train_df_clf = pd.merge(train_df_clf, user_lecture_stats_part, on='user_id', how=\"left\",right_index=True)\n#     valid_df_newuser = pd.merge(valid_df_newuser, user_lecture_stats_part, on='user_id', how=\"left\",right_index=True)\n\n    valid_df=train_df_clf.sample(frac=0.15)\n    train_df_clf.drop(valid_df.index, inplace=True)\n   \n    valid_df = valid_df.append(valid_df_newuser)\n    del valid_df_newuser\n    gc.collect()\n    #\n\n    trains.append(train_df_clf)\n    valids.append(valid_df)\n    print('valid_df length：',len(valid_df))\n    #train_df=train_df.reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#del train_df\ndel train_df_clf\ndel valid_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfor i in range(0,num):\n\n#     \n    tr_data = lgb.Dataset(trains[i][features], label=trains[i][target])\n    va_data = lgb.Dataset(valids[i][features], label=valids[i][target])\n    \n#     del train_df_clf\n#     del valid_df\n#     gc.collect()\n    del trains\n    del valids\n    gc.collect()\n\n    model = lgb.train(\n        params, \n        tr_data,\n#         train_df[features],\n#         train_df[target],\n        num_boost_round=5000,\n        #valid_sets=[(train_df[features],train_df[target]), (valid_df[features],valid_df[target])], \n        valid_sets=[tr_data, va_data],\n        early_stopping_rounds=50,\n        feature_name=features,\n        categorical_feature=categorical_columns,\n        verbose_eval=50\n    )\n    clfs.append(model)\n    #print('auc:', roc_auc_score(valid_df[target], model.predict(valid_df[features])))\n    #model.save_model(f'model.txt')\n    lgb.plot_importance(model, importance_type='gain')\n    plt.show()\n\n    del tr_data\n    del va_data\n    gc.collect()\n#    \n# del trains\n# del valids\n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"user_sum_dict = user_agg['sum'].astype('int16').to_dict(defaultdict(int))\nuser_count_dict = user_agg['count'].astype('int16').to_dict(defaultdict(int))\ncontent_sum_dict = content_agg['sum'].astype('int32').to_dict(defaultdict(int))\ncontent_count_dict = content_agg['count'].astype('int32').to_dict(defaultdict(int))\n\ndel user_agg\ndel content_agg\ngc.collect()\n\ntask_container_sum_dict = task_container_agg['sum'].astype('int32').to_dict(defaultdict(int))\ntask_container_count_dict = task_container_agg['count'].astype('int32').to_dict(defaultdict(int))\ntask_container_std_dict = task_container_agg['var'].astype('float16').to_dict(defaultdict(int))\n\nexplanation_sum_dict = explanation_agg['sum'].astype('int16').to_dict(defaultdict(int))\nexplanation_count_dict = explanation_agg['count'].astype('int16').to_dict(defaultdict(int))\ndel task_container_agg\ndel explanation_agg\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_lecture_sum_dict = user_lecture_agg['sum'].astype('int16').to_dict(defaultdict(int))\nuser_lecture_count_dict = user_lecture_agg['count'].astype('int16').to_dict(defaultdict(int))\n\nlagtime_mean_dict = lagtime_agg['mean'].astype('int32').to_dict(defaultdict(int))\n#del prior_question_elapsed_time_agg\ndel user_lecture_agg\ndel lagtime_agg\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"attempt_no_agg=attempt_no_agg[attempt_no_agg['sum'] >1]\nattempt_no_sum_dict = attempt_no_agg['sum'].to_dict(defaultdict(int))\n\ndel attempt_no_agg\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"max_timestamp_u_dict=max_timestamp_u.set_index('user_id').to_dict()\nuser_prior_question_elapsed_time_dict=user_prior_question_elapsed_time.set_index('user_id').to_dict()\n#del question_elapsed_time_agg\ndel max_timestamp_u\ndel user_prior_question_elapsed_time\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(max_timestamp_u_dict['max_time_stamp'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_max_attempt(user_id,content_id):\n    k = (user_id,content_id)\n\n    if k in attempt_no_sum_dict.keys():\n        attempt_no_sum_dict[k]+=1\n        return attempt_no_sum_dict[k]\n\n    attempt_no_sum_dict[k] = 1\n    return attempt_no_sum_dict[k]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(psutil.virtual_memory().percent)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# SAKT Part II"},{"metadata":{"trusted":true},"cell_type":"code","source":"#HDKIM SAKT\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\n\nclass FFN(nn.Module):\n    def __init__(self, state_size=200):\n        super(FFN, self).__init__()\n        self.state_size = state_size\n\n        self.lr1 = nn.Linear(state_size, state_size)\n        self.relu = nn.ReLU()\n        self.lr2 = nn.Linear(state_size, state_size)\n        self.dropout = nn.Dropout(0.2)\n    \n    def forward(self, x):\n        x = self.lr1(x)\n        x = self.relu(x)\n        x = self.lr2(x)\n        return self.dropout(x)\n\ndef future_mask(seq_length):\n    future_mask = np.triu(np.ones((seq_length, seq_length)), k=1).astype('bool')\n    return torch.from_numpy(future_mask)\n\nclass SAKTModel(nn.Module):\n    def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128): #HDKIM 100\n        super(SAKTModel, self).__init__()\n        self.n_skill = n_skill\n        self.embed_dim = embed_dim\n\n        self.embedding = nn.Embedding(2*n_skill+1, embed_dim)\n        self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)\n        self.e_embedding = nn.Embedding(n_skill+1, embed_dim)\n\n        self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)\n\n        self.dropout = nn.Dropout(0.2)\n        self.layer_normal = nn.LayerNorm(embed_dim) \n\n        self.ffn = FFN(embed_dim)\n        self.pred = nn.Linear(embed_dim, 1)\n    \n    def forward(self, x, question_ids):\n        device = x.device        \n        x = self.embedding(x)\n        pos_id = torch.arange(x.size(1)).unsqueeze(0).to(device)\n\n        pos_x = self.pos_embedding(pos_id)\n        x = x + pos_x\n\n        e = self.e_embedding(question_ids)\n\n        x = x.permute(1, 0, 2) # x: [bs, s_len, embed] => [s_len, bs, embed]\n        e = e.permute(1, 0, 2)\n        att_mask = future_mask(x.size(0)).to(device)\n        att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)\n        att_output = self.layer_normal(att_output + e)\n        att_output = att_output.permute(1, 0, 2) # att_output: [s_len, bs, embed] => [bs, s_len, embed]\n\n        x = self.ffn(att_output)\n        x = self.layer_normal(x + att_output)\n        x = self.pred(x)\n\n        return x.squeeze(-1), att_weight\n    \nclass TestDataset(Dataset):\n    def __init__(self, samples, test_df, skills, max_seq=MAX_SEQ): #HDKIM 100\n        super(TestDataset, self).__init__()\n        self.samples = samples\n        self.user_ids = [x for x in test_df[\"user_id\"].unique()]\n        self.test_df = test_df\n        self.skills = skills\n        self.n_skill = len(skills)\n        self.max_seq = max_seq\n\n    def __len__(self):\n        return self.test_df.shape[0]\n\n    def __getitem__(self, index):\n        test_info = self.test_df.iloc[index]\n\n        user_id = test_info[\"user_id\"]\n        target_id = test_info[\"content_id\"]\n\n        q = np.zeros(self.max_seq, dtype=int)\n        qa = np.zeros(self.max_seq, dtype=int)\n\n        if user_id in self.samples.index:\n            q_, qa_ = self.samples[user_id]\n            \n            seq_len = len(q_)\n\n            if seq_len >= self.max_seq:\n                q = q_[-self.max_seq:]\n                qa = qa_[-self.max_seq:]\n            else:\n                q[-seq_len:] = q_\n                qa[-seq_len:] = qa_          \n        \n        x = np.zeros(self.max_seq-1, dtype=int)\n        x = q[1:].copy()\n        x += (qa[1:] == 1) * self.n_skill\n        \n        questions = np.append(q[2:], [target_id])\n        \n        return x, questions\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nSAKT_model = SAKTModel(n_skill, embed_dim=128)\n\ntry:\n    SAKT_model.load_state_dict(torch.load(\"../input/sakt-with-randomization-state-updates/SAKT-HDKIM.pt\"))\nexcept:\n    SAKT_model.load_state_dict(torch.load(\"../input/sakt-with-randomization-state-updates/SAKT-HDKIM.pt\", map_location='cpu'))\n\nSAKT_model.to(device)\nSAKT_model.eval()\n\nimport pickle\ngroup = pickle.load(open(\"group.pkl\", \"rb\"))\n\nprint(psutil.virtual_memory().percent)\n\n#HDKIMHDKIM","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = lgb.Booster(model_file='../input/riiid-lgbm-starter/model.txt')\nenv = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iter_test = env.iter_test()\nprior_test_df = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nfor (test_df, sample_prediction_df) in iter_test:    \n    if (prior_test_df is not None) & (psutil.virtual_memory().percent<90):\n        print(psutil.virtual_memory().percent)\n        prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])\n        prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)       \n        prior_test_df['prior_question_had_explanation'].fillna(False, inplace=True)       \n        prior_test_df.prior_question_had_explanation=prior_test_df.prior_question_had_explanation.astype('int8')\n        \n        #HDKIM SAKT State Update\n        prev_group = prior_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id').apply(lambda r: (\n            r['content_id'].values,\n            r['answered_correctly'].values))\n        for prev_user_id in prev_group.index:\n            prev_group_content = prev_group[prev_user_id][0]\n            prev_group_ac = prev_group[prev_user_id][1]\n            if prev_user_id in group.index:\n                group[prev_user_id] = (np.append(group[prev_user_id][0],prev_group_content), \n                                       np.append(group[prev_user_id][1],prev_group_ac))\n            else:\n                group[prev_user_id] = (prev_group_content,prev_group_ac)\n            if len(group[prev_user_id][0])>MAX_SEQ:\n                new_group_content = group[prev_user_id][0][-MAX_SEQ:]\n                new_group_ac = group[prev_user_id][1][-MAX_SEQ:]\n                group[prev_user_id] = (new_group_content,new_group_ac)\n\n        #HDKIMHDKIM\n    \n        user_ids = prior_test_df['user_id'].values\n        content_ids = prior_test_df['content_id'].values\n        task_container_ids = prior_test_df['task_container_id'].values\n        prior_question_had_explanations = prior_test_df['prior_question_had_explanation'].values\n        targets = prior_test_df[target].values\n       \n        for user_id, content_id,prior_question_had_explanation,task_container_id,answered_correctly in zip(user_ids, content_ids, prior_question_had_explanations,task_container_ids,targets):\n            user_sum_dict[user_id] += answered_correctly\n            user_count_dict[user_id] += 1         \n            explanation_sum_dict[user_id] += prior_question_had_explanation\n            explanation_count_dict[user_id] += 1\n            \n\n    prior_test_df = test_df.copy()\n    lecture_test_df = test_df[test_df['content_type_id'] == 1].reset_index(drop=True)\n    \n    for i, (user_id,content_type_id, content_id) in enumerate(zip(lecture_test_df['user_id'].values,lecture_test_df['content_type_id'].values,lecture_test_df['content_id'].values)):\n      \n        user_lecture_sum_dict[user_id] += content_type_id\n        user_lecture_count_dict[user_id] += 1\n        #\n        if(len(user_lecture_stats_part[user_lecture_stats_part.user_id==user_id])==0):\n            user_lecture_stats_part = user_lecture_stats_part.append([{'user_id':user_id}], ignore_index=True)\n            user_lecture_stats_part.fillna(0, inplace=True)\n            user_lecture_stats_part.loc[user_lecture_stats_part.user_id==user_id,part_lectures_columns + types_of_lectures_columns]+=lectures_df[lectures_df.lecture_id==content_id][part_lectures_columns + types_of_lectures_columns].values\n        else:\n            user_lecture_stats_part.loc[user_lecture_stats_part.user_id==user_id,part_lectures_columns + types_of_lectures_columns]+=lectures_df[lectures_df.lecture_id==content_id][part_lectures_columns + types_of_lectures_columns].values\n  \n        \n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n   \n    #HDKIM SAKT\n    test_dataset = TestDataset(group, test_df, skills)\n    test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)\n    \n    SAKT_outs = []\n\n    for item in test_dataloader:\n        x = item[0].to(device).long()\n        target_id = item[1].to(device).long()\n\n        with torch.no_grad():\n            output, att_weight = SAKT_model(x, target_id)\n \n        output = torch.sigmoid(output)\n        output = output[:, -1]\n        SAKT_outs.extend(output.view(-1).data.cpu().numpy())\n    \n    #HDKIMHDKIM\n\n    test_df['prior_question_had_explanation'].fillna(False, inplace=True)\n    test_df.prior_question_had_explanation=test_df.prior_question_had_explanation.astype('int8')\n    test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)\n    \n\n    user_lecture_sum = np.zeros(len(test_df), dtype=np.int16)\n    user_lecture_count = np.zeros(len(test_df), dtype=np.int16) \n    \n    user_sum = np.zeros(len(test_df), dtype=np.int16)\n    user_count = np.zeros(len(test_df), dtype=np.int16)\n    content_sum = np.zeros(len(test_df), dtype=np.int32)\n    content_count = np.zeros(len(test_df), dtype=np.int32)\n    task_container_sum = np.zeros(len(test_df), dtype=np.int32)\n    task_container_count = np.zeros(len(test_df), dtype=np.int32)\n    task_container_std = np.zeros(len(test_df), dtype=np.float16)\n    content_task_mean = np.zeros(len(test_df), dtype=np.float16)\n    explanation_sum = np.zeros(len(test_df), dtype=np.int32)\n    explanation_count = np.zeros(len(test_df), dtype=np.int32)\n    delta_prior_question_elapsed_time = np.zeros(len(test_df), dtype=np.int32)\n\n    attempt_no_count = np.zeros(len(test_df), dtype=np.int16)\n    lagtime = np.zeros(len(test_df), dtype=np.int32)\n    lagtime_mean = np.zeros(len(test_df), dtype=np.int32)\n   \n    \n    for i, (user_id,prior_question_had_explanation,content_type_id,prior_question_elapsed_time,timestamp, content_id,task_container_id) in enumerate(zip(test_df['user_id'].values,test_df['prior_question_had_explanation'].values,test_df['content_type_id'].values,test_df['prior_question_elapsed_time'].values,test_df['timestamp'].values, test_df['content_id'].values, test_df['task_container_id'].values)):\n         \n        user_lecture_sum_dict[user_id] += content_type_id\n        user_lecture_count_dict[user_id] += 1\n        \n        user_lecture_sum[i] = user_lecture_sum_dict[user_id]\n        user_lecture_count[i] = user_lecture_count_dict[user_id]\n        \n        user_sum[i] = user_sum_dict[user_id]\n        user_count[i] = user_count_dict[user_id]\n        content_sum[i] = content_sum_dict[content_id]\n        content_count[i] = content_count_dict[content_id]\n        task_container_sum[i] = task_container_sum_dict[task_container_id]\n        task_container_count[i] = task_container_count_dict[task_container_id]\n        task_container_std[i]=task_container_std_dict[task_container_id]\n      \n        explanation_sum[i] = explanation_sum_dict[user_id]\n        explanation_count[i] = explanation_count_dict[user_id]\n  \n        if user_id in max_timestamp_u_dict['max_time_stamp'].keys():\n            lagtime[i]=timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]\n            max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp\n            lagtime_mean[i]=(lagtime_mean_dict[user_id]+lagtime[i])/2           \n        else:\n            lagtime[i]=0\n            max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})\n            lagtime_mean_dict.update({user_id:timestamp})\n            lagtime_mean[i]=(lagtime_mean_dict[user_id]+lagtime[i])/2\n            \n        if user_id in user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].keys():            \n            delta_prior_question_elapsed_time[i]=prior_question_elapsed_time-user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]\n            user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]=prior_question_elapsed_time\n        else:           \n            delta_prior_question_elapsed_time[i]=0    \n            user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].update({user_id:prior_question_elapsed_time})\n           \n        \n        \n    \n    #\n    #test_df = pd.merge(test_df, questions_df, on='content_id', how='left',right_index=True)    \n    #test_df = pd.concat([test_df.reset_index(drop=True), questions_df.reindex(test_df['content_id'].values).reset_index(drop=True)], axis=1)\n    test_df=test_df.merge(questions_df.loc[questions_df.index.isin(test_df['content_id'])],\n                  how='left', on='content_id', right_index=True)\n    \n    #test_df = pd.merge(test_df, user_lecture_stats_part, on=['user_id'], how=\"left\",right_index=True)\n    #test_df = pd.concat([test_df.reset_index(drop=True), user_lecture_stats_part.reindex(test_df['user_id'].values).reset_index(drop=True)], axis=1)\n#     test_df=test_df.merge(user_lecture_stats_part.loc[user_lecture_stats_part.index.isin(test_df['user_id'])],\n#                   how='left', on='user_id', right_index=True)\n \n    test_df['user_lecture_lv'] = user_lecture_sum / user_lecture_count\n    test_df['user_lecture_cumsum'] = user_lecture_sum\n    test_df['user_correctness'] = user_sum / user_count\n    test_df['user_correct_cumcount'] =user_count\n    test_df['user_correct_cumsum'] =user_sum\n    #\n    test_df['content_correctness'] = content_sum / content_count\n    test_df['content_count'] = content_count\n    test_df['content_sum'] = content_sum\n    \n    test_df['task_container_correctness'] = task_container_sum / task_container_count\n    test_df['task_container_sum'] = task_container_sum \n    test_df['task_container_std'] = task_container_std \n    #test_df['content_task_mean'] = content_task_mean \n    \n    test_df['explanation_mean'] = explanation_sum / explanation_count\n    test_df['explanation_cumsum'] = explanation_sum \n    \n    #\n    test_df['delta_prior_question_elapsed_time'] = delta_prior_question_elapsed_time \n    \n  \n \n    test_df[\"attempt_no\"] = test_df[[\"user_id\", \"content_id\"]].apply(lambda row: get_max_attempt(row[\"user_id\"], row[\"content_id\"]), axis=1)\n    test_df[\"lagtime\"]=lagtime\n    test_df[\"lagtime_mean\"]=lagtime_mean\n\n    test_df['user_correctness'].fillna( 1, inplace=True)\n    test_df['attempt_no'].fillna(1, inplace=True)\n    #\n    test_df.fillna(0, inplace=True)\n    \n\n    test_df['timestamp']=test_df['timestamp']/(1000*3600)\n    test_df.timestamp=test_df.timestamp.astype('int16')\n\n\n    sub_preds = np.zeros(test_df.shape[0])\n    for i, model in enumerate(clfs, 1):\n        test_preds  = model.predict(test_df[features])\n        sub_preds += test_preds\n    #HDKIM\n    #test_df[target] = sub_preds / len(clfs) #HDKIM\n    \n    lgbm_final = sub_preds / len(clfs)   \n    test_df[target] = np.array(SAKT_outs) * 0.5 + lgbm_final * 0.5\n    #HDKIMHDKIM\n\n    env.predict(test_df[['row_id', target]])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}