{"cells":[{"metadata":{},"cell_type":"markdown","source":"# This is a notebook for preprocessing.\n- **inference notebook**: https://www.kaggle.com/tkyiws/single-lgb-model-with-about-23-features\n\nThank you very much for your big help.\n- https://www.kaggle.com/ldevyataykina/riiid-exploratory-data-analysis-baseline?scriptVersionId=48691010  \n- https://www.kaggle.com/shoheiazuma/riiid-lgbm-starter  \n- https://www.kaggle.com/markwijkhuizen/riiid-training-and-prediction-using-a-state  \n- https://www.kaggle.com/its7171/time-series-api-iter-test-emulator"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgb\nfrom tqdm import tqdm\nimport gc\nimport pickle\nimport joblib\n\npd.set_option('display.max_columns', 50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_types_dict = {\n    'row_id': 'int64',\n    'timestamp': 'int64',\n    'user_id': 'int32',\n    'content_id': 'int16',\n    'content_type_id': 'int8',\n    'task_container_id': 'int16',\n    'answered_correctly':'int8',\n    'prior_question_elapsed_time': 'float32',\n    'prior_question_had_explanation': 'boolean'\n}\n                   \ntarget = 'answered_correctly'\n\nfeatures_dtypes = {\n    'content_id': 'int16',\n    'content_mean': 'float32',\n    'prior_question_elapsed_time': 'float64',\n    'prior_question_had_explanation': 'bool',\n    'user_correctness': 'float32',\n    'content_count': 'int32',\n    'part': 'int8',\n    'cumcount_u': 'uint16',\n    'cumcount_p': 'uint16',\n    'attempt': 'uint16',\n    'part_avg': 'float32',\n    'timestamp_diff1': 'float64',\n    'timestamp_diff2': 'float64',\n    'cluster_id': 'int8',\n    'cluster_avg': 'float32',\n    'cumcount_cl': 'uint16',\n    'target_lag': 'int8',\n    'cluster0_avg': 'float32',\n    'cluster1_avg': 'float32',\n    'cluster2_avg': 'float32',\n    'prior_tag': 'int16',\n    'task_num': 'int8',\n    'user_rating': 'float32',\n    'time_mean_diff': 'float32',\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Loading"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/riiid-test-answer-prediction/train.csv',\n                       usecols=[0, 1, 2, 3, 4, 5, 7, 8, 9],\n                       dtype=data_types_dict,\n                       nrows=10_000_000, # Some data will be used due to RAM constraints.\n                      )\nquestions_df = pd.read_csv(\n    '../input/riiid-test-answer-prediction/questions.csv', \n    usecols=[0, 1, 3],\n    dtype={'question_id': 'int16', 'bundle_id': 'int16', 'part': 'int8'}\n)\n\nlectures_df = pd.read_csv('../input/riiid-test-answer-prediction/lectures.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# prior_tag (1/2)\nTags from a last-minute lecture.\nIt will be reset if you take a series of lectures or answer a question.(Tag number or -1)"},{"metadata":{"trusted":true},"cell_type":"code","source":"lectures_df['content_type_id'] = 1\nlectures_df.columns = ['content_id', 'lecture_tag', 'lecture_part', 'type_of', 'content_type_id']\nlectures_df = lectures_df[['content_id', 'lecture_tag', 'content_type_id']].astype({'content_id': 'int16', 'lecture_tag': 'int16', 'content_type_id': 'int8'})\n# lectures_df.to_pickle('riiid_pre_data/lectures_df.pickle')\n# lectures_df = pd.read_pickle('riiid_pre_data/lectures_df.pickle')\n\ntrain_df = pd.merge(train_df, lectures_df, on=['content_id', 'content_type_id'], how='left')\ntrain_df['lecture_tag'].fillna(-1, inplace=True)\ntrain_df['prior_tag'] = train_df.groupby('user_id')['lecture_tag'].shift()\ntrain_df['prior_tag'].fillna(-1, inplace=True)\n\nlast_lecture_dict = train_df.groupby('user_id').tail(1)[['user_id', 'lecture_tag']].set_index('user_id')['lecture_tag'].astype('int16').to_dict()\n# joblib.dump(last_lecture_dict, \"dict_data/last_lecture_dict.pkl.zip\")\n# last_lecture_dict = joblib.load(\"dict_data/last_lecture_dict.pkl.zip\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.loc[89:91, ['user_id', 'content_type_id', 'lecture_tag', 'prior_tag']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('After the lecture:', train_df[(train_df[target]!=-1)&(train_df['prior_tag']!=-1)][target].mean())\nprint('-1               :', train_df[(train_df[target]!=-1)&(train_df['prior_tag']==-1)][target].mean())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The questions about the last lecture I took are simple."},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,5))\nplt.subplot(121)\nplt.title('Top 10 correct answers by prior_tag')\ntrain_df.groupby('prior_tag')[target].mean().sort_values().iloc[-10:].plot.barh()\nplt.subplot(122)\nplt.title('Worst 10 correct answers by prior_tag')\ntrain_df.groupby('prior_tag')[target].mean().sort_values(ascending=False).iloc[-10:].plot.barh()\nplt.show()\nplt.figure(figsize=(10,5))\nplt.title('number of occurances')\ntrain_df[train_df['prior_tag']!=-1]['prior_tag'].value_counts().iloc[:30].plot.bar();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"For the full data, the least number of occurrences is 366."},{"metadata":{},"cell_type":"markdown","source":"#  preprocessing (1/2)"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df[train_df[target] != -1].reset_index(drop=True)\ntrain_df['prior_question_had_explanation'].fillna(False, inplace=True)\ntrain_df = train_df.astype(data_types_dict)\n\n# Delete test users.\n# Intentionally? Deletes users who have answered the same question incorrectly in succession.\ntrain_df = train_df.drop(index=train_df[train_df['user_id']==1509564249].index).reset_index(drop=True)\n\n# task_num: Number of content_ids that share the task_container_id.\nquestions_df['task_num'] = questions_df['bundle_id'].map(questions_df.groupby('bundle_id')['question_id'].nunique())\nquestions_df.drop(columns=['bundle_id'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# cluster_id\nBased on the percentage of correct answers, median, standard deviation, and skewness of content_id, we clustered \"content_id\" into three classes."},{"metadata":{"trusted":true},"cell_type":"code","source":"cluster_data = pd.read_pickle('../input/sc-cluster-data/sc_cluster_data.pickle')\nquestions_df['cluster_id'] = questions_df['question_id'].map(cluster_data)\ndel cluster_data\n\n# questions_df.to_pickle('riiid_pre_data/questions_df.pickle')\n# questions_df = pd.read_pickle('riiid_pre_data/questions_df.pickle')\n\ntrain_df = pd.merge(train_df, questions_df, left_on='content_id', right_on='question_id', how='left')\ntrain_df.drop(columns=['question_id'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.groupby('cluster_id')[target].agg(['mean', 'count'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# timestamp_diff\nThe time between the completion of the last event and the completion of the current event."},{"metadata":{"trusted":true},"cell_type":"code","source":"timestamp_df= train_df.groupby(['user_id', 'task_container_id']).head(1)[['user_id', 'task_container_id', 'timestamp']]\n\ntimestamp_df['timestamp_diff1'] = timestamp_df.groupby('user_id')['timestamp'].diff()\ntimestamp_df['timestamp_diff2'] = timestamp_df.groupby('user_id')['timestamp'].diff(2)\n\n# time_dict1 = timestamp_df.groupby('user_id')['timestamp'].max().to_dict()\n# timestamp_df['timestamp'] = timestamp_df.groupby('user_id')['timestamp'].shift()\n# time_dict2 = timestamp_df.groupby('user_id')['timestamp'].max().to_dict()\n# timestamp_df['timestamp'] = timestamp_df.groupby('user_id')['timestamp'].shift()\n# time_dict3 = timestamp_df.groupby('user_id')['timestamp'].max().to_dict()\n\ntimestamp_df.drop(columns=['timestamp'], inplace=True)\n\ntrain_df = pd.merge(train_df, timestamp_df, on=['user_id', 'task_container_id'], how='left')\n\ndel timestamp_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# joblib.dump(time_dict1, \"riiid_pre_data/time_dict1.pkl.zip\")\n# joblib.dump(time_dict2, \"riiid_pre_data/time_dict2.pkl.zip\")\n# joblib.dump(time_dict3, \"riiid_pre_data/time_dict3.pkl.zip\")\n\n# time_dict1 = joblib.load(\"riiid_pre_data/time_dict1.pkl.zip\")\n# time_dict2 = joblib.load(\"riiid_pre_data/time_dict2.pkl.zip\")\n# time_dict3 = joblib.load(\"riiid_pre_data/time_dict3.pkl.zip\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Divide by task_num.\ntrain_df['timestamp_diff1'] = train_df['timestamp_diff1'] / train_df['task_num']\ntrain_df['timestamp_diff2'] = train_df['timestamp_diff2'] / train_df['task_num']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[train_df['user_id']==124][['timestamp', 'user_id', 'task_container_id', 'timestamp_diff1', 'timestamp_diff2']].head(7)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# part_avg\nCumulative average per \"part\".\n# cluster_avg\nCumulative average per \"cluster_id\"."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['lag'] = train_df.groupby('user_id')[target].shift()\ncum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])\ntrain_df['cumcount_u'] = cum['cumcount']\ntrain_df['user_correctness'] = cum['cumsum'] / cum['cumcount']\ntrain_df.drop(columns=['lag'], inplace=True)\n\ntrain_df['lag'] = train_df.groupby(['user_id', 'part'])[target].shift()\ncum = train_df.groupby(['user_id', 'part'])['lag'].agg(['cumsum', 'cumcount'])\ntrain_df['cumcount_p'] = cum['cumcount']\ntrain_df['part_avg'] = cum['cumsum'] / cum['cumcount']\ntrain_df.drop(columns=['lag'], inplace=True)\n\ntrain_df['lag'] = train_df.groupby(['user_id', 'cluster_id'])[target].shift()\ncum = train_df.groupby(['user_id', 'cluster_id'])['lag'].agg(['cumsum', 'cumcount'])\ntrain_df['cumcount_cl'] = cum['cumcount']\ntrain_df['cluster_avg'] = cum['cumsum'] / cum['cumcount']\ntrain_df.drop(columns=['lag'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Share task_container_id.\ndf_ = train_df.groupby(['user_id', 'task_container_id']).head(1)[['user_id', 'task_container_id', 'user_correctness', 'cumcount_u', 'part_avg', 'cumcount_p']]\ntrain_df.drop(columns=['user_correctness', 'part_avg', 'cumcount_u', 'cumcount_p'], inplace=True)\ntrain_df = pd.merge(train_df, df_, on=['user_id', 'task_container_id'], how='left')\n\ndf_ = train_df.groupby(['user_id', 'task_container_id', 'cluster_id']).head(1)[['user_id', 'task_container_id', 'cluster_id', 'cluster_avg', 'cumcount_cl']]\ntrain_df.drop(columns=['cluster_avg', 'cumcount_cl'], inplace=True)\ntrain_df = pd.merge(train_df, df_, on=['user_id', 'task_container_id', 'cluster_id'], how='left')\n\ndel cum, df_\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[train_df['user_id']==124][['user_id', 'task_container_id', target, 'part', 'part_avg', 'cluster_id', 'cluster_avg']].head(7)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# preprocessing (2/2)"},{"metadata":{"trusted":true},"cell_type":"code","source":"part_null_data = train_df[train_df['part_avg'].isna()].groupby('part')[target].mean()\ncluster_null_data = train_df.groupby('cluster_id')[target].mean()\n\n# part_null_data.to_pickle('riiid_pre_data//part_null_data.pickle')\n# cluster_null_data.to_pickle('riiid_pre_data//cluster_null_data.pickle')\n\n# part_null_data = pd.read_pickle('riiid_pre_data//part_null_data.pickle')\n# cluster_null_data = pd.read_pickle('riiid_pre_data//cluster_null_data.pickle')\n\ncontent_agg = train_df.groupby('content_id')[target].agg(['sum', 'count'])\n\n# content_agg.to_pickle('riiid_pre_data/content_agg.pickle')\n\n# content_agg = pd.read_pickle('riiid_pre_data/content_agg.pickle')\ntrain_df['content_count'] = train_df['content_id'].map(content_agg['count']).astype('int32')\ntrain_df['content_mean'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'])\n\ntrain_df[\"attempt\"] = train_df.groupby([\"user_id\",\"content_id\"])[target].cumcount()\ntrain_df['attempt'] = np.where(train_df['attempt']>6, 6, train_df['attempt'])\n\ntrain_df['prior_question_had_explanation'].fillna(False, inplace=True)\ntrain_df['user_correctness'].fillna(0.68, inplace=True)\ntrain_df['prior_question_had_explanation'] = train_df['prior_question_had_explanation'].astype('bool')\ntrain_df.loc[train_df['part_avg'].isna(), 'part_avg'] = train_df[train_df['part_avg'].isna()]['part'].map(part_null_data)\ntrain_df.loc[train_df['cluster_avg'].isna(), 'cluster_avg'] = train_df[train_df['cluster_avg'].isna()]['cluster_id'].map(cluster_null_data)\ntrain_df['timestamp_diff1'].fillna(25572., inplace=True)\ntrain_df['timestamp_diff2'].fillna(53309., inplace=True)\ntrain_df['prior_question_elapsed_time'].fillna(22000., inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# data for state"},{"metadata":{"trusted":true},"cell_type":"code","source":"# data = pd.read_csv('riiid-test-answer-prediction/train.csv',\n#                    usecols=[2, 3, 7],\n#                    dtype=data_types_dict\n#                   )\n# questions_df_ = pd.read_csv(\n#     'riiid-test-answer-prediction/questions.csv', \n#     usecols=[0, 3],\n#     dtype={'question_id': 'int16', 'part': 'int8'}\n# )\n# data = data[data[target] != -1].reset_index(drop=True)\n# data = pd.merge(data, questions_df_, left_on='content_id', right_on='question_id', how='left')\n# data.drop(columns=['question_id'], inplace=True)\n\n# data.to_pickle('riiid_pre_data/state_data.pickle')\n\n# del data, questions_df_\n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# make dict"},{"metadata":{"trusted":true},"cell_type":"code","source":"# from collections import defaultdict\n\n# train_df = pd.read_pickle('my_data/train_df.pickle')\n\n# user_dict_sum = train_df.groupby('user_id')[target].agg('sum').astype('uint16').to_dict(defaultdict(int))\n# user_dict_count = train_df.groupby('user_id')[target].agg('count').astype('uint16').to_dict(defaultdict(int))\n\n# part_dict_sum = train_df.groupby(['user_id', 'part'])[target].agg('sum').astype('uint16').to_dict(defaultdict(int))\n# part_dict_count = train_df.groupby(['user_id', 'part'])[target].agg('count').astype('uint16').to_dict(defaultdict(int))\n\n# cluster_dict_sum = train_df.groupby(['user_id', 'cluster_id'])[target].agg('sum').astype('uint16').to_dict(defaultdict(int))\n# cluster_dict_count = train_df.groupby(['user_id', 'cluster_id'])[target].agg('count').astype('uint16').to_dict(defaultdict(int))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# joblib.dump(user_dict_sum, \"dict_data/user_dict_sum.pkl.zip\")\n# joblib.dump(user_dict_count, \"dict_data/user_dict_count.pkl.zip\")\n# joblib.dump(part_dict_sum, \"dict_data/part_dict_sum.pkl.zip\")\n# joblib.dump(part_dict_count, \"dict_data/part_dict_count.pkl.zip\")\n# joblib.dump(cluster_dict_sum, \"dict_data/cluster_dict_sum.pkl.zip\")\n# joblib.dump(cluster_dict_count, \"dict_data/cluster_dict_count.pkl.zip\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# cluster0_avg, cluster1_avg, cluster1_avg\nConvert \"cluster_avg\" to the respective column."},{"metadata":{"trusted":true},"cell_type":"code","source":"user_idx = train_df[train_df['cumcount_u']==0].index\nfor cluster_id in range(0, 3):\n    df = train_df[train_df['cluster_id']==cluster_id].groupby('user_id')[target].agg(['cumsum', 'cumcount'])\n    df['cumcount'] += 1\n    df['mean'] = df['cumsum'] / df['cumcount']\n    idx = df.index\n    ar = np.empty(len(train_df))\n    ar[:] = np.nan\n    ar[idx] = df.loc[idx, 'mean']\n    train_df[f'cluster{cluster_id}_avg'] = ar\n    train_df[f'cluster{cluster_id}_avg'] = train_df.groupby('user_id')[f'cluster{cluster_id}_avg'].shift()\n    train_df.loc[user_idx, f'cluster{cluster_id}_avg'] = cluster_null_data[cluster_id]\n    train_df[f'cluster{cluster_id}_avg'].fillna(method='ffill', inplace=True)\n\ndf = train_df.groupby(['user_id', 'task_container_id']).head(1)[['user_id', 'task_container_id', 'cluster0_avg', 'cluster1_avg', 'cluster2_avg']]\ntrain_df.drop(columns=['cluster0_avg', 'cluster1_avg', 'cluster2_avg'], inplace=True)\ntrain_df = pd.merge(train_df, df, on=['user_id', 'task_container_id'], how='left')\n\ndel df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[train_df['user_id']==124][['user_id', 'task_container_id', target, 'cluster_id', 'cluster0_avg', 'cluster1_avg', 'cluster2_avg']].head(7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display(train_df[[target, 'cluster0_avg', 'cluster1_avg', 'cluster2_avg']].corr())\nprint('Average of cluster1_avg when cluster_id is 0: ', train_df[train_df['cluster_id']==0]['cluster1_avg'].mean())\nprint('Average of cluster1_avg when cluster_id is 0 and incorrect answer: ', train_df[(train_df['cluster_id']==0)&(train_df[target]==0)]['cluster1_avg'].mean())\nprint('Average of cluster1_avg when cluster_id is 0 and correct answer: ', train_df[(train_df['cluster_id']==0)&(train_df[target]==1)]['cluster1_avg'].mean())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# target_lag"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['target_lag'] = train_df.groupby('user_id')[target].shift()\ntrain_df['target_lag'].fillna(1, inplace=True)\n\ndf_ = train_df.groupby(['user_id', 'task_container_id']).head(1)[['user_id', 'task_container_id', 'target_lag']]\ntrain_df.drop(columns=['target_lag'], inplace=True)\ntrain_df = pd.merge(train_df, df_, on=['user_id', 'task_container_id'], how='left')\n\n# lag_dict = train_df.groupby('user_id').tail(1)[['user_id', target]].set_index('user_id')[target].astype('uint8').to_dict()\n\n# joblib.dump(lag_dict, \"dict_data/lag_dict.pkl.zip\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# prior_tag (2/2)"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_ = train_df.groupby(['user_id', 'task_container_id']).head(1)[['user_id', 'task_container_id', 'prior_tag']]\ntrain_df.drop(columns=['prior_tag'], inplace=True)\ntrain_df = pd.merge(train_df, df_, on=['user_id', 'task_container_id'], how='left')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# questions_df"},{"metadata":{"trusted":true},"cell_type":"code","source":"# content_agg['mean'] = content_agg['sum'] / content_agg['count']\n\n# questions_df['content_mean'] = questions_df['question_id'].map(content_agg['mean'])\n# questions_df['content_count'] = questions_df['question_id'].map(content_agg['count'])\n\n# questions_df = questions_df.astype({'question_id': 'int16', 'part': 'int8', 'task_num': 'int8', 'cluster_id': 'int8', 'content_mean': 'float32', 'content_count': 'int32'})\n\n# questions_df.to_pickle('riiid_pre_data/questions_df.pickle')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# user_rating\nAverage difference between \"answered_correctly\" and \"content_mean\"."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['user_rating'] = train_df[target] - train_df['content_mean']\ntrain_df['user_rating'] = train_df.groupby('user_id')['user_rating'].shift()\ntrain_df['user_rating'] = train_df.groupby('user_id')['user_rating'].cumsum()\n\ndf_ = train_df.groupby(['user_id', 'task_container_id']).head(1)[['user_id', 'task_container_id', 'user_rating']]\ntrain_df.drop(columns=['user_rating'], inplace=True)\ntrain_df = pd.merge(train_df, df_, on=['user_id', 'task_container_id'], how='left')\n\ntrain_df['user_rating'] = train_df['user_rating'] / train_df['cumcount_u']\n\ntrain_df['user_rating'].fillna(0, inplace=True)\n\n# content_mean_sum_dict = train_df.groupby('user_id')['content_mean'].agg('sum').astype('float32').to_dict(defaultdict(int))\n\n# joblib.dump(content_mean_sum_dict, \"dict_data/content_mean_sum_dict.pkl.zip\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[train_df['user_id']==124][['user_id', 'task_container_id', target, 'content_mean', 'user_rating']].head(7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.groupby(target)['user_rating'].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist([train_df[train_df[target]==0]['user_rating'].sample(10000),\n          train_df[train_df[target]==1]['user_rating'].sample(10000)], label=['0', '1'])\nplt.legend();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Students who answer difficult questions correctly are more versatile."},{"metadata":{},"cell_type":"markdown","source":"# upper limit"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['cumcount_u'] = np.where(train_df['cumcount_u']>7500, 7500, train_df['cumcount_u'])\ntrain_df['cumcount_p'] = np.where(train_df['cumcount_p']>7500, 7500, train_df['cumcount_p'])\ntrain_df['cumcount_cl'] = np.where(train_df['cumcount_cl']>7500, 7500, train_df['cumcount_cl'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# time_mean_diff\nThe difference between the past \"timestamp_diff1\" and the current one.The upper limit is set to 100 seconds."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['time_adm'] = np.where(train_df['timestamp_diff1']>100000, 100000, train_df['timestamp_diff1'])\n\ntrain_df['time_mean'] = train_df.groupby('user_id')['time_adm'].cumsum() / (train_df.groupby('user_id')[target].cumcount() + 1)\ntrain_df['time_mean'] = train_df.groupby('user_id')['time_mean'].shift()\n\ndf_ = train_df.groupby(['user_id', 'task_container_id']).head(1)[['user_id', 'task_container_id', 'time_mean']]\ntrain_df.drop(columns=['time_mean'], inplace=True)\ntrain_df = pd.merge(train_df, df_, on=['user_id', 'task_container_id'], how='left')\n\ntrain_df['time_mean'].fillna(25572., inplace=True)\ntrain_df['time_mean_diff'] = train_df['time_adm'] - train_df['time_mean']\n\n# time_adm_dict = train_df.groupby('user_id')['time_adm'].agg('sum').to_dict(defaultdict(int))\n\n# joblib.dump(time_adm_dict, \"dict_data/time_adm_dict.pkl.zip\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[train_df['user_id']==124][['user_id', 'task_container_id', target, 'timestamp_diff1', 'time_mean_diff']].head(7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist([train_df[train_df[target]==0]['time_mean_diff'].sample(10000),\n          train_df[train_df[target]==1]['time_mean_diff'].sample(10000)], label=['0', '1'])\nplt.legend();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Take more time than usual for problems you are not confident in."},{"metadata":{"trusted":true},"cell_type":"code","source":"features = [\n    'content_id',\n    'prior_question_elapsed_time',\n    'prior_question_had_explanation',\n    'user_correctness',\n    'content_count',\n    'part',\n    'content_mean',\n    'cumcount_u',\n    'cumcount_p',\n    'attempt',\n    'part_avg',\n    'timestamp_diff1',\n    'timestamp_diff2',\n    'cluster_id',\n    'cumcount_cl',\n    'target_lag',\n    'cluster0_avg',\n    'cluster1_avg',\n    'cluster2_avg',\n    'prior_tag',\n    'task_num',\n    'user_rating',\n    'time_mean_diff',\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.astype(features_dtypes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[features].head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# save"},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_df.to_pickle('my_data/train_df.pickle')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Thank you for a great competition...!!**"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}