{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport matplotlib.style as style\nstyle.use('fivethirtyeight')\nimport seaborn as sns\nimport os\nfrom matplotlib.ticker import FuncFormatter\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input/riiid-test-answer-prediction'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#this clears everything loaded in RAM, including the libraries\n%reset -f","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport riiideducation\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport matplotlib.style as style\nstyle.use('fivethirtyeight')\nimport seaborn as sns\nimport os\nimport lightgbm as lgb\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.preprocessing import LabelEncoder\nimport gc\nimport sys\npd.set_option('display.max_rows', None)\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input/riiid-test-answer-prediction'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dir_path = '/kaggle/input/riiid-test-answer-prediction/'\nfile_train = 'train.csv'\nfile_questions = 'questions.csv'\nfile_lectures = 'lectures.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nrows = 100 * 10000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\n                    dir_path + file_train, \n                    nrows=nrows, \n                    usecols=['row_id', 'timestamp', 'user_id', 'content_id', \n                             'content_type_id', 'task_container_id', 'answered_correctly',\n                            'prior_question_elapsed_time','prior_question_had_explanation'],\n                    dtype={\n                            'row_id': 'int64',\n                            'timestamp': 'int64',\n                            'user_id': 'int32',\n                            'content_id': 'int16',\n                            'content_type_id': 'int8',\n                            'task_container_id': 'int8',\n                            'answered_correctly': 'int8',\n                            'prior_question_elapsed_time': 'float32',\n                            'prior_question_had_explanation': 'str'\n                        }\n                   )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lectures = pd.read_csv(\n                       dir_path + file_lectures, \n                       usecols=['lecture_id','tag','part','type_of'], \n                       nrows=nrows,\n                       dtype={\n                           'lecture_id': 'int16',\n                           'tag': 'int16',\n                           'part': 'int8',\n                           'type_of': 'str'\n                       }\n                    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(lectures)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions = pd.read_csv(\n                        dir_path + file_questions, \n                        nrows=nrows,\n                        usecols=['question_id','bundle_id','part','tags'], \n                        dtype={\n                           'question_id': 'int16',\n                           'bundle_id': 'int16',\n                           'part': 'int8',\n                           'tags': 'str'\n                       }\n                    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(questions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['prior_question_had_explanation'] = train['prior_question_had_explanation'].map({'True':1,'False':0}).fillna(-1).astype(np.int8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(train['prior_question_had_explanation'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lectures['type_of'] = lectures['type_of'].map({'concept':0, 'intention':1, 'solving question':2, 'starter':3}).fillna(-1).astype(np.int8)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions['tags'] = questions['tags'].map(lambda x:len(str(x).split(' ')))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"max_num = 3000\ntrain = train.groupby(['user_id']).tail(max_num)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_lectures = train[train['content_type_id']==1]\ntrain_questions = train[train['content_type_id']==0]\ndel train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 关联数据\n# 将train_lectures中的用户数据与lectures中的课程数据关联起来\ntrain_lectures_info = pd.merge(\n        left=train_lectures,\n        right=lectures,\n        how='left',\n        left_on='content_id',\n        right_on='lecture_id'\n        )\n\n# 将train_questions中的用户数据与questions中的课程数据关联起来\ntrain_questions_info = pd.merge(\n        left=train_questions,\n        right=questions,\n        how='left',\n        left_on='content_id',\n        right_on='question_id'\n        )\n\ndel train_lectures\ndel train_questions\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_lecture_basic_features__user(train_lectures_info):\n    gb_columns = ['user_id']\n    gb_suffixes = 'lecture_'+'_'.join(gb_columns)\n    \n    agg_func = {\n        'lecture_id': [np.size],\n        'task_container_id': [lambda x: len(set(x))],\n        'tag': [lambda x: len(set(x))],\n\n        # part 展开\n        'part': [lambda x: len(set(x))],\n\n        # type_of 展开\n        'type_of': [lambda x: len(set(x))],\n    }\n    columns = [\n           gb_suffixes+'_size_lecture_id', \n           gb_suffixes+'_unique_task_container_id',\n           gb_suffixes+'_unique_tag',\n           gb_suffixes+'_unique_part',\n           gb_suffixes+'_unique_type_of'\n          ]  \n    train_lectures_info__user_f = train_lectures_info.                                groupby(gb_columns).                                agg(agg_func).                                reset_index()\n    \n    train_lectures_info__user_f.columns = gb_columns + columns\n    return train_lectures_info__user_f\n\ndef get_lecture_basic_features__user_tag(train_lectures_info):\n    gb_columns = ['user_id','tag']\n    gb_suffixes = 'lecture_'+'_'.join(gb_columns)\n    agg_func = {\n        'lecture_id': [np.size],\n        'task_container_id': [lambda x: len(set(x))],\n        'tag': [lambda x: len(set(x))],\n\n        # part 展开\n        'part': [lambda x: len(set(x))],\n    }\n    columns = [\n               gb_suffixes+'_size_lecture_id', \n               gb_suffixes+'_unique_task_container_id',\n               gb_suffixes+'_unique_tag',\n               gb_suffixes+'_unique_part'\n              ]    \n    train_lectures_info__user_tag_f = train_lectures_info.                                    groupby(gb_columns).                                    agg(agg_func).                                    reset_index()\n    train_lectures_info__user_tag_f.columns = gb_columns + columns    \n    return train_lectures_info__user_tag_f\n\n# 问答类函数\ndef get_questions_basic_features__user(train_questions_info):\n    gb_columns = ['user_id']\n    gb_suffixes = 'question_'+'_'.join(gb_columns)\n    agg_func = {\n        'answered_correctly': [np.mean,np.sum,np.std],\n\n        'question_id': [np.size],\n        'task_container_id': [lambda x: len(set(x))],\n\n        'prior_question_elapsed_time': [np.mean,np.max,np.min],\n\n        'prior_question_had_explanation': [lambda x: len(set(x))],\n\n        'bundle_id': [lambda x: len(set(x))],\n\n        # part 展开\n        'part': [lambda x: len(set(x))],\n        'tags': [lambda x: len(set(x))],\n    }\n    columns = [\n               gb_suffixes+'_answered_correctly_mean',\n               gb_suffixes+'_answered_correctly_max',\n               gb_suffixes+'_answered_correctly_min',\n\n               gb_suffixes+'_size_question_id', \n               gb_suffixes+'_unique_task_container_id',\n               gb_suffixes+'_prior_question_elapsed_time_mean',\n               gb_suffixes+'_prior_question_elapsed_time_max',\n               gb_suffixes+'_prior_question_elapsed_time_min',\n\n               gb_suffixes+'_unique_prior_question_had_explanation',\n\n               gb_suffixes+'_unique_bundle_id',\n               gb_suffixes+'_unique_part',\n               gb_suffixes+'_unique_tags',\n              ]\n    train_questions_info__user_f = train_questions_info.                                    groupby(gb_columns).                                    agg(agg_func).                                    reset_index()\n    train_questions_info__user_f.columns = gb_columns + columns    \n\n    return train_questions_info__user_f\n\ndef get_questions_basic_features__user_part(train_questions_info):\n    gb_columns = ['user_id','part']\n    gb_suffixes = 'question_'+'_'.join(gb_columns)\n    agg_func = {\n        'answered_correctly': [np.mean,np.sum,np.std],\n\n        'question_id': [np.size],\n        'task_container_id': [lambda x: len(set(x))],\n\n        'prior_question_elapsed_time': [np.mean,np.max,np.min],\n\n        'prior_question_had_explanation': [lambda x: len(set(x))],\n\n        'bundle_id': [lambda x: len(set(x))],\n\n        # part 展开\n        'part': [lambda x: len(set(x))],\n        'tags': [lambda x: len(set(x))],\n    }\n    columns = [\n               gb_suffixes+'_answered_correctly_mean',\n               gb_suffixes+'_answered_correctly_max',\n               gb_suffixes+'_answered_correctly_min',\n\n               gb_suffixes+'_size_question_id', \n               gb_suffixes+'_unique_task_container_id',\n               gb_suffixes+'_prior_question_elapsed_time_mean',\n               gb_suffixes+'_prior_question_elapsed_time_max',\n               gb_suffixes+'_prior_question_elapsed_time_min',\n\n               gb_suffixes+'_unique_prior_question_had_explanation',\n\n               gb_suffixes+'_unique_bundle_id',\n               gb_suffixes+'_unique_part',\n               gb_suffixes+'_unique_tags',\n              ]    \n    train_questions_info__user_part_f = train_questions_info.                                    groupby(gb_columns).                                    agg(agg_func).                                    reset_index()\n    train_questions_info__user_part_f.columns = gb_columns + columns    \n\n    return train_questions_info__user_part_f\n\ndef get_questions_basic_features__content(train_questions_info):\n    gb_columns = ['content_id']\n    gb_suffixes = 'question_'+'_'.join(gb_columns)\n    agg_func = {\n        'answered_correctly': [np.mean,np.sum,np.std],\n\n        'user_id': [np.size],\n\n        'prior_question_elapsed_time': [np.mean,np.max,np.min],\n\n        'prior_question_had_explanation': [lambda x: len(set(x))],\n    }\n    columns = [\n               gb_suffixes+'_answered_correctly_mean',\n               gb_suffixes+'_answered_correctly_max',\n               gb_suffixes+'_answered_correctly_min',\n\n               gb_suffixes+'_size_user_id', \n               gb_suffixes+'_prior_question_elapsed_time_mean',\n               gb_suffixes+'_prior_question_elapsed_time_max',\n               gb_suffixes+'_prior_question_elapsed_time_min',\n\n               gb_suffixes+'_unique_prior_question_had_explanation',\n              ]    \n    \n    train_questions_info__user_content_f = train_questions_info.                                    groupby(gb_columns).                                    agg(agg_func).                                    reset_index()\n    train_questions_info__user_content_f.columns = gb_columns + columns\n    \n    return train_questions_info__user_content_f","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 特征提取\ntest_lectures_info__user_f = get_lecture_basic_features__user(train_lectures_info)\n# test_lectures_info__user_tag_f = get_lecture_basic_features__user_tag(train_lectures_info)\ntest_questions_info__user_f = get_questions_basic_features__user(train_questions_info)\n# test_questions_info__user_part_f = get_questions_basic_features__user_part(train_questions_info)\ntest_questions_info__user_content_f = get_questions_basic_features__content(train_questions_info)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_data = pd.DataFrame()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n'''\n这里在做的事是：\n比如一个 用户有5条数据，我们将第5条选取出来，并根据前4条数据按照user_id或者content_id进行分组\n分组后提取特征，将这些特征认为是第5条记录所对应的特征\n对上面的过程进行重复，于是可以得到从第五条记录的特征到第一条记录的特征\n当然，如果记录太少，得到的特征不能用来训练模型\n'''\nfor i in range(3):\n    \n    # 获取训练标签数据(每个用户的最后一条记录)\n    last_records = train_questions_info.drop_duplicates('user_id', keep='last')\n    \n    # 获取训练标签以前的数据\n    # zip函数将对应的数据组成元组\n    # dict函数将元组转化成字典\n    # 得到的是user_id:row_id，这里的row_id是每个用户的最后一条数据\n    map__last_records__user_row = dict(zip(last_records['user_id'],last_records['row_id']))\n    \n    # 如果train_questions_info和train_lectures_info中user_id为某一值\n    # 则新建字段filter_row，其值为该user的最后一条数据的row_id\n    train_questions_info['filter_row'] = train_questions_info['user_id'].map(map__last_records__user_row)\n    train_lectures_info['filter_row'] = train_lectures_info['user_id'].map(map__last_records__user_row)\n    \n    # 如果train_questions_info和train_lectures_info中每一条记录，row_id<filter_row\n    # 则留下这条记录，其余记录删除\n    # 实际上就是删除了每个用户的最后一条记录\n    train_questions_info = train_questions_info[train_questions_info['row_id']<train_questions_info['filter_row']]\n    train_lectures_info = train_lectures_info[train_lectures_info['row_id']<train_lectures_info['filter_row']]\n    \n    # 对删除了最后一条记录的新的train_questions_info和train_lectures_info再一次提取特征\n    train_lectures_info__user_f = get_lecture_basic_features__user(train_lectures_info)\n    # train_lectures_info__user_tag_f = get_lecture_basic_features__user_tag(train_lectures_info)\n    train_questions_info__user_f = get_questions_basic_features__user(train_questions_info)\n    # train_questions_info__user_part_f = get_questions_basic_features__user_part(train_questions_info)\n    train_questions_info__user_content_f = get_questions_basic_features__content(train_questions_info)\n    \n    # 将提取到的特征与用户的最后一条记录合并\n    last_records = last_records.merge(train_lectures_info__user_f,on=['user_id'],how='left')\n    last_records = last_records.merge(train_questions_info__user_f,on=['user_id'],how='left')\n    last_records = last_records.merge(train_questions_info__user_content_f,on=['content_id'],how='left')\n    \n    # 将提取到的特征加入valid_data\n    valid_data = valid_data.append(last_records)\n    print(len(valid_data))\n\n\n# In[105]:\n\n\n# 训练数据\ntrain_data = pd.DataFrame()\n\n\n'''\n对训练数据也进行相同的特征提取\n'''\nfor i in range(10):\n    \n    # 获取训练标签数据\n    last_records = train_questions_info.drop_duplicates('user_id', keep='last')\n    \n    # 获取训练标签以前的数据\n    map__last_records__user_row = dict(zip(last_records['user_id'],last_records['row_id']))\n    \n    train_questions_info['filter_row'] = train_questions_info['user_id'].map(map__last_records__user_row)\n    train_lectures_info['filter_row'] = train_lectures_info['user_id'].map(map__last_records__user_row)\n\n    train_questions_info = train_questions_info[train_questions_info['row_id']<train_questions_info['filter_row']]\n    train_lectures_info = train_lectures_info[train_lectures_info['row_id']<train_lectures_info['filter_row']]\n    \n    # 获取特征\n    train_lectures_info__user_f = get_lecture_basic_features__user(train_lectures_info)\n    # train_lectures_info__user_tag_f = get_lecture_basic_features__user_tag(train_lectures_info)\n    train_questions_info__user_f = get_questions_basic_features__user(train_questions_info)\n    # train_questions_info__user_part_f = get_questions_basic_features__user_part(train_questions_info)\n    train_questions_info__user_content_f = get_questions_basic_features__content(train_questions_info)\n\n    last_records = last_records.merge(train_lectures_info__user_f,on=['user_id'],how='left')\n    last_records = last_records.merge(train_questions_info__user_f,on=['user_id'],how='left')\n    last_records = last_records.merge(train_questions_info__user_content_f,on=['content_id'],how='left')\n    \n    # 特征加入训练集\n    train_data = train_data.append(last_records)\n    print(len(train_data))\n\n\n# In[106]:\n\n\n# 删除不需要的字段\nremove_columns = ['user_id','row_id','content_type_id','user_answer','answered_correctly','filter_row']\nfeatures_columns = [c for c in train_data.columns if c not in remove_columns]\n\n# 得到最终验证集\nX_test, y_test = valid_data[features_columns].values, valid_data['answered_correctly'].values\n\n# 得到最终训练集\nX_train, y_train = train_data[features_columns].values, train_data['answered_correctly'].values\n\n# 设置lgb模型训练参数\nparams = {\n    'task': 'train',\n    'boosting_type': 'gbdt',\n    'objective': 'binary',\n    'metric': 'auc',\n    'num_leaves': 9,\n    'learning_rate': 0.3,\n    'feature_fraction_seed': 2,\n    'feature_fraction': 0.9,\n    'bagging_fraction': 0.8,\n    'bagging_freq': 5,\n    'min_data': 20,\n    'min_hessian': 1,\n    'verbose': -1,\n    'silent': 0\n    }\n\n# 设置lgb训练参数\nlgb_train = lgb.Dataset(X_train, y_train)\n\n# 设置lgb模型测试参数\nlgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)\n\n# 开始训练和测试\n# lgb_train就是输入之前设置的训练集\n# valid_sets就是输入之前的验证集\ngbm = lgb.train(\n            params,\n            lgb_train,\n            num_boost_round=10000,\n            valid_sets=lgb_eval,\n            early_stopping_rounds=20\n            )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\niter_test = env.iter_test()\n\nfor (test_df, sample_prediction_df) in iter_test:\n    \n    test_questions = test_df[test_df['content_type_id']==0]\n    test_questions_info = pd.merge(\n            left=test_questions,\n            right=questions,\n            how='left',\n            left_on='content_id',\n            right_on='question_id'\n            )\n    \n    test_questions_info['prior_question_had_explanation'] = test_questions_info['prior_question_had_explanation'].map({'True':1,'False':0}).fillna(-1).astype(np.int8)\n\n    test_questions_info = test_questions_info.merge(test_lectures_info__user_f,on=['user_id'],how='left')\n    test_questions_info = test_questions_info.merge(test_questions_info__user_f,on=['user_id'],how='left')\n    test_questions_info = test_questions_info.merge(test_questions_info__user_content_f,on=['content_id'],how='left')\n        \n    # 修改\n    #remove_columns = ['user_id','row_id','content_type_id','user_answer','answered_correctly','filter_row']\n    #features_columns = [c for c in train_data.columns if c not in remove_columns]\n\n\n    X_test = test_questions_info[features_columns].values\n    \n    test_questions_info['answered_correctly'] =  gbm.predict(X_test)\n    \n    env.predict(test_questions_info.loc[test_questions_info['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}