{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport riiideducation\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom scipy.stats import skew\nfrom collections import defaultdict, Counter\n# import cudf\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\npd.set_option('display.max_rows', 100)\npd.set_option('display.max_columns', 100)\nis_test=False\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%%time\nsmp = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/example_sample_submission.csv')\nex_test = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/example_test.csv')\nquestions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')\nlectures = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv')\ntrain = pd.read_pickle('/kaggle/input/riiid-data/train.pkl')\ntrain = train.loc[:3e+07]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if is_test:\n    train = train.loc[:10000]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocessing"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# train 데이터 > type 필터링 & merging\ntrain = train[train['content_type_id']==0]\nquestions_ = questions.copy()\nquestions_['content_type_id'] = 0\nquestions_.rename(columns = {'question_id': 'content_id'}, inplace = True)\nquestions_.fillna('-1', inplace=True) # na 딱 1개있다 \nquestions_['tags'] = questions_['tags'].apply(lambda x: int(x.split()[0]))\ntrain = train.merge(questions_, how='left', on=['content_type_id', 'content_id'])\n\n\n# # Dictionary: user_id별 count, answered_correctly sum, prior_question_elapsed_time sum\nvalues = train.groupby('user_id').agg({'user_id': len,\n                                     'answered_correctly':np.sum,\n                                     'prior_question_elapsed_time':np.sum})\nuserid_dic = defaultdict(lambda:[0,0,0], zip(values.index, np.array(values)))\n\n\n# Dictionary: content_id별 count, answered_correctly sum, prior_question_elapsed_time sum\nvalues = train.groupby('content_id').agg({'content_id': len,\n                                         'answered_correctly':np.sum,\n                                         'prior_question_elapsed_time': [np.sum, np.min]}) # min 추가 \ncontentid_dic = defaultdict(lambda:[0,0,0,0], zip(values.index, np.array(values)))\n\n\n# Dictionary: content_id별 count, answered_correctly sum, prior_question_elapsed_time sum\nvalues = train.groupby('tags').agg({'tags': len,\n                                     'answered_correctly':np.sum,\n                                     'prior_question_elapsed_time':np.sum})\ntag_dic = defaultdict(lambda:[0,0,0], zip(values.index, np.array(values)))\n\n\n# Dictionary: content_id별 count, answered_correctly sum, prior_question_elapsed_time sum\nvalues = train.groupby('part').agg({'part': len,\n                                     'answered_correctly':np.sum,\n                                     'prior_question_elapsed_time':np.sum})\npart_dic = defaultdict(lambda:[0,0,0], zip(values.index, np.array(values)))\n\n# Dictionary: task_container_id별 count, answered_correctly sum, prior_question_elapsed_time sum\nvalues = train.groupby('task_container_id').agg({'task_container_id': len,\n                                                 'answered_correctly':np.sum,\n                                                 'prior_question_elapsed_time':np.sum})\ntask_dic = defaultdict(lambda:[0,0,0], zip(values.index, np.array(values)))\n\n\n\n# NA\ntrain.fillna(0, inplace=True)\n    \n    \n# Data Type\ntrain['prior_question_had_explanation'] = train['prior_question_had_explanation'].astype(np.int8)\n\n\n# user_id & content_id 별 answer mean / time mean\ntrain['userid_answer_mean'] = train['user_id'].apply(lambda x: userid_dic[x][1]) / train['user_id'].apply(lambda x: userid_dic[x][0])\ntrain['userid_time_mean'] = train['user_id'].apply(lambda x: userid_dic[x][2]) / train['user_id'].apply(lambda x: userid_dic[x][0])\ntrain['userid_cnt'] = train['user_id'].apply(lambda x:userid_dic[x][0])\n\ntrain['contentid_answer_mean'] = train['content_id'].apply(lambda x: contentid_dic[x][1]) / train['content_id'].apply(lambda x: contentid_dic[x][0])\ntrain['contentid_time_mean'] = train['content_id'].apply(lambda x: contentid_dic[x][2]) / train['content_id'].apply(lambda x: contentid_dic[x][0])\ntrain['contentid_time_min'] = train['content_id'].apply(lambda x: contentid_dic[x][3])\ntrain['contentid_cnt'] = train['content_id'].apply(lambda x:contentid_dic[x][0])\n\ntrain['tag_answer_mean'] = train['tags'].apply(lambda x: tag_dic[x][1]) / train['tags'].apply(lambda x: tag_dic[x][0])\ntrain['tag_time_mean'] = train['tags'].apply(lambda x: tag_dic[x][2]) / train['tags'].apply(lambda x: tag_dic[x][0])\ntrain['tag_cnt'] = train['tags'].apply(lambda x:tag_dic[x][0])\n\ntrain['part_answer_mean'] = train['part'].apply(lambda x: part_dic[x][1]) / train['part'].apply(lambda x: part_dic[x][0])\ntrain['part_time_mean'] = train['part'].apply(lambda x: part_dic[x][2]) / train['part'].apply(lambda x: part_dic[x][0])\n\n# train['task_answer_mean'] = train['task_container_id'].apply(lambda x: task_dic[x][1]) / train['task_container_id'].apply(lambda x: task_dic[x][0])\n# train['task_time_mean'] = train['task_container_id'].apply(lambda x: task_dic[x][2]) / train['task_container_id'].apply(lambda x: task_dic[x][0])\n\nprint(train.isnull().sum()/len(train))\n\n\n# 정답통계량, merge 시 발생한 NA 처리\ntrain.fillna(0, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not is_test:\n    train_smp = train.sample(n=13000000, random_state = 123)\n    train = train_smp[:10000000]\n    val = train_smp[10000000:]\n    del train_smp\nelse:\n    val = train[7000:]\n    train = train[:7000]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Modeling"},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\nimport lightgbm as lgb\n\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"col_del = ['row_id', 'user_answer', 'answered_correctly', 'user_id', 'content_type_id']\n\nX = train.columns[~pd.Series(train.columns).isin(col_del)]\ny = 'answered_correctly'\nX","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params = {'objective': 'binary',\n          'metric': 'auc',\n          'learning_rate': 0.1, #default\n          \"boosting_type\": \"gbdt\", #default\n#           'device': 'gpu',\n#           'gpu_platform_id': 0,\n#           'gpu_device_id': 0,\n          'seed': 42\n         }\n\nlgb_train = lgb.Dataset(train[X], train[y], categorical_feature = [])\nlgb_eval = lgb.Dataset(val[X], val[y], categorical_feature = [])\n\nmodel = lgb.train(\n    params, lgb_train,\n    valid_sets=[lgb_train, lgb_eval],\n    verbose_eval=50,\n    num_boost_round=10000,\n    early_stopping_rounds=8\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb.plot_importance(model)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()\niter_test = env.iter_test()\ni=0\nprior_test = None\n\n\nif not is_test:\n    for (test, sample_prediction_df) in iter_test:\n        test['answered_correctly']=0\n        if prior_test is None:\n            # filtering & merging\n            test = test[test['content_type_id']==0]\n            test = test.merge(questions_, how='left', on=['content_type_id', 'content_id'])\n\n            # NA\n            test.fillna(0, inplace=True)\n\n            # Data Type\n            test['prior_question_had_explanation'] = test['prior_question_had_explanation'].astype(np.int8)\n        \n        if prior_test is not None:            \n            prior_answers = np.array(eval(test.iloc[0]['prior_group_answers_correct']))\n            prior_answers = prior_answers[prior_answers!=-1]\n            prior_test['answered_correctly'] = prior_answers\n            \n            # filtering & merging\n            test = test[test['content_type_id']==0]\n            test = test.merge(questions_, how='left', on=['content_type_id', 'content_id'])\n\n            # NA\n            test.fillna(0, inplace=True)\n\n            # Data Type\n            test['prior_question_had_explanation'] = test['prior_question_had_explanation'].astype(np.int8)            \n            \n            # Dictionary Upgrade: answered_correctly sum\n            values = prior_test.groupby('user_id').agg({'user_id': lambda x: 0,\n                                                         'answered_correctly':np.sum,\n                                                         'prior_question_elapsed_time':lambda x: 0})\n            for k, v in zip(values.index, np.array(values)):\n                userid_dic[k] += v\n\n            values = prior_test.groupby('content_id').agg({'content_id': lambda x: 0,\n                                                             'answered_correctly':np.sum,\n                                                             'prior_question_elapsed_time':[lambda x: 0, lambda x:0]})\n            for k, v in zip(values.index, np.array(values)):\n                contentid_dic[k] += v\n                \n            values = prior_test.groupby('tags').agg({'tags': lambda x: 0,\n                                                     'answered_correctly':np.sum,\n                                                     'prior_question_elapsed_time':lambda x: 0})\n            for k, v in zip(values.index, np.array(values)):\n                tag_dic[k] += v       \n                \n            values = prior_test.groupby('part').agg({'part': lambda x: 0,\n                                                     'answered_correctly':np.sum,\n                                                     'prior_question_elapsed_time':lambda x: 0})\n            for k, v in zip(values.index, np.array(values)):\n                part_dic[k] += v\n                \n#             values = prior_test.groupby('task_container_id').agg({'task_container_id': lambda x: 0,\n#                                                                  'answered_correctly':np.sum,\n#                                                                  'prior_question_elapsed_time':lambda x: 0})\n#             for k, v in zip(values.index, np.array(values)):\n#                 task_dic[k] += v                \n            \n        # user_id & content_id 별 answer mean / time mean       &&       Dictionary Upgrade: Variables excl. answered_correctly sum\n        \n        # user_id\n        test['userid_answer_mean'] = test['user_id'].apply(lambda x: userid_dic[x][1]) / test['user_id'].apply(lambda x: userid_dic[x][0]) \n        values = test.groupby('user_id').agg({'user_id':len,                                \n                                              'answered_correctly':lambda x:0,\n                                              'prior_question_elapsed_time':np.sum})\n        for k, v in zip(values.index, np.array(values)): \n            userid_dic[k] += v \n        test['userid_time_mean'] = test['user_id'].apply(lambda x: userid_dic[x][2]) / test['user_id'].apply(lambda x: userid_dic[x][0]) \n        test['userid_cnt'] = test['user_id'].apply(lambda x:userid_dic[x][0])\n        \n        \n        # content_id\n        test['contentid_answer_mean'] = test['content_id'].apply(lambda x: contentid_dic[x][1]) / test['content_id'].apply(lambda x: contentid_dic[x][0])\n        values = test.groupby('content_id').agg({'content_id':len,\n                                                 'answered_correctly':lambda x:0,\n                                                 'prior_question_elapsed_time':[np.sum, np.min]})\n        for k, v in zip(values.index, np.array(values)): \n            contentid_dic[k][:3] += v[:3]\n            contentid_dic[k][3] = min(contentid_dic[k][3], v[3])\n        test['contentid_time_mean'] = test['content_id'].apply(lambda x: contentid_dic[x][2]) / test['content_id'].apply(lambda x: contentid_dic[x][0])\n        test['contentid_time_min'] = test['content_id'].apply(lambda x: contentid_dic[x][3])\n        test['contentid_cnt'] = test['content_id'].apply(lambda x:contentid_dic[x][0])\n        \n        # tag\n        test['tag_answer_mean'] = test['tags'].apply(lambda x: tag_dic[x][1]) / test['tags'].apply(lambda x: tag_dic[x][0])\n        values = test.groupby('tags').agg({'tags':len,   \n                                           'answered_correctly':lambda x:0,\n                                           'prior_question_elapsed_time':np.sum})\n        for k, v in zip(values.index, np.array(values)): \n            tag_dic[k] += v \n        test['tag_time_mean'] = test['tags'].apply(lambda x: tag_dic[x][2]) / test['tags'].apply(lambda x: tag_dic[x][0])\n        test['tag_cnt'] = test['tags'].apply(lambda x:tag_dic[x][0])\n        \n        # part\n        test['part_answer_mean'] = test['part'].apply(lambda x: part_dic[x][1]) / test['part'].apply(lambda x: part_dic[x][0])\n        values = test.groupby('user_id').agg({'part':len,                                \n                                              'answered_correctly':lambda x:0,\n                                              'prior_question_elapsed_time':np.sum})\n        for k, v in zip(values.index, np.array(values)): \n            part_dic[k] += v \n        test['part_time_mean'] = test['part'].apply(lambda x: part_dic[x][2]) / test['part'].apply(lambda x: part_dic[x][0])\n        \n#         # task_container_id\n#         test['task_answer_mean'] = test['task_container_id'].apply(lambda x: task_dic[x][1]) / test['task_container_id'].apply(lambda x: task_dic[x][0])\n#         values = test.groupby('task_container_id').agg({'task_container_id':len,                                \n#                                                       'answered_correctly':lambda x:0,\n#                                                       'prior_question_elapsed_time':np.sum})\n#         for k, v in zip(values.index, np.array(values)): \n#             task_dic[k] += v \n#         test['task_time_mean'] = test['task_container_id'].apply(lambda x: task_dic[x][2]) / test['task_container_id'].apply(lambda x: task_dic[x][0])\n        \n        print(test.isnull().sum()/len(test))\n\n        # 정답통계량, merge 시 발생한 NA 처리\n        test.fillna(0, inplace=True)\n        test['answered_correctly'] = model.predict(test[X])\n        env.predict(test.loc[test['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n        prior_test = test.copy()\n                                                          \nelse:\n    for (test, sample_prediction_df) in iter_test:\n        test['answered_correctly']=0\n        if prior_test is None:\n            # filtering & merging\n            test = test[test['content_type_id']==0]\n            test = test.merge(questions_, how='left', on=['content_type_id', 'content_id'])\n\n            # NA\n            test.fillna(0, inplace=True)\n\n            # Data Type\n            test['prior_question_had_explanation'] = test['prior_question_had_explanation'].astype(np.int8)\n        \n        if prior_test is not None:            \n            prior_answers = np.array(eval(test.iloc[0]['prior_group_answers_correct']))\n            prior_answers = prior_answers[prior_answers!=-1]\n            prior_test['answered_correctly'] = prior_answers\n            \n            # filtering & merging\n            test = test[test['content_type_id']==0]\n            test = test.merge(questions_, how='left', on=['content_type_id', 'content_id'])\n\n            # NA\n            test.fillna(0, inplace=True)\n\n            # Data Type\n            test['prior_question_had_explanation'] = test['prior_question_had_explanation'].astype(np.int8)            \n            \n            # Dictionary Upgrade: answered_correctly sum\n            values = prior_test.groupby('user_id').agg({'user_id': lambda x: 0,\n                                                         'answered_correctly':np.sum,\n                                                         'prior_question_elapsed_time':lambda x: 0})\n            for k, v in zip(values.index, np.array(values)):\n                userid_dic[k] += v\n\n            values = prior_test.groupby('content_id').agg({'content_id': lambda x: 0,\n                                                             'answered_correctly':np.sum,\n                                                             'prior_question_elapsed_time':[lambda x: 0, lambda x:0]})\n            for k, v in zip(values.index, np.array(values)):\n                contentid_dic[k] += v\n                \n            values = prior_test.groupby('tags').agg({'tags': lambda x: 0,\n                                                     'answered_correctly':np.sum,\n                                                     'prior_question_elapsed_time':lambda x: 0})\n            for k, v in zip(values.index, np.array(values)):\n                tag_dic[k] += v       \n                \n            values = prior_test.groupby('part').agg({'part': lambda x: 0,\n                                                     'answered_correctly':np.sum,\n                                                     'prior_question_elapsed_time':lambda x: 0})\n            for k, v in zip(values.index, np.array(values)):\n                part_dic[k] += v\n                \n#             values = prior_test.groupby('task_container_id').agg({'task_container_id': lambda x: 0,\n#                                                                  'answered_correctly':np.sum,\n#                                                                  'prior_question_elapsed_time':lambda x: 0})\n#             for k, v in zip(values.index, np.array(values)):\n#                 task_dic[k] += v                \n            \n        # user_id & content_id 별 answer mean / time mean       &&       Dictionary Upgrade: Variables excl. answered_correctly sum\n        \n        # user_id\n        test['userid_answer_mean'] = test['user_id'].apply(lambda x: userid_dic[x][1]) / test['user_id'].apply(lambda x: userid_dic[x][0]) \n        values = test.groupby('user_id').agg({'user_id':len,                                \n                                              'answered_correctly':lambda x:0,\n                                              'prior_question_elapsed_time':np.sum})\n        for k, v in zip(values.index, np.array(values)): \n            userid_dic[k] += v \n        test['userid_time_mean'] = test['user_id'].apply(lambda x: userid_dic[x][2]) / test['user_id'].apply(lambda x: userid_dic[x][0]) \n        test['userid_cnt'] = test['user_id'].apply(lambda x:userid_dic[x][0])\n        \n        \n        # content_id\n        test['contentid_answer_mean'] = test['content_id'].apply(lambda x: contentid_dic[x][1]) / test['content_id'].apply(lambda x: contentid_dic[x][0])\n        values = test.groupby('content_id').agg({'content_id':len,\n                                                 'answered_correctly':lambda x:0,\n                                                 'prior_question_elapsed_time':[np.sum, np.min]})\n        for k, v in zip(values.index, np.array(values)): \n            contentid_dic[k][:3] += v[:3]\n            contentid_dic[k][3] = min(contentid_dic[k][3], v[3])\n        test['contentid_time_mean'] = test['content_id'].apply(lambda x: contentid_dic[x][2]) / test['content_id'].apply(lambda x: contentid_dic[x][0])\n        test['contentid_time_min'] = test['content_id'].apply(lambda x: contentid_dic[x][3])\n        test['contentid_cnt'] = test['content_id'].apply(lambda x:contentid_dic[x][0])\n        \n        # tag\n        test['tag_answer_mean'] = test['tags'].apply(lambda x: tag_dic[x][1]) / test['tags'].apply(lambda x: tag_dic[x][0])\n        values = test.groupby('tags').agg({'tags':len,   \n                                           'answered_correctly':lambda x:0,\n                                           'prior_question_elapsed_time':np.sum})\n        for k, v in zip(values.index, np.array(values)): \n            tag_dic[k] += v \n        test['tag_time_mean'] = test['tags'].apply(lambda x: tag_dic[x][2]) / test['tags'].apply(lambda x: tag_dic[x][0])\n        test['tag_cnt'] = test['tags'].apply(lambda x:tag_dic[x][0])\n        \n        # part\n        test['part_answer_mean'] = test['part'].apply(lambda x: part_dic[x][1]) / test['part'].apply(lambda x: part_dic[x][0])\n        values = test.groupby('user_id').agg({'part':len,                                \n                                              'answered_correctly':lambda x:0,\n                                              'prior_question_elapsed_time':np.sum})\n        for k, v in zip(values.index, np.array(values)): \n            part_dic[k] += v \n        test['part_time_mean'] = test['part'].apply(lambda x: part_dic[x][2]) / test['part'].apply(lambda x: part_dic[x][0])\n        \n#         # task_container_id\n#         test['task_answer_mean'] = test['task_container_id'].apply(lambda x: task_dic[x][1]) / test['task_container_id'].apply(lambda x: task_dic[x][0])\n#         values = test.groupby('task_container_id').agg({'task_container_id':len,                                \n#                                                       'answered_correctly':lambda x:0,\n#                                                       'prior_question_elapsed_time':np.sum})\n#         for k, v in zip(values.index, np.array(values)): \n#             task_dic[k] += v \n#         test['task_time_mean'] = test['task_container_id'].apply(lambda x: task_dic[x][2]) / test['task_container_id'].apply(lambda x: task_dic[x][0])\n        \n        print(test.isnull().sum()/len(test))\n\n        # 정답통계량, merge 시 발생한 NA 처리\n        test.fillna(0, inplace=True)\n        test.to_csv(\"test{}.csv\".format(i))\n        \n        test['answered_correctly'] = model.predict(test[X])\n        env.predict(test.loc[test['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n        i+=1\n        prior_test = test.copy()\n        print(userid_dic)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}