{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# train.csv"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"data_types_dict = {\n    'row_id': 'int64',\n    'timestamp': 'int64',\n    'user_id': 'int32',\n    'content_id': 'int16',\n    'content_type_id': 'int8',\n    'task_container_id': 'int16',\n    'user_answer': 'int8',\n    'answered_correctly': 'int8',\n    'prior_question_elapsed_time': 'float16',\n    'prior_question_had_explanation': 'boolean'\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',\n                       low_memory=False,\n                       nrows=10**7,\n                       dtype=data_types_dict, \n                      )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### timestamp"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['timestamp'].hist(bins=100);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"timestamp・・・ユーザーとの対話からそのイベント終了までの時間"},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_by_user_df = train_df.groupby('user_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_by_user_df.agg({'timestamp':'max'}).hist(bins=100);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"↑各ユーザーの最大のtimestampの分布・・・ほとんどのユーザーがすぐにプラットフォームを離れるようだ。"},{"metadata":{},"cell_type":"markdown","source":"### Answered correctly\n ユーザーが正しく応答したかどうか。講義と質問がある。講義（lectures）の場合は、-1をnullとして読み取ります。質問の場合は、正答１、誤答０"},{"metadata":{"trusted":true},"cell_type":"code","source":"# 講義の割合  # 平均 -1 (True)の割合\n(train_df['answered_correctly'] == -1).mean()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"trainデータの約2%は、「講義」である。→回答分析から除外する必要がある。"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_questions_only_df = train_df[train_df['answered_correctly'] != -1]\ntrain_questions_only_df['answered_correctly'].mean()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"平均して、ユーザーは最大６６％の質問に正しく答えている。 →ユーザーごとにどのくらい違うかも見てみる"},{"metadata":{},"cell_type":"markdown","source":"### Answers by users"},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_by_user_df = train_questions_only_df.groupby('user_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 回答率('mean')と回答数（'count'）で分ける\nuser_answers_df = grouped_by_user_df.agg({'answered_correctly': ['mean', 'count']})\nuser_answers_df[('answered_correctly', 'mean')].hist(bins=100); # bins = 棒の数","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_answers_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_answers_df[('answered_correctly', 'count')].hist(bins=100);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"(user_answers_df[('answered_correctly','count')]< 50).mean()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"ユーザーの54％が、50未満の質問に回答。 → すべてのユーザーを「初心者」と「アクティブユーザー」に分けてみる。"},{"metadata":{"trusted":true},"cell_type":"code","source":"# 初心者の正答率\nuser_answers_df[user_answers_df[('answered_correctly', 'count')] < 50][('answered_correctly', 'mean')].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_answers_df[user_answers_df[('answered_correctly', 'count')] < 50][('answered_correctly', 'mean')].hist(bins=100);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# アクティブユーザーの正答率\nuser_answers_df[user_answers_df[('answered_correctly', 'count')] >= 50][('answered_correctly', 'mean')].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_answers_df[user_answers_df[('answered_correctly', 'count')] >= 50][('answered_correctly', 'mean')].hist(bins=100);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"アクティブユーザーは、初心者よりもはるかに優れている。 \\\n全体の平均66%　しかし、平均ユーザースコアは、正解の全体の66％よりも低くなっている。→これは、ヘビーユーザーのスコアがさらに高くなることを意味する。"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ヘビーユーザーの割合 500以上questionを回答しているユーザーの割合\n(user_answers_df[('answered_correctly','count')] >= 500).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ヘビーユーザーの回答率の分布\nuser_answers_df[user_answers_df[('answered_correctly', 'count')] >= 500][('answered_correctly', 'mean')].hist(bins=100);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ヘビーユーザーの正答率\nuser_answers_df[user_answers_df[('answered_correctly', 'count')] >= 500][('answered_correctly', 'mean')].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.scatter(x = user_answers_df[('answered_correctly', 'count')], y = user_answers_df[('answered_correctly', 'mean')]);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### これまでのまとめ\n・Timestamp, ・アクティブユーザーの平均スコア, ・回答された質問の数、はベースラインの作成に役立ちそう。"},{"metadata":{},"cell_type":"markdown","source":"### Answers by content"},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_by_content_df = train_questions_only_df.groupby('content_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"content_answers_df = grouped_by_content_df.agg({'answered_correctly': ['mean', 'count']})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"content_answers_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"content_answers_df[('answered_correctly', 'count')].hist(bins=100);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"content_answers_df[('answered_correctly', 'mean')].hist(bins=100);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"質問(content_id)が異なれば、answered_correctlyも異なるため、ベースラインに使えそう。"},{"metadata":{"trusted":true},"cell_type":"code","source":"content_answers_df[content_answers_df[('answered_correctly','count')]>50][('answered_correctly','mean')].hist(bins = 100);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Questions.csv"},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df = pd.read_csv('../input/riiid-test-answer-prediction/questions.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"There are {len(questions_df['part'].unique())} different parts\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df['tags'].values[-1] # なんで最後の行のtagを取得してるのか？","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unique_tags = set().union(*[y.split() for y in questions_df['tags'].astype(str).values])\n\nprint(f\"There are {len(unique_tags)} different tags\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# [question_id] content_type_idが質問(0)のとき、train/test content_id列の外部キー / [bundle_id] 質問と一緒に提供されるコード\n(questions_df['question_id'] != questions_df['bundle_id']).mean()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Baseline\n\nこれまでの発見した特徴をモデルで使用して、正解の確率を予測する。"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',\n                       low_memory=False,\n                       nrows=10**7,\n                       dtype=data_types_dict, \n                      )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 900万行\nfeatures_part_df = train_df.iloc[:int( 9 / 10 * len(train_df) )]\n# 100万行\ntrain_part_df = train_df.iloc[int( 9 / 10 * len(train_df) ):]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_questions_only_df = features_part_df[features_part_df['answered_correctly'] != -1]\n\ngrouped_by_user_df = train_questions_only_df.groupby('user_id')\n\nuser_answers_df = grouped_by_user_df.agg({'answered_correctly': ['mean', 'count']}).copy()\nuser_answers_df.columns = ['mean_user_accuracy', 'questions_answered']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_answers_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_by_content_df = train_questions_only_df.groupby('content_id')\n\ncontent_answers_df = grouped_by_content_df.agg({'answered_correctly': ['mean', 'count'] }).copy()\ncontent_answers_df.columns = ['mean_accuracy', 'question_asked']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"content_answers_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"questions_df　を使用して追加の特徴量を作成"},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df = questions_df.merge(content_answers_df, left_on = 'question_id', right_on = 'content_id', how = 'left')\n\n# [question_id] content_id列の外部キー ・・・　content_type_idが質問(0)のとき、train/test content_id列の外部キー","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bundle_dict = questions_df['bundle_id'].value_counts().to_dict()\n# value_count ・・・　ユニークな要素の値とその出現回数をpandas.Seriesで返す","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# [bundle_id] 質問と一緒に提供されるコード\nbundle_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# right_answers 正解数\nquestions_df['right_answers'] = questions_df['mean_accuracy'] * questions_df['question_asked']\n\nquestions_df['bundle_size'] = questions_df['bundle_id'].apply(lambda x: bundle_dict[x])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_by_bundle_df = questions_df.groupby('bundle_id')\n\nbundle_answers_df = grouped_by_bundle_df.agg({'right_answers': 'sum', 'question_asked': 'sum'}).copy()\nbundle_answers_df.columns = ['bundle_right_answers', 'bundle_questions_asked']\n\nbundle_answers_df['bundle_accuracy'] = bundle_answers_df['bundle_right_answers'] / bundle_answers_df['bundle_questions_asked']\n\nbundle_answers_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_by_part_df = questions_df.groupby('part')\n\npart_answers_df = grouped_by_part_df.agg({'right_answers': 'sum', 'question_asked': 'sum'}).copy()\n\npart_answers_df.columns = ['part_right_answers', 'part_questions_asked']\npart_answers_df['part_accuracy'] = part_answers_df['part_right_answers'] / part_answers_df['part_questions_asked']\n\npart_answers_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_df\ndel features_part_df\ndel grouped_by_user_df\ndel grouped_by_content_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# python のメモリ管理\nimport gc\n# 何も考えずにとりあえずGCを動かすには以下の通り。回収可能なオブジェクトを削除。\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = [\n    'timestamp','mean_user_accuracy', 'questions_answered','mean_accuracy',\n    'question_asked','prior_question_elapsed_time', 'prior_question_had_explanation',\n    'bundle_size', 'bundle_accuracy','part_accuracy', 'right_answers'\n]\n\ntarget = 'answered_correctly'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 講義(-1)以外を抽出 train\ntrain_part_df = train_part_df[train_part_df[target] != -1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_part_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 追加した特徴量のdfをマージ\n\n# user_answers_df\ntrain_part_df = train_part_df.merge(user_answers_df, how='left', on='user_id')\n\n# questions_df\ntrain_part_df = train_part_df.merge(questions_df, how='left', left_on='content_id', right_on='question_id')\n\n# bundle_answers_df\ntrain_part_df = train_part_df.merge(bundle_answers_df, how='left', on='bundle_id')\n\n# part_answers_df\ntrain_part_df = train_part_df.merge(part_answers_df, how='left', on='part')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_part_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ユーザーが質問に回答した後、説明と正しい回答を確認したかどうか 欠損値をFalseと置く、 astypeでデータ型の変換(キャスト)\ntrain_part_df['prior_question_had_explanation'] = train_part_df['prior_question_had_explanation'].fillna(value=False).astype(bool)\n\ntrain_part_df.fillna(value = -1, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_part_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_part_df.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_part_df = train_part_df[features + [target]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_part_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from lightgbm import LGBMClassifier","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgbm = LGBMClassifier(\n    num_leaves=31, \n    max_depth= 2, \n    n_estimators = 25, \n    min_child_samples = 1000, \n    subsample=0.7, \n    subsample_freq=5,\n    n_jobs= -1,\n    is_higher_better = True,\n    first_metric_only = True\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgbm.fit(train_part_df[features], train_part_df[target])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"roc_auc_score(train_part_df[target].values, lgbm.predict_proba(train_part_df[features])[:,1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    test_df = test_df.merge(user_answers_df, how = 'left', on = 'user_id')\n    test_df = test_df.merge(questions_df, how = 'left', left_on = 'content_id', right_on = 'question_id')\n    test_df = test_df.merge(bundle_answers_df, how = 'left', on = 'bundle_id')\n    test_df = test_df.merge(part_answers_df, how = 'left', on = 'part')\n    \n    test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(value = False).astype(bool)\n    test_df.fillna(value = -1, inplace = True)\n\n    test_df['answered_correctly'] = lgbm.predict_proba(test_df[features])[:,1]\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}