{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl\n\n# Download from internet\n# !pip install datatable==0.11.0 > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import riiideducation\nimport gc\nimport numpy as np\nimport pandas as pd\nfrom collections import defaultdict\nimport datatable as dt\nimport lightgbm as lgb\nfrom matplotlib import pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Read Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Read in train.csv\n#Using pandas to read in will be too slow, so we use datatable instead.\ntrain_df = dt.fread('../input/riiid-test-answer-prediction/train.csv').to_pandas()\ntrain_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# Find the max value of each column to determine data types\ntrain_df.max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.memory_usage(deep=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# Decrease memory use by convert original data types to smaller data types.\ntrain_data_types = {\n    'row_id': 'int32',\n    'timestamp': 'int64',\n    'user_id': 'int32', \n    'content_id': 'int16', \n    'content_type_id': 'int8',\n    'task_container_id': 'int16',\n    'user_answer': 'int8',\n    'answered_correctly': 'int8', \n    'prior_question_elapsed_time': 'float32', \n    'prior_question_had_explanation': 'bool',\n}\nfor column, dtype in train_data_types.items():\n    train_df[column] = train_df[column].astype(dtype) \ntrain_df.memory_usage(deep=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Explore Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1. Relationship between timestamp and answer_correctly"},{"metadata":{"trusted":true},"cell_type":"code","source":"ms_per_year = 1000 * 60 * 60 * 24 * 365\nts = train_df['timestamp']/(ms_per_year/365)\nfig = plt.figure(figsize=(12,6))\nts.plot.hist(bins=100)\nplt.title(\"Histogram of timestamp\")\nplt.xticks(rotation=0)\nplt.xlabel(\"Days between this user interaction and the first event completion from that user\")\nplt.show()\ndel ts","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Return a reshaped dataframe organized by specified field with respect to percentage of correct answers\ndef correct(field):\n    correct = train_df[train_df.answered_correctly != -1].groupby([field, 'answered_correctly'], as_index=False).size()\n    correct = correct.pivot(index= field, columns='answered_correctly', values='size')\n    correct['Percent_correct'] = round(correct.iloc[:,1]/(correct.iloc[:,0] + correct.iloc[:,1]),2)\n    correct = correct.sort_values(by = \"Percent_correct\", ascending = False)\n    correct = correct.iloc[:,2]\n    return correct","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group_labels_6 = ['Group_1', 'Group_2', 'Group_3', 'Group_4', 'Group_5', 'Group_6']\ntrain_df['timestamp_group'] = pd.qcut(train_df['timestamp'], q=6, labels=group_labels_6)\n\nts_correct = correct(\"timestamp_group\")\nts_correct = ts_correct.sort_index()\n\nfig = plt.figure(figsize=(12,6))\nts_correct.plot.bar()\nplt.title(\"Percentage of answered_correctly for 6 groups of timestamp\")\nplt.xticks(rotation=0)\nplt.show()\ndel ts_correct","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Users in Group_1 have relatively worst percentage of correctness. Difference of performance of other groups is not significant much.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Use a new column to indicate users with shortest timestamp (\"Group_1\")\ntrain_df['new_users'] = np.where(train_df['timestamp_group'] == 'Group_1', True, False)\ndel train_df['timestamp_group']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"2. Relationship between number of questions answered per user and answer_correctly"},{"metadata":{"trusted":true},"cell_type":"code","source":"user_percent = train_df[train_df.answered_correctly != -1].groupby('user_id')['answered_correctly'].agg(Mean='mean', Answers='count')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_percent = user_percent.query('Answers <= 2000').sample(n=1000, random_state=1)\n\nfig = plt.figure(figsize=(12,6))\nx = user_percent.Answers\ny = user_percent.Mean\nplt.scatter(x, y, marker='o')\nplt.title(\"Percent answered correctly versus number of questions answered\")\nplt.xticks(rotation=0)\nplt.xlabel(\"Number of questions answered\")\nplt.ylabel(\"Percent answered correctly\")\nz = np.polyfit(x, y, 1)\np = np.poly1d(z)\nplt.plot(x,p(x),\"r--\")\n\nplt.show()\ndel user_percent","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Increasing number of questions answered only slightly increases the percentage of correctness.**"},{"metadata":{},"cell_type":"markdown","source":"3. Relationship between prior_question_elapsed_time and answer_correctly"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['prior_question_elapsed_time'] = train_df['prior_question_elapsed_time'].fillna(0)\nelapse_labels_5 = ['Bin_1', 'Bin_2', 'Bin_3', 'Bin_4', 'Bin_5']\ntrain_df['elapse_bin'] = pd.qcut(train_df['prior_question_elapsed_time'], q=5, labels=elapse_labels_5)\n\nelapse_correct = correct(\"elapse_bin\")\nelapse_correct = elapse_correct.sort_index()\n\nfig = plt.figure(figsize=(12,6))\nelapse_correct.plot.bar()\nplt.title(\"Percent answered_correctly for 5 bins of prior_question_elapsed_time\")\nplt.xticks(rotation=0)\nplt.show()\ndel elapse_correct","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**'prior_question_elapsed_time' does not have a strong correlation with 'answer_correctly'.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_df[\"elapse_bin\"]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"4. Relationship between prior_question_had_explanation and answer_correctly"},{"metadata":{"trusted":true},"cell_type":"code","source":"pq = train_df[train_df.answered_correctly != -1].groupby(['prior_question_had_explanation']).agg({'answered_correctly': ['mean']})\nfig = plt.figure(figsize=(12,10))\npq.plot.bar(legend=None)\nplt.title(\"Answered_correctly versus Prior_question_had_explanation\")\nplt.xlabel(\"Prior question had explanation\")\nplt.ylabel(\"Percent answered correctly\")\nplt.xticks(rotation=0)\nplt.show()\ndel pq","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Prior question having explanation help user to increase percentage of answer correctly.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"There are {train_df.user_id.nunique()} unique users in Train.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Read Lectures"},{"metadata":{"trusted":true},"cell_type":"code","source":"lectures = pd.read_csv('../input/riiid-test-answer-prediction/lectures.csv')\nlectures.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lectures.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Explore Lectures"},{"metadata":{"trusted":true},"cell_type":"code","source":"lect_type_of = lectures.type_of.value_counts()\n\nfig = plt.figure(figsize=(12,6))\nlect_type_of.plot.barh()\nplt.title(\"Counts of different types of lectures\")\nplt.xlabel(\"Count of lectures\")\nplt.xticks(rotation=0)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1. Relationship between watching lecture or not and answer_correctly"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Group train_df by 'user_id' and 'answer_correctly'\nuser_lect = train_df.groupby([\"user_id\", \"answered_correctly\"]).size().unstack()\n# Changed [-1, 0, 1] to ['Lecture', 'Wrong', 'Right']\nuser_lect.columns = ['Lecture', 'Wrong', 'Right']\nuser_lect['Lecture'] = user_lect['Lecture'].fillna(0)\n\n# Add another column to indicate whether the user watch lectures or not\nuser_lect = user_lect.astype('Int64')\nuser_lect['Watched_lecture'] = np.where(user_lect.Lecture > 0, True, False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reshape user_lect by grouping 'Watched_lectures' and count the sum of wrong and right answers\nwatched_l = user_lect.groupby(\"Watched_lecture\").agg({'Wrong': ['sum'], 'Right': ['sum']})\n(t, f) = user_lect.Watched_lecture.value_counts()\nprint(f\"Watched lecture(s): \\t{t}\\nNot watched lecture(s): {f}\")\n\n# Add a column to compute percentage of correct answers\nwatched_l['Percent_correct'] = watched_l.Right/(watched_l.Right + watched_l.Wrong)\nwatched_l = watched_l.iloc[:,2]\n\nfig = plt.figure(figsize=(8,6))\nwatched_l.plot.bar()\nplt.title(\"User Watched Lectures Versus Percent of Correctness\")\nplt.xlabel(\"User watched at least one lecture\")\nplt.ylabel(\"Percent of correctness\")\nplt.xticks(rotation=0)\nplt.show()\ndel watched_l","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Watching lectures help increase correctness of answering questions.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"user_lect = user_lect.reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.merge(train_df, user_lect[['user_id', 'Watched_lecture']], how='left', on=['user_id', 'user_id'])\ndel user_lect","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Read Questions"},{"metadata":{"trusted":true},"cell_type":"code","source":"questions = pd.read_csv('../input/riiid-test-answer-prediction/questions.csv')\nquestions.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Explore Questions"},{"metadata":{"trusted":true},"cell_type":"code","source":"questions[questions.tags.isna()]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Counting tags"},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"questions['tags'] = questions['tags'].astype(str)\n\ntags = [x.split() for x in questions[questions.tags != \"nan\"].tags.values]\ntags = [item for elem in tags for item in elem]\nprint(f'There are {len(set(tags))} different tags')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tags_list = [x.split() for x in questions.tags.values]\nquestions['tags'] = tags_list\nquestions.head()\n\ncorrect = train_df[train_df.answered_correctly != -1].groupby([\"content_id\", 'answered_correctly'], as_index=False).size()\ncorrect = correct.pivot(index= \"content_id\", columns='answered_correctly', values='size')\ncorrect.columns = ['Wrong', 'Right']\ncorrect = correct.fillna(0)\ncorrect[['Wrong', 'Right']] = correct[['Wrong', 'Right']].astype(int)\nquestions = questions.merge(correct, left_on = \"question_id\", right_on = \"content_id\", how = \"left\")\nquestions.head()\ndel correct","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"------------------"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.drop(labels=['timestamp','content_type_id','task_container_id','user_answer','prior_question_elapsed_time'],axis=1)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#using one of the validation sets composed by tito\ncv_train = pd.read_pickle(\"../input/cv-index-for-riiid/train_index.pkl\")\ncv_valid = pd.read_pickle(\"../input/cv-index-for-riiid/valid_index.pkl\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cv_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Split the train set as train and validation set.\nvalidation_df = train_df[train_df.row_id.isin(cv_valid)]\ntrain_df = train_df[train_df.row_id.isin(cv_train)]\n\nvalidation_df = validation_df.drop(columns = \"row_id\")\ntrain_df = train_df.drop(columns = \"row_id\")\n\ndel cv_train, cv_valid\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Fill na in the merged dataset\n#current we do not merge Questions and Lectures, so this Function simply return the original dataframe\ndef merge_fill_na(df):\n    #df = df.merge(user_df, on = \"user_id\", how = \"left\")\n    #df = df.merge(content_df, on = \"content_id\", how = \"left\")\n    #df['content_questions'].fillna(0, inplace = True)\n    #df['content_mean'].fillna(0.5, inplace = True)\n    #df['watches_lecture'].fillna(0, inplace = True)\n    #df['user_questions'].fillna(0, inplace = True)\n    #df['user_mean'].fillna(0.5, inplace = True)\n    #df[['content_questions', 'user_questions']] = df[['content_questions', 'user_questions']].astype(int)\n    df['prior_question_had_explanation'].fillna(True, inplace = True)\n    return(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Read in Questions, Lectures and two tests files\nquestions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')\nlectures = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv')\nexample_test = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/example_test.csv')\nexample_sample_submission = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/example_sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain_df = merge_fill_na(train_df)\n# validation_df = merge_fill_na(validation_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#build final train/validation set\nfeatures = ['content_id', 'prior_question_had_explanation']\n\ntrain_df = train_df.sample(n=10000000, random_state = 1)\ny_train = train_df['answered_correctly']\ntrain = train_df[features]\n\ny_val = validation_df['answered_correctly']\nvalidation = validation_df[features]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#define LGBM params\nparams = {'objective': 'binary',\n          'metric': 'auc',\n          'seed': 2020,\n          'learning_rate': 0.1, #default\n          \"boosting_type\": \"gbdt\" #default\n         }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_train = lgb.Dataset(train, y_train, categorical_feature = ['prior_question_had_explanation'])\nlgb_eval = lgb.Dataset(validation, y_val, categorical_feature = ['prior_question_had_explanation'])\ndel train, y_train, validation, y_val\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#train\nmodel = lgb.train(\n    params, lgb_train,\n    valid_sets=[lgb_train, lgb_eval],\n    verbose_eval=50,\n    num_boost_round=10000,\n    early_stopping_rounds=8\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb.plot_importance(model)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# You can only call make_env() once, so don't lose it!\nenv = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    test_df = merge_fill_na(test_df)\n    test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].astype('bool')\n    test_df['answered_correctly'] =  model.predict(test_df[features])\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}