{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport os","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Import"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_pickle(\"../input/riiid-train-data-multiple-formats/riiid_train.pkl.gzip\")\n\nprint(\"Train size:\", train_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_df.dtypes)\ntrain_df['prior_question_had_explanation'] = train_df['prior_question_had_explanation'].astype('bool')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')\nlectures = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv')\nexample_test = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/example_test.csv')\nexample_sample_submission = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/example_sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data exploration"},{"metadata":{},"cell_type":"markdown","source":"## Train dataset"},{"metadata":{},"cell_type":"markdown","source":"### First look"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Number of rows : {}\".format(train_df.shape[0]))\nprint(\"Number of individual user : {}\".format(train_df.user_id.nunique()))\nprint(\"Number of individual question : {}\".format(train_df.content_id.nunique()))\nprint(\"Number of individual tasks: {}\".format(train_df.task_container_id.nunique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"groupby_user_id = train_df.groupby(\"user_id\").count()\n\nplt.figure(figsize=(15,5))\n\nplt.subplot(1,2,1)\nsns.distplot(groupby_user_id.row_id, kde=False)\nplt.title(\"Distribution of number of questions per users\")\n\nplt.subplot(1,2,2)\nthreshold = 200\nsns.distplot(groupby_user_id[groupby_user_id.row_id<threshold].row_id, kde=False, bins=100)\nplt.title(\"Distribution of number of questions per users (<{})\".format(threshold))\n\nprint(\"Average number of questions per user : {}\".format(groupby_user_id.row_id.mean()))\nprint(\"Median number of questions per user : {}\".format(groupby_user_id.row_id.median()))\nprint(\"75% quantile : {}\".format(groupby_user_id.row_id.quantile(0.75)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,5))\n\nplt.subplot(1,2,1)\n\n\nts = train_df['timestamp']/(31536000000/12)\nsns.distplot(ts, kde=False, bins=100, color=\"b\")\nplt.title(\"Timestamp histogram\")\nplt.xlabel(\"Month between first user interaction & current interaction\")\n\nplt.subplot(1,2,2)\n\nts = train_df.groupby('user_id').max()[\"timestamp\"]/(31536000000/12)\n\nsns.distplot(ts, kde=False, bins=100, color=\"b\")\nplt.title(\"Histogram of last registered interactions\")\nplt.xlabel(\"Month between first & last interaction for each user\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"It seems that most of the users don't stay active for a long time."},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,7))\nsns.countplot(train_df.user_answer)\nplt.title(\"User answers count\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We would expect to have a uniform repartition between 0,1,2 and 3, but it seems that the answer #2 is significantly less choosen."},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,7))\nsns.countplot(train_df.answered_correctly, orient=\"v\")\nplt.title(\"Question answered correctly\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"Approximately one third of the answers are incorrect."},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,7))\n\ntrain_df_f = train_df[train_df[\"user_answer\"]!=-1]\nsns.countplot(x=\"user_answer\", hue=\"answered_correctly\", data=train_df_f)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Expectedly, the proportion of right and wrong answers is the same for each answer number."},{"metadata":{"trusted":true},"cell_type":"code","source":"time_prior_question = train_df[train_df.prior_question_elapsed_time.isna() == False].prior_question_elapsed_time\nplt.figure(figsize=(15,7))\n\nsns.distplot(time_prior_question, kde=False, color=\"b\")\nplt.title(\"Prior question elapsed time histogram\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,7))\n\nsns.countplot(x = \"prior_question_had_explanation\" , hue=\"answered_correctly\", data=train_df_f)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The proportion of false answer is higher when the prior question didn't have an explanation."},{"metadata":{},"cell_type":"markdown","source":"## Question Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(questions.head(5))\nprint(questions.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,7))\nsns.countplot(questions.correct_answer)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This explains why the answer #2 was less picked than the others"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,7))\nsns.countplot(questions.part)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}