{"cells":[{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"!pip install dabl","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport string\nimport time\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import roc_auc_score\nfrom xgboost import XGBClassifier\nimport lightgbm as lgb\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold, KFold, RepeatedKFold, GroupKFold, GridSearchCV, train_test_split, TimeSeriesSplit\nfrom sklearn import metrics\n\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set_style(\"ticks\")\n%matplotlib inline\nimport dabl\n\nimport plotly.offline as py\nimport plotly.graph_objs as go\nimport plotly.tools as tls\npy.init_notebook_mode(connected=True)\n\n\nfrom IPython.display import Markdown\ndef bold(string):\n    display(Markdown(string))\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \nimport warnings as wrn\nwrn.filterwarnings('ignore', category = DeprecationWarning) \nwrn.filterwarnings('ignore', category = FutureWarning) \nwrn.filterwarnings('ignore', category = UserWarning) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"![](https://www.koreatechtoday.com/wp-content/uploads/2020/04/riiid-logo-background-scaled.jpg)\n\n**In this competition, your challenge is to create algorithms for \"Knowledge Tracing,\" the modeling of student knowledge over time. The goal is to accurately predict how students will perform on future interactions. You will pair your machine learning skills using Riiid’s EdNet data. [source](https://www.kaggle.com/c/riiid-test-answer-prediction)**"},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '/kaggle/input'\n\ntrain = pd.read_csv(f'{path}/riiid-test-answer-prediction/train.csv', low_memory=False, nrows=9 * (10**5), \n                       dtype={'row_id': 'int64',\n                              'timestamp': 'int64',\n                              'user_id': 'int32',\n                              'content_id': 'int16',\n                              'content_type_id': 'int8',\n                              'task_container_id': 'int16',\n                              'user_answer': 'int8',\n                              'answered_correctly': 'int8',\n                              'prior_question_elapsed_time': 'float32', \n                              'prior_question_had_explanation': 'boolean',\n                             }\n                      )\n\ntest = pd.read_csv(f'{path}/riiid-test-answer-prediction/example_test.csv')\nsubmit = pd.read_csv(f'{path}/riiid-test-answer-prediction/example_sample_submission.csv')\nquestions = pd.read_csv(f'{path}/riiid-test-answer-prediction/questions.csv')\nlectures = pd.read_csv(f'{path}/riiid-test-answer-prediction/lectures.csv')\nprint('Train shapes: ', train.shape)\nprint('Test shapes: ', test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def description(df):\n    summary = pd.DataFrame(df.dtypes,columns=['dtypes'])\n    summary = summary.reset_index()\n    summary['Name'] = summary['index']\n    summary = summary[['Name','dtypes']]\n    summary['Missing'] = df.isnull().sum().values    \n    summary['Uniques'] = df.nunique().values\n    summary['First Value'] = df.iloc[0].values\n    summary['Second Value'] = df.iloc[1].values\n    summary['Third Value'] = df.iloc[2].values\n    return summary","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"description(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## pandas describe\n\ntrain.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Files\n**train.csv**\n\n* `row_id:` (int64) ID code for the row.\n\n* `timestamp:` (int64) the time between this user interaction and the first event from that user.\n\n* `user_id:` (int32) ID code for the user.\n\n* `content_id:` (int16) ID code for the user interaction\n\n* `content_type_id:` (int8) 0 if the event was a question being posed to the user, 1 if the event was the user watching a lecture.\n\n* `task_container_id:` (int16) Id code for the batch of questions or lectures. For example, a user might see three questions in a row before seeing the explanations for any of them. Those three would all share a task_container_id. Monotonically increasing for each user.\n\n* `user_answer:` (int8) the user's answer to the question, if any. Read -1 as null, for lectures.\n\n* `answered_correctly:` (int8) if the user responded correctly. Read -1 as null, for lectures.\n\n* `prior_question_elapsed_time:` (float32) How long it took a user to answer their previous question bundle, ignoring any lectures in between. The value is shared across a single question bundle, and is null for a user's first question bundle or lecture. Note that the time is the total time a user took to solve all the questions in the previous bundle.\n\n* `prior_question_had_explanation:` (bool) Whether or not the user saw an explanation and the correct response(s) after answering the previous question bundle, ignoring any lectures in between. The value is shared across a single question bundle, and is null for a user's first question bundle or lecture. Typically the first several questions a user sees were part of an onboarding diagnostic test where they did not get any feedback.\n\n**questions.csv: metadata for the questions posed to users.**\n\n* `question_id:` foreign key for the train/test content_id column, when the content type is question (0).\n\n* `bundle_id:` code for which questions are served together.\n\n* `correct_answer:` the answer to the question. Can be compared with the train user_answer column to check if the user was right.\n\n* `part:` top level category code for the question.\n\n* `tags:` one or more detailed tag codes for the question. The meaning of the tags will not be provided, but these codes are sufficient for clustering the questions together.\n\n**lectures.csv: metadata for the lectures watched by users as they progress in their education.**\n\n* `lecture_id:` foreign key for the train/test content_id column, when the content type is lecture (1).\n\n* `part:` top level category code for the lecture.\n\n* `tag:` one tag codes for the lecture. The meaning of the tags will not be provided, but these codes are sufficient for clustering the lectures together.\n\n* `type_of:` brief description of the core purpose of the lecture\n\n"},{"metadata":{},"cell_type":"markdown","source":"# Train.csv\n\n## Target Distribution- Answered correctly"},{"metadata":{"trusted":true},"cell_type":"code","source":"total = len(train)\nplt.figure(figsize=(10,6))\n\ng = sns.countplot(x='answered_correctly', data=train, palette='viridis')\ng.set_title(\"TARGET DISTRIBUTION\", fontsize = 20)\ng.set_xlabel(\"Target Vaues\", fontsize = 15)\ng.set_ylabel(\"Count\", fontsize = 15)\nsizes=[] # Get highest values in y\nfor p in g.patches:\n    height = p.get_height()\n    sizes.append(height)\n    g.text(p.get_x()+p.get_width()/2.,\n            height + 3,\n            '{:1.2f}%'.format(height/total*100),\n            ha=\"center\", fontsize=14) \ng.set_ylim(0, max(sizes) * 1.15) # set y limit based on highest heights\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**On average users answer 64.33% questions correctly. -1 as null, for lectures, we should exclude them for answers analysis.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"id_col = ['user_id', 'content_id', 'content_type_id', 'task_container_id']\nplt.figure(figsize=(10,6))\nfor i, col in enumerate(id_col):\n    plt.subplot(2, 2, i + 1)\n    sns.distplot(train[col], color='green', \n                 hist_kws={'alpha':1,\"linewidth\": 2},\n                 kde_kws={\"color\": \"red\", \"lw\": 2, 'bw':0.01})\n    plt.title(col)\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"time_col = ['timestamp', 'prior_question_elapsed_time',]\nplt.figure(figsize=(10,6))\nfor i, col in enumerate(time_col):\n    plt.subplot(1, 2, i + 1)\n    train[col].hist(bins = 50,color='red')\n    plt.title(col)\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Timestamp represents the time from the first user interaction to the current one and Prior question elapsed time represents how long it took a user to answer their previous question bundle.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"col = ['prior_question_had_explanation', 'user_answer',]\n\ntotal = len(train)\nplt.figure(figsize=(12,5), dpi=60)\n\nfor i, col in enumerate(col):\n    plt.subplot(1, 2, i + 1)\n    g=sns.countplot(train[col], palette='cividis')\n    sizes=[] # Get highest values in y\n    for p in g.patches:\n        height = p.get_height()\n        sizes.append(height)\n        g.text(p.get_x()+p.get_width()/2.,\n                height + 3,\n                '{:1.2f}%'.format(height/total*100),\n                ha=\"center\", fontsize=14) \n    g.set_ylim(0, max(sizes) * 1.15) # set y limit based on highest heights\n    plt.title(col)\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ans_col = ['answered_correctly', 'user_answer',]\n\ntotal = len(train)\nplt.figure(figsize=(12,5), dpi=100)\n\nfor i, col in enumerate(ans_col):\n    plt.subplot(1, 2, i + 1)\n    sns.countplot(train[col], palette='cividis', hue = train['prior_question_had_explanation'])\n    plt.title(col)\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Majority user saw an explanation and the correct responses after answering the previous question bundle.**"},{"metadata":{},"cell_type":"markdown","source":"### Correct Answers by users\n\nsome code is taken from https://www.kaggle.com/ilialar/simple-eda-and-baseline https://www.kaggle.com/lgreig/simple-lgbm-baseline"},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain_only_df = train[train['answered_correctly']!=-1]\ngrouped_by_user_df = train_only_df.groupby('user_id')\nuser_answers_df = grouped_by_user_df.agg({'answered_correctly': ['mean', 'count', 'sum', 'std', 'median', 'skew']})\n\nfig,ax=plt.subplots(figsize=(15,8), dpi=100)\n\nplt.subplot(2, 2, 1)\ng1=user_answers_df[('answered_correctly','mean')].hist(bins=100, color='teal')\ng1.set_title(\"users correct answer mean dist.\", fontweight='bold')\n\nplt.subplot(2, 2, 2)\ng2=user_answers_df[('answered_correctly','count')].hist(bins=100, color='teal')\ng2.set_title('users correct answer count dist.', fontweight='bold')\n\nplt.subplot(2, 2, 3)\ng3=user_answers_df[user_answers_df[('answered_correctly','count')]<= 100][('answered_correctly','mean')].hist(bins=100, color='teal')\ng3.set_title('users correct answer mean dist. less than 100 question', fontweight='bold')\n\nplt.subplot(2, 2, 4)\ng4=user_answers_df[user_answers_df[('answered_correctly','count')]>=100][('answered_correctly','mean')].hist(bins=100, color='teal')\ng4.set_title('users correct answer count dist. more than 100 question', fontweight='bold')\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Average user score is lower than the overall % of correct answers(bottom left graph). It means heavy users have even better scores(bottom right graph).**"},{"metadata":{"trusted":true},"cell_type":"code","source":"user_time_answers_df = grouped_by_user_df.agg({'answered_correctly': ['mean', 'count'],\n                                        'timestamp': ['mean', 'count']})\n\nfig,ax=plt.subplots(figsize=(15,10), dpi=100)\n\nplt.subplot(3, 2, 1)\nplt.scatter(x = user_answers_df[('answered_correctly','count')], \n            y = user_answers_df[ ('answered_correctly','mean')], color='teal')\nplt.title('relation b/w correct answer mean and count',fontweight='bold')\n\nplt.subplot(3, 2, 2)\nplt.scatter(x = user_answers_df[('answered_correctly','std')], \n            y = user_answers_df[ ('answered_correctly','mean')], color='teal')\nplt.title('relation b/w correct answer mean and std',fontweight='bold')\n\nplt.subplot(3, 2, 3)\nplt.scatter(x = user_answers_df[('answered_correctly','median')], \n            y = user_answers_df[ ('answered_correctly','mean')], color='teal')\nplt.title('relation b/w correct answer mean and mediam',fontweight='bold')\n\nplt.subplot(3, 2, 4)\nplt.scatter(x = user_answers_df[('answered_correctly','skew')], \n            y = user_answers_df[ ('answered_correctly','mean')], color='teal')\nplt.title('relation b/w correct answer mean and skew',fontweight='bold')\n\nplt.subplot(3, 2, 5)\nplt.scatter(x = user_answers_df[('answered_correctly','sum')], \n            y = user_answers_df[ ('answered_correctly','mean')], color='teal')\nplt.title('relation b/w correct answer mean and sum',fontweight='bold')\n\nplt.subplot(3, 2, 6)\nplt.scatter(x = user_time_answers_df[ ('timestamp','mean')], \n            y = user_time_answers_df[ ('answered_correctly','mean')], color='teal')\nplt.title('relation b/w  timestamp mean and correct answer mean',fontweight='bold')\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**There is relationship between the average score for the active user, and the number of questions answered;  there is relation average timestamp and average correct answer can be useful for baseline.**"},{"metadata":{},"cell_type":"markdown","source":"### Correct Answers by Content"},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_by_content_df = train_only_df.groupby('content_id')\ncontent_answers_df = grouped_by_content_df.agg({'answered_correctly': ['mean', 'count', 'sum','std', 'median', 'skew']})\n\nfig,ax=plt.subplots(figsize=(15,8), dpi=100)\n\nplt.subplot(2, 2, 1)\ng1=content_answers_df[('answered_correctly','mean')].hist(bins=100, color='darkred')\ng1.set_title(\"content correct answer mean dist.\", fontweight='bold')\n\nplt.subplot(2, 2, 2)\ng2=content_answers_df[('answered_correctly','count')].hist(bins=100, color='darkred')\ng2.set_title('content answer count dist.', fontweight='bold')\n\nplt.subplot(2, 2, 3)\ng3=content_answers_df[content_answers_df[('answered_correctly','count')]<= 100][('answered_correctly','mean')].hist(bins=100, color='darkred')\ng3.set_title('content correct answer mean dist. less than 100 question', fontweight='bold')\n\nplt.subplot(2, 2, 4)\ng4=content_answers_df[content_answers_df[('answered_correctly','count')]>=100][('answered_correctly','mean')].hist(bins=100, color='darkred')\ng4.set_title('content correct answer count dist. more than 100 question', fontweight='bold')\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_time_content_df = grouped_by_content_df.agg({'answered_correctly': ['mean', 'count'],\n                                        'timestamp': ['mean', 'count']})\n\nfig,ax=plt.subplots(figsize=(15,10), dpi=100)\n\nplt.subplot(3, 2, 1)\nplt.scatter(x = content_answers_df[('answered_correctly','count')], \n            y=content_answers_df[ ('answered_correctly','mean')], color='darkred')\nplt.title('relation b/w correct answer mean and count', fontweight='bold')\n\nplt.subplot(3, 2, 2)\nplt.scatter(x = content_answers_df[('answered_correctly','std')], \n            y=content_answers_df[ ('answered_correctly','mean')], color='darkred')\nplt.title('relation b/w correct answer mean and std', fontweight='bold')\n\nplt.subplot(3, 2, 3)\nplt.scatter(x = content_answers_df[('answered_correctly','median')], \n            y=content_answers_df[ ('answered_correctly','mean')], color='darkred')\nplt.title('relation b/w correct answer mean and median', fontweight='bold')\n\nplt.subplot(3, 2, 4)\nplt.scatter(x = content_answers_df[('answered_correctly','skew')], \n            y=content_answers_df[ ('answered_correctly','mean')], color='darkred')\nplt.title('relation b/w correct answer mean and skew', fontweight='bold')\n\nplt.subplot(3, 2, 5)\nplt.scatter(x = content_answers_df[('answered_correctly','sum')], \n            y=content_answers_df[ ('answered_correctly','mean')], color='darkred')\nplt.title('relation b/w correct answer mean and sum', fontweight='bold')\n\nplt.subplot(3, 2, 6)\nplt.scatter(x = user_time_content_df[ ('timestamp','mean')], \n            y = user_time_content_df[ ('answered_correctly','mean')], color='darkred')\nplt.title('relation b/w  timestamp mean and correct answer count', fontweight='bold')\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Question.csv\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"description(questions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"id_col = ['question_id', 'bundle_id']\nplt.figure(figsize=(10,6))\nfor i, col in enumerate(id_col):\n    plt.subplot(1, 2, i + 1)\n    sns.distplot(questions[col], color='green',bins=100, \n                 hist_kws={'alpha':1,\"linewidth\": 1},\n                 kde_kws={\"color\": \"red\", \"lw\": 2, 'bw':0.01})\n    plt.title(col)\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df = questions.merge(content_answers_df, left_on = 'question_id', right_on = 'content_id', how = 'left')\nbundle_dict = questions_df['bundle_id'].value_counts().to_dict()\n\nquestions_df['right_answers'] = questions_df[('answered_correctly', 'mean')] * questions_df[('answered_correctly', 'count')]\nquestions_df['bundle_size'] = questions_df['bundle_id'].apply(lambda x: bundle_dict[x])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"col = ['correct_answer', 'part', 'bundle_size']\n\ntotal = len(questions_df)\nplt.figure(figsize=(15,8), dpi=100)\n\nfor i, col in enumerate(col):\n    plt.subplot(2, 2, i + 1)\n    g=sns.countplot(questions_df[col], palette='gist_yarg')\n    sizes=[] # Get highest values in y\n    for p in g.patches:\n        height = p.get_height()\n        sizes.append(height)\n        g.text(p.get_x()+p.get_width()/2.,\n                height + 3,\n                '{:1.2f}%'.format(height/total*100),\n                ha=\"center\", fontsize=14) \n    g.set_ylim(0, max(sizes) * 1.15) # set y limit based on highest heights\n    plt.title(col)\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig,ax=plt.subplots(figsize=(15,6), dpi=50)\nplt.subplot(1, 2, 1)\nplt.scatter(x = questions_df[('answered_correctly','count')], \n            y=questions_df['right_answers'], color='royalblue')\nplt.title('relation b/w right answer and question asked (count)', fontweight='bold')\n\nplt.subplot(1, 2, 2)\nplt.scatter(x = questions_df['right_answers'], \n            y = questions_df[ ('answered_correctly','mean')], color='royalblue')\nplt.title('relation b/w  right_answers and correct answer mean', fontweight='bold')\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_by_bundle_df = questions_df.groupby('bundle_id')\nbundle_answers_df = grouped_by_bundle_df.agg({'right_answers': 'sum', ('answered_correctly', 'count'): 'sum'}).copy()\nbundle_answers_df.columns = ['bundle_rignt_answers', 'bundle_questions_asked']\nbundle_answers_df['bundle_accuracy'] = bundle_answers_df['bundle_rignt_answers'] / bundle_answers_df['bundle_questions_asked']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bundle_answers_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig,ax=plt.subplots(figsize=(15,6), dpi=50)\nplt.subplot(1, 2, 1)\nplt.scatter(x = bundle_answers_df['bundle_questions_asked'], \n            y=bundle_answers_df['bundle_accuracy'], color='dodgerblue')\nplt.title('relation b/w bundle_questions_asked and bundle_accuracy', fontweight='bold')\n\nplt.subplot(1, 2, 2)\nplt.scatter(x = bundle_answers_df['bundle_rignt_answers'], \n            y = bundle_answers_df['bundle_accuracy'], color='dodgerblue')\nplt.title('relation b/w  bundle_rignt_answers and bundle_accuracy', fontweight='bold')\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_by_part_df = questions_df.groupby('part')\npart_answers_df = grouped_by_part_df.agg({'right_answers': 'sum', ('answered_correctly', 'count'): 'sum'}).copy()\npart_answers_df.columns = ['part_rignt_answers', 'part_questions_asked']\npart_answers_df['part_accuracy'] = part_answers_df['part_rignt_answers'] / part_answers_df['part_questions_asked']\npart_answers_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Lectures.csv"},{"metadata":{"trusted":true},"cell_type":"code","source":"description(lectures)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"col = ['type_of', 'part']\n\ntotal = len(lectures)\nplt.figure(figsize=(15,8), dpi=100)\n\nfor i, col in enumerate(col):\n    plt.subplot(2, 2, i + 1)\n    g=sns.countplot(lectures[col], palette='RdGy')\n    sizes=[] # Get highest values in y\n    for p in g.patches:\n        height = p.get_height()\n        sizes.append(height)\n        g.text(p.get_x()+p.get_width()/2.,\n                height + 3,\n                '{:1.2f}%'.format(height/total*100),\n                ha=\"center\", fontsize=14) \n    g.set_ylim(0, max(sizes) * 1.15) # set y limit based on highest heights\n    plt.title(col)\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Feature Engineering"},{"metadata":{},"cell_type":"markdown","source":"**Let's use this new feature in our baseline model**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train[train['answered_correctly']!=-1]\ngrouped_by_user_df = train_df.groupby('user_id')\nuser_answers_df = grouped_by_user_df.agg({'answered_correctly': ['mean', 'count', 'sum','std', 'median', 'skew']}).copy()\nuser_answers_df.columns = ['mean_user_accuracy', 'questions_answered', 'sum_user_accuracy', 'std_user_accuracy', 'median_user_accuracy', 'skew_user_accuracy']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_by_content_df = train_df.groupby('content_id')\ncontent_answers_df = grouped_by_content_df.agg({'answered_correctly': ['mean', 'count', 'sum', 'std', 'median', 'skew'] }).copy()\ncontent_answers_df.columns = ['mean_accuracy', 'question_asked', 'sum_accuracy', 'std_accuracy', 'median_accuracy', 'skew_accuracy']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df = questions.merge(content_answers_df, left_on = 'question_id', right_on = 'content_id', how = 'left')\nbundle_dict = questions_df['bundle_id'].value_counts().to_dict()\n\nquestions_df['right_answers'] = questions_df['mean_accuracy'] * questions_df['question_asked']\nquestions_df['bundle_size'] =questions_df['bundle_id'].apply(lambda x: bundle_dict[x])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_by_bundle_df = questions_df.groupby('bundle_id')\nbundle_answers_df = grouped_by_bundle_df.agg({'right_answers': 'sum', 'question_asked': 'sum'}).copy()\nbundle_answers_df.columns = ['bundle_rignt_answers', 'bundle_questions_asked']\nbundle_answers_df['bundle_accuracy'] = bundle_answers_df['bundle_rignt_answers'] / bundle_answers_df['bundle_questions_asked']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_by_part_df = questions_df.groupby('part')\npart_answers_df = grouped_by_part_df.agg({'right_answers': 'sum', 'question_asked': 'sum'}).copy()\npart_answers_df.columns = ['part_rignt_answers', 'part_questions_asked']\npart_answers_df['part_accuracy'] = part_answers_df['part_rignt_answers'] / part_answers_df['part_questions_asked']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lectures_df = lectures.groupby('part')\nlectures_df = lectures_df.agg({'type_of': 'count'})\nlectures_df.columns = ['type_of_count']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_train_df = train_df.merge(user_answers_df, how = 'left', on = 'user_id')\\\n                        .merge(questions_df, how = 'left', left_on = 'content_id', right_on = 'question_id')\\\n                        .merge(bundle_answers_df, how = 'left', on = 'bundle_id')\\\n                        .merge(part_answers_df, how = 'left', on = 'part')\\\n                        .merge(lectures_df, how='left',on='part')\ndel train\ndel questions\ndel lectures\ndel train_only_df\ndel grouped_by_user_df\ndel grouped_by_content_df\ndel grouped_by_bundle_df\ndel grouped_by_part_df\n\nimport gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_train_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"le = LabelEncoder()\nnew_train_df['prior_question_had_explanation'] = new_train_df['prior_question_had_explanation'].fillna(value = False).astype(bool)\nnew_train_df[\"prior_question_had_explanation\"] = le.fit_transform(new_train_df[\"prior_question_had_explanation\"])\nnew_train_df.fillna(0, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#https://github.com/dabl/dabl\nplt.rcParams['figure.figsize'] = (18, 6)\nplt.style.use('fivethirtyeight')\ndabl.plot(new_train_df, target_col = 'answered_correctly')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"corr = new_train_df.corr()\nmask = np.zeros_like(corr)\nmask[np.triu_indices_from(mask)] = True\nwith sns.axes_style(\"white\"):\n    f, ax = plt.subplots(figsize=(25, 25))\n    ax = sns.heatmap(corr,mask=mask,square=True,linewidths=.5,cmap=\"coolwarm\",annot=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## Baseline Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"features = ['timestamp', 'sum_user_accuracy',\n       'prior_question_elapsed_time', 'prior_question_had_explanation',\n       'mean_user_accuracy', 'questions_answered', 'std_user_accuracy',\n       'median_user_accuracy', 'skew_user_accuracy', 'correct_answer', 'mean_accuracy',\n       'question_asked', 'sum_accuracy','std_accuracy', 'median_accuracy', 'skew_accuracy',\n       'right_answers', 'bundle_size', 'bundle_rignt_answers',  \n       'bundle_questions_asked', 'bundle_accuracy', 'part_rignt_answers',\n       'part_questions_asked', 'part_accuracy', 'type_of_count']\n\ntarget = 'answered_correctly'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_train_df = new_train_df.sort_values(['user_id'])\n\ny = new_train_df[target]\n\nX = new_train_df[features]\n\ndel new_train_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scores = []\nfeature_importance = pd.DataFrame()\nmodels = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params = {'num_leaves': 40,\n          'max_depth': 4,\n          'subsample':0.8,\n          'objective': 'binary',\n          'learning_rate': 0.001,\n          \"boosting_type\": \"gbdt\",\n          \"metric\": 'auc',\n          'n_estimators': 100,\n          'min_child_samples':30,\n          'num_parallel_tree': 1000,\n          'subsample_freq':15,\n          'n_jobs':-1,\n          'is_higher_better': True,\n          'first_metric_only': True\n         }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#https://www.kaggle.com/artgor/riiid-eda-feature-engineering-and-models\nfolds = StratifiedKFold(n_splits=5, shuffle=False)\nfor fold_n, (train_index, valid_index) in enumerate(folds.split(X, y)):\n    print(f'Fold {fold_n} started at {time.ctime()}')\n    X_train, X_valid = X[features].iloc[train_index], X[features].iloc[valid_index]\n    y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]\n    model = lgb.LGBMClassifier(**params)\n    model.fit(X_train, y_train, \n            eval_set=[(X_train, y_train), (X_valid, y_valid)], eval_metric='auc',\n            verbose=50, early_stopping_rounds=10)\n    score = max(model.evals_result_['valid_1']['auc'])\n    \n    models.append(model)\n    scores.append(score)\n\n    fold_importance = pd.DataFrame()\n    fold_importance[\"feature\"] = features\n    fold_importance[\"importance\"] = model.feature_importances_\n    fold_importance[\"fold\"] = fold_n + 1\n    feature_importance = pd.concat([feature_importance, fold_importance], axis=0)\n    break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_importance[\"importance\"] /= 1\ncols = feature_importance[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(\n    by=\"importance\", ascending=False)[:50].index\n\nbest_features = feature_importance.loc[feature_importance.feature.isin(cols)]\n\nplt.figure(figsize=(16, 12));\nsns.barplot(x=\"importance\", y=\"feature\", data=best_features.sort_values(by=\"importance\", ascending=False));\nplt.title('LGB Features (avg over folds)');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    y_preds = []\n    test_df = test_df.merge(user_answers_df, how = 'left', on = 'user_id')\\\n                        .merge(questions_df, how = 'left', left_on = 'content_id', right_on = 'question_id')\\\n                        .merge(bundle_answers_df, how = 'left', on = 'bundle_id')\\\n                        .merge(part_answers_df, how = 'left', on = 'part')\\\n                        .merge(lectures_df, how='left',on='part')\n    test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(value = False).astype(bool)\n    \n    test_df.fillna(-1, inplace = True)\n    test_df[\"prior_question_had_explanation_enc\"] = le.fit_transform(test_df[\"prior_question_had_explanation\"])\n\n    for model in models:\n        y_pred = model.predict_proba(test_df[features], num_iteration=model.best_iteration_)[:, 1]\n        y_preds.append(y_pred)\n\n    y_preds = sum(y_preds) / len(y_preds)\n    test_df['answered_correctly'] = y_preds\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# params = {              'n_estimators': 100,\n#                         'seed': 44,\n#                         'colsample_bytree': 0.8,\n#                         'subsample': 0.7,\n#                         'learning_rate': 0.01,\n#                         'objective': 'binary:logistic',\n#                         'max_depth': 5,\n#                         'num_parallel_tree': 1000,\n#                         'min_child_weight': 20,\n#                         'eval_metric':'auc',\n#                         'gamma':0.1,\n#                         'tree_method':'gpu_hist'}\n\n# model = XGBClassifier(**params)\n# eval_set = [(X_train, y_train), (X_valid, y_valid)]\n# model.fit(X_train, y_train, early_stopping_rounds=20, eval_metric=\"auc\", eval_set=eval_set, verbose=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # plot AUC\n# results = model.evals_result()\n# epochs = len(results['validation_0']['auc'])\n# x_axis = range(0, epochs)\n# fig, ax = plt.subplots(figsize=(8,5))\n# ax.plot(x_axis, results['validation_0']['auc'], label='Train')\n# ax.plot(x_axis, results['validation_1']['auc'], label='Test')\n# ax.legend()\n# plt.ylabel('AUC')\n# plt.title('XGBoost AUC')\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from xgboost import plot_importance\n# fig,ax=plt.subplots(figsize=(8,5))\n# plot_importance(model, color='red', height=0.5,ax=ax, importance_type='weight')\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from xgboost import plot_importance\n# fig,ax=plt.subplots(figsize=(8,5))\n# plot_importance(model, color='red', height=0.5,ax=ax, importance_type='gain')\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}