{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2020-12-22T06:19:45.572572Z","iopub.status.busy":"2020-12-22T06:19:45.571779Z","iopub.status.idle":"2020-12-22T06:19:45.590808Z","shell.execute_reply":"2020-12-22T06:19:45.590102Z"},"papermill":{"duration":0.095101,"end_time":"2020-12-22T06:19:45.590943","exception":false,"start_time":"2020-12-22T06:19:45.495842","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.068326,"end_time":"2020-12-22T06:19:45.727361","exception":false,"start_time":"2020-12-22T06:19:45.659035","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Riiid! Answer Correctness Prediction"},{"metadata":{"papermill":{"duration":0.066454,"end_time":"2020-12-22T06:19:45.860905","exception":false,"start_time":"2020-12-22T06:19:45.794451","status":"completed"},"tags":[]},"cell_type":"markdown","source":"The main target of this notebook is giving the base understanding of our data and some useful features.\n\nFirst of all you can find here:\n\nComprehensive description of our data\n\nFeature Engineering\n\nBaseline with LightGBM without serious tuning model's parameters"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:19:46.014466Z","iopub.status.busy":"2020-12-22T06:19:46.013704Z","iopub.status.idle":"2020-12-22T06:19:55.571144Z","shell.execute_reply":"2020-12-22T06:19:55.570356Z"},"papermill":{"duration":9.642826,"end_time":"2020-12-22T06:19:55.571274","exception":false,"start_time":"2020-12-22T06:19:45.928448","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nimport warnings\nwarnings.simplefilter('ignore')\n\nimport plotly.express as px\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objs as go\n\nimport optuna\nfrom optuna.samplers import TPESampler\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import train_test_split\nfrom lightgbm import LGBMClassifier\n\nimport pandas as pd\nfrom pandas.plotting import scatter_matrix\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nwarnings.simplefilter('ignore')\nfrom sklearn.metrics import roc_auc_score, confusion_matrix\nfrom sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV, learning_curve\nfrom sklearn.utils import shuffle\nimport lightgbm as lgb\nfrom lightgbm import LGBMClassifier\nimport eli5\n\nfrom catboost import CatBoostClassifier\nimport xgboost as xgb\n\nimport riiideducation\n\n%matplotlib inline\n# for heatmap and other plots\ncolorMap1 = sns.color_palette(\"RdBu_r\")\n# for countplot and others plots\ncolorMap2 = 'Blues_r'\n\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:19:55.718532Z","iopub.status.busy":"2020-12-22T06:19:55.717529Z","iopub.status.idle":"2020-12-22T06:19:55.720825Z","shell.execute_reply":"2020-12-22T06:19:55.720164Z"},"papermill":{"duration":0.081918,"end_time":"2020-12-22T06:19:55.720964","exception":false,"start_time":"2020-12-22T06:19:55.639046","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"sampler = TPESampler(\n    seed=666\n)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:19:55.864538Z","iopub.status.busy":"2020-12-22T06:19:55.863445Z","iopub.status.idle":"2020-12-22T06:19:55.866881Z","shell.execute_reply":"2020-12-22T06:19:55.866288Z"},"papermill":{"duration":0.077549,"end_time":"2020-12-22T06:19:55.867012","exception":false,"start_time":"2020-12-22T06:19:55.789463","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"types = {\n        'row_id': 'int64', \n        'timestamp': 'int64', \n        'user_id': 'int32', \n        'content_id': 'int16', \n        'content_type_id': 'int8',\n        'task_container_id': 'int16', \n        'user_answer': 'int8', \n        'answered_correctly': 'int8', \n        'prior_question_elapsed_time': 'float32', \n        'prior_question_had_explanation': 'boolean'\n}","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:19:56.012104Z","iopub.status.busy":"2020-12-22T06:19:56.011296Z","iopub.status.idle":"2020-12-22T06:19:59.798887Z","shell.execute_reply":"2020-12-22T06:19:59.797573Z"},"papermill":{"duration":3.863422,"end_time":"2020-12-22T06:19:59.799026","exception":false,"start_time":"2020-12-22T06:19:55.935604","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\n    '/kaggle/input/riiid-test-answer-prediction/train.csv', \n    low_memory=False, \n    nrows=10**6, \n    dtype=types\n)\nquestions=pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')\nlectures=pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv')\ntest=pd.read_csv('/kaggle/input/riiid-test-answer-prediction/example_test.csv')\n\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.082971,"end_time":"2020-12-22T06:19:59.969001","exception":false,"start_time":"2020-12-22T06:19:59.88603","status":"completed"},"tags":[]},"cell_type":"markdown","source":"#                  DATA EXPLORATION & EDA"},{"metadata":{"papermill":{"duration":0.068312,"end_time":"2020-12-22T06:20:00.105649","exception":false,"start_time":"2020-12-22T06:20:00.037337","status":"completed"},"tags":[]},"cell_type":"markdown","source":"**TRAIN**"},{"metadata":{"papermill":{"duration":0.06852,"end_time":"2020-12-22T06:20:00.242031","exception":false,"start_time":"2020-12-22T06:20:00.173511","status":"completed"},"tags":[]},"cell_type":"markdown","source":"**row_id**: (int64) ID code for the row.\n\n**timestamp**: (int64) the time in milliseconds between this user interaction and the first event completion from that user.\n\n**user_id**: (int32) ID code for the user.\n\n**content_id**: (int16) ID code for the user interaction\n\n**content_type_id**: (int8) 0 if the event was a question being posed to the user, 1 if the event was the user watching a lecture.\n\n**task_container_id**: (int16) Id code for the batch of questions or lectures. For example, a user might see three questions in a row before seeing the explanations for any of them. Those three would all share a task_container_id.\n\n**user_answer**: (int8) the user's answer to the question, if any. Read -1 as null, for lectures.\n\n**answered_correctly**: (int8) if the user responded correctly. Read -1 as null, for lectures.\n\n**prior_question_elapsed_time**: (float32) The average time in milliseconds it took a user to answer each question in the previous question bundle, ignoring any lectures\nin between. Is null for a user's first question bundle or lecture. Note that the time is the average time a user took to solve each question in the previous bundle.\n\n**prior_question_had_explanation**: (bool) Whether or not the user saw an explanation and the correct response(s) after answering the previous question bundle, ignoring any lectures in between. The value is shared across a single question bundle, and is null for a user's first question bundle or lecture. Typically the first several questions a user sees were part of an onboarding diagnostic test where they did not get any feedback."},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:00.384127Z","iopub.status.busy":"2020-12-22T06:20:00.38335Z","iopub.status.idle":"2020-12-22T06:20:00.387216Z","shell.execute_reply":"2020-12-22T06:20:00.386212Z"},"papermill":{"duration":0.076952,"end_time":"2020-12-22T06:20:00.387336","exception":false,"start_time":"2020-12-22T06:20:00.310384","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"print(f\"Train shape: {train.shape}\")","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:00.540425Z","iopub.status.busy":"2020-12-22T06:20:00.539678Z","iopub.status.idle":"2020-12-22T06:20:00.546192Z","shell.execute_reply":"2020-12-22T06:20:00.545554Z"},"papermill":{"duration":0.090099,"end_time":"2020-12-22T06:20:00.54631","exception":false,"start_time":"2020-12-22T06:20:00.456211","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:00.712023Z","iopub.status.busy":"2020-12-22T06:20:00.699354Z","iopub.status.idle":"2020-12-22T06:20:01.074718Z","shell.execute_reply":"2020-12-22T06:20:01.073974Z"},"papermill":{"duration":0.458964,"end_time":"2020-12-22T06:20:01.074842","exception":false,"start_time":"2020-12-22T06:20:00.615878","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train.describe().style.background_gradient(cmap='Blues')","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:01.223084Z","iopub.status.busy":"2020-12-22T06:20:01.222291Z","iopub.status.idle":"2020-12-22T06:20:01.242659Z","shell.execute_reply":"2020-12-22T06:20:01.242014Z"},"papermill":{"duration":0.096718,"end_time":"2020-12-22T06:20:01.242779","exception":false,"start_time":"2020-12-22T06:20:01.146061","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"print(f'Number of unique users: {len(np.unique(train.user_id))}')\n","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.070844,"end_time":"2020-12-22T06:20:01.385006","exception":false,"start_time":"2020-12-22T06:20:01.314162","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Lets look at the missing data"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:01.533943Z","iopub.status.busy":"2020-12-22T06:20:01.533162Z","iopub.status.idle":"2020-12-22T06:20:01.563176Z","shell.execute_reply":"2020-12-22T06:20:01.562011Z"},"papermill":{"duration":0.107271,"end_time":"2020-12-22T06:20:01.563362","exception":false,"start_time":"2020-12-22T06:20:01.456091","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"print(train.isnull().sum()/len(train))","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.072552,"end_time":"2020-12-22T06:20:01.709473","exception":false,"start_time":"2020-12-22T06:20:01.636921","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Also let's check a correlation matrix to get more information between the columns"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:01.862402Z","iopub.status.busy":"2020-12-22T06:20:01.861138Z","iopub.status.idle":"2020-12-22T06:20:02.181869Z","shell.execute_reply":"2020-12-22T06:20:02.182442Z"},"papermill":{"duration":0.399669,"end_time":"2020-12-22T06:20:02.182628","exception":false,"start_time":"2020-12-22T06:20:01.782959","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"corr_matrix=train.corr()\ncorr_matrix['answered_correctly'].sort_values(ascending=True)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:02.334174Z","iopub.status.busy":"2020-12-22T06:20:02.333323Z","iopub.status.idle":"2020-12-22T06:20:03.069634Z","shell.execute_reply":"2020-12-22T06:20:03.07024Z"},"papermill":{"duration":0.814909,"end_time":"2020-12-22T06:20:03.070393","exception":false,"start_time":"2020-12-22T06:20:02.255484","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(13,10))\nsns.heatmap(corr_matrix,annot=True,\n           linewidths=5,cmap=colorMap1)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.088328,"end_time":"2020-12-22T06:20:03.236447","exception":false,"start_time":"2020-12-22T06:20:03.148119","status":"completed"},"tags":[]},"cell_type":"markdown","source":"\n\n\n\n\n\nLet's check the distribution of **prior_question_elapsed_time** "},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:03.402731Z","iopub.status.busy":"2020-12-22T06:20:03.401929Z","iopub.status.idle":"2020-12-22T06:20:23.154202Z","shell.execute_reply":"2020-12-22T06:20:23.15353Z"},"papermill":{"duration":19.836322,"end_time":"2020-12-22T06:20:23.154331","exception":false,"start_time":"2020-12-22T06:20:03.318009","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15, 10))\nax = sns.countplot(x=\"prior_question_elapsed_time\", \n                   data=train[train['prior_question_elapsed_time'].notnull()],\n                   palette=colorMap2)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.076485,"end_time":"2020-12-22T06:20:23.306819","exception":false,"start_time":"2020-12-22T06:20:23.230334","status":"completed"},"tags":[]},"cell_type":"markdown","source":"\nLet's check any connection between our target value and a frequency of answering questions"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:23.471474Z","iopub.status.busy":"2020-12-22T06:20:23.470714Z","iopub.status.idle":"2020-12-22T06:20:23.63687Z","shell.execute_reply":"2020-12-22T06:20:23.636194Z"},"papermill":{"duration":0.254214,"end_time":"2020-12-22T06:20:23.637003","exception":false,"start_time":"2020-12-22T06:20:23.382789","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"freq_answered_tasks = train['task_container_id'].value_counts().reset_index()\nfreq_answered_tasks.columns = [\n    'task_container_id', \n    'freq'\n]\ntrain['freq_task_id'] = ''\ntrain.loc[train['task_container_id'].isin(freq_answered_tasks[freq_answered_tasks['freq'] < 10000]['task_container_id'].values), 'freq_task_id'] = 'very rare answered'\ntrain.loc[train['task_container_id'].isin(freq_answered_tasks[freq_answered_tasks['freq'] >= 10000]['task_container_id'].values), 'freq_task_id'] = 'rare answered'\ntrain.loc[train['task_container_id'].isin(freq_answered_tasks[freq_answered_tasks['freq'] >= 50000]['task_container_id'].values), 'freq_task_id'] = 'normal answered'\ntrain.loc[train['task_container_id'].isin(freq_answered_tasks[freq_answered_tasks['freq'] >= 200000]['task_container_id'].values), 'freq_task_id'] = 'often answered'\ntrain.loc[train['task_container_id'].isin(freq_answered_tasks[freq_answered_tasks['freq'] >= 400000]['task_container_id'].values), 'freq_task_id'] = 'very often answered'\n\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:23.795587Z","iopub.status.busy":"2020-12-22T06:20:23.794847Z","iopub.status.idle":"2020-12-22T06:20:23.843574Z","shell.execute_reply":"2020-12-22T06:20:23.844144Z"},"papermill":{"duration":0.130413,"end_time":"2020-12-22T06:20:23.844306","exception":false,"start_time":"2020-12-22T06:20:23.713893","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train.sample(5)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:24.068253Z","iopub.status.busy":"2020-12-22T06:20:24.067424Z","iopub.status.idle":"2020-12-22T06:20:24.706395Z","shell.execute_reply":"2020-12-22T06:20:24.705682Z"},"papermill":{"duration":0.784326,"end_time":"2020-12-22T06:20:24.706511","exception":false,"start_time":"2020-12-22T06:20:23.922185","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15, 10))\nsns.countplot(x='freq_task_id', hue='answered_correctly', data=train, palette=colorMap2)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:24.870843Z","iopub.status.busy":"2020-12-22T06:20:24.869861Z","iopub.status.idle":"2020-12-22T06:20:24.87305Z","shell.execute_reply":"2020-12-22T06:20:24.87233Z"},"papermill":{"duration":0.088257,"end_time":"2020-12-22T06:20:24.873176","exception":false,"start_time":"2020-12-22T06:20:24.784919","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"WIDTH=800","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:25.041154Z","iopub.status.busy":"2020-12-22T06:20:25.040314Z","iopub.status.idle":"2020-12-22T06:20:26.158744Z","shell.execute_reply":"2020-12-22T06:20:26.159328Z"},"papermill":{"duration":1.206698,"end_time":"2020-12-22T06:20:26.15949","exception":false,"start_time":"2020-12-22T06:20:24.952792","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"ds = train['answered_correctly'].value_counts().reset_index()\n\nds.columns = [\n    'answered_correctly', \n    'percent_of_answers'\n]\n\nds['percent_of_answers'] /= len(train)\nds = ds.sort_values(['percent_of_answers'])\n\nfig = px.pie(\n    ds, \n    names='answered_correctly', \n    values='percent_of_answers', \n    title='Percent of correct answers', \n    width=WIDTH,\n    height=500 \n)\n\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.083314,"end_time":"2020-12-22T06:20:26.324143","exception":false,"start_time":"2020-12-22T06:20:26.240829","status":"completed"},"tags":[]},"cell_type":"markdown","source":"prior_question_had_explanation has a medium correlation with the target value. So let's look at his distribution\n\n"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:26.493755Z","iopub.status.busy":"2020-12-22T06:20:26.492967Z","iopub.status.idle":"2020-12-22T06:20:27.110598Z","shell.execute_reply":"2020-12-22T06:20:27.109848Z"},"papermill":{"duration":0.705806,"end_time":"2020-12-22T06:20:27.11074","exception":false,"start_time":"2020-12-22T06:20:26.404934","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15, 11))\nax = sns.countplot(x=\"prior_question_had_explanation\", hue=\"answered_correctly\", data=train[train['prior_question_had_explanation'].notnull()], palette=colorMap2)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.083349,"end_time":"2020-12-22T06:20:27.275858","exception":false,"start_time":"2020-12-22T06:20:27.192509","status":"completed"},"tags":[]},"cell_type":"markdown","source":"The answer's showing increases the probability of a successful answering. Let's go further.\nCheck the most active user_id\n\n"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:27.44959Z","iopub.status.busy":"2020-12-22T06:20:27.44884Z","iopub.status.idle":"2020-12-22T06:20:27.887541Z","shell.execute_reply":"2020-12-22T06:20:27.88694Z"},"papermill":{"duration":0.530035,"end_time":"2020-12-22T06:20:27.887696","exception":false,"start_time":"2020-12-22T06:20:27.357661","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"N=30\n\nuser_freq = train['user_id'].value_counts().reset_index()\nuser_freq.columns = [\n    'user_id', \n    'count'\n]\n\n# Add ' - ' to convert user_id to str and not sort\nuser_freq['user_id'] = user_freq['user_id'].astype(str) + ' - '\nuser_freq = user_freq.sort_values(['count'], ascending=False).head(N)\n\nplt.figure(figsize=(15, 15))\nsns.barplot(x='count', y='user_id', data=user_freq, orient='h', palette=colorMap2)\nplt.title(f'Top {N} the most active users', fontsize=14)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.084018,"end_time":"2020-12-22T06:20:28.055969","exception":false,"start_time":"2020-12-22T06:20:27.971951","status":"completed"},"tags":[]},"cell_type":"markdown","source":"And the most useful content_id"},{"metadata":{"papermill":{"duration":0.083504,"end_time":"2020-12-22T06:20:28.223906","exception":false,"start_time":"2020-12-22T06:20:28.140402","status":"completed"},"tags":[]},"cell_type":"markdown","source":""},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:28.399827Z","iopub.status.busy":"2020-12-22T06:20:28.399052Z","iopub.status.idle":"2020-12-22T06:20:28.403034Z","shell.execute_reply":"2020-12-22T06:20:28.402087Z"},"papermill":{"duration":0.09409,"end_time":"2020-12-22T06:20:28.403205","exception":false,"start_time":"2020-12-22T06:20:28.309115","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"WIDTH=800","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.137642,"end_time":"2020-12-22T06:20:28.625373","exception":false,"start_time":"2020-12-22T06:20:28.487731","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Also we need to check distribution in content_type_id: number of video lectures and questions"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:28.802095Z","iopub.status.busy":"2020-12-22T06:20:28.801293Z","iopub.status.idle":"2020-12-22T06:20:28.865926Z","shell.execute_reply":"2020-12-22T06:20:28.86525Z"},"papermill":{"duration":0.156759,"end_time":"2020-12-22T06:20:28.866048","exception":false,"start_time":"2020-12-22T06:20:28.709289","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"ds = train['content_type_id'].value_counts().reset_index()\n\nds.columns = [\n    'content_type_id', \n    'percent'\n]\n\nds['percent'] /=len(train)\n\nfig = px.pie(\n    ds, \n    names='content_type_id', \n    values='percent', \n    title='Lecures & questions', \n    width=WIDTH,\n    height=500 \n)\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:29.04647Z","iopub.status.busy":"2020-12-22T06:20:29.045674Z","iopub.status.idle":"2020-12-22T06:20:29.150126Z","shell.execute_reply":"2020-12-22T06:20:29.150765Z"},"papermill":{"duration":0.198585,"end_time":"2020-12-22T06:20:29.150935","exception":false,"start_time":"2020-12-22T06:20:28.95235","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"ds=train['user_answer'].value_counts().reset_index()\nds.columns = [\n    'user_answer', \n    'percent_of_answers'\n]\nds['percent_of_answers']/=len(train)\nds = ds.sort_values(['percent_of_answers'])\nfig = px.bar(\n    ds, \n    x='user_answer', \n    y='percent_of_answers', \n    orientation='v', \n    title='Percent of user answers for every option', \n    width=WIDTH,\n    height=400 \n)\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.087359,"end_time":"2020-12-22T06:20:29.325281","exception":false,"start_time":"2020-12-22T06:20:29.237922","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:29.509958Z","iopub.status.busy":"2020-12-22T06:20:29.509109Z","iopub.status.idle":"2020-12-22T06:20:55.033666Z","shell.execute_reply":"2020-12-22T06:20:55.032785Z"},"papermill":{"duration":25.621842,"end_time":"2020-12-22T06:20:55.033801","exception":false,"start_time":"2020-12-22T06:20:29.411959","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"task_ids_freq = train['task_container_id'].value_counts().reset_index()\ntask_ids_freq.columns = ['task_container_id', 'count']\n\nfig, ax = plt.subplots(figsize=(15, 10))\n\nsns.pointplot(x='task_container_id', y='count', data=task_ids_freq, palette=colorMap2)\nxticks_range = range(min(task_ids_freq['task_container_id']), \n                     max(task_ids_freq['task_container_id']),\n                     1000)\nplt.xticks(list(xticks_range), list(xticks_range))","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.087281,"end_time":"2020-12-22T06:20:55.209977","exception":false,"start_time":"2020-12-22T06:20:55.122696","status":"completed"},"tags":[]},"cell_type":"markdown","source":"**QUESTIONS.CSV**"},{"metadata":{"papermill":{"duration":0.087894,"end_time":"2020-12-22T06:20:55.392451","exception":false,"start_time":"2020-12-22T06:20:55.304557","status":"completed"},"tags":[]},"cell_type":"markdown","source":"\nquestion_id: foreign key for the train/test content_id column, when the content type is question (0).\n\nbundle_id: code for which questions are served together.\n\ncorrect_answer: the answer to the question. Can be compared with the train user_answer column to check if the user was right.\n\npart: top level category code for the question.\n\ntags: one or more detailed tag codes for the question. The meaning of the tags will not be provided, but these codes are sufficient for clustering the questions together."},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:55.581158Z","iopub.status.busy":"2020-12-22T06:20:55.580029Z","iopub.status.idle":"2020-12-22T06:20:55.584577Z","shell.execute_reply":"2020-12-22T06:20:55.584007Z"},"papermill":{"duration":0.104382,"end_time":"2020-12-22T06:20:55.584724","exception":false,"start_time":"2020-12-22T06:20:55.480342","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"questions.head()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:55.772163Z","iopub.status.busy":"2020-12-22T06:20:55.771317Z","iopub.status.idle":"2020-12-22T06:20:55.805196Z","shell.execute_reply":"2020-12-22T06:20:55.804425Z"},"papermill":{"duration":0.131601,"end_time":"2020-12-22T06:20:55.805328","exception":false,"start_time":"2020-12-22T06:20:55.673727","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"questions.describe().style.background_gradient(cmap='Blues')","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:55.993484Z","iopub.status.busy":"2020-12-22T06:20:55.992693Z","iopub.status.idle":"2020-12-22T06:20:56.000072Z","shell.execute_reply":"2020-12-22T06:20:55.999429Z"},"papermill":{"duration":0.104441,"end_time":"2020-12-22T06:20:56.000195","exception":false,"start_time":"2020-12-22T06:20:55.895754","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"print(questions.isnull().sum() / len(questions))","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:56.206602Z","iopub.status.busy":"2020-12-22T06:20:56.191379Z","iopub.status.idle":"2020-12-22T06:20:56.295591Z","shell.execute_reply":"2020-12-22T06:20:56.294924Z"},"papermill":{"duration":0.205138,"end_time":"2020-12-22T06:20:56.295734","exception":false,"start_time":"2020-12-22T06:20:56.090596","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"\n\n\n\n\nquestions['tag'] = questions['tags'].str.split(' ')\nquestions = questions.explode('tag')\nquestions = pd.merge(\n    questions, \n    questions.groupby('question_id')['tag'].count().reset_index(), \n    on='question_id'\n    )\n\nquestions = questions.drop(['tag_x'], axis=1)\n\nquestions.columns = [\n    'question_id', \n    'bundle_id', \n    'correct_answer', \n    'part', \n    'tags', \n    'tags_number'\n]\nquestions = questions.drop_duplicates()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:56.487115Z","iopub.status.busy":"2020-12-22T06:20:56.485393Z","iopub.status.idle":"2020-12-22T06:20:56.551644Z","shell.execute_reply":"2020-12-22T06:20:56.550887Z"},"papermill":{"duration":0.165903,"end_time":"2020-12-22T06:20:56.551785","exception":false,"start_time":"2020-12-22T06:20:56.385882","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"ds = questions['correct_answer'].value_counts().reset_index()\n\nds.columns = [\n    'correct_answer', \n    'number_of_answers'\n]\n\nds['correct_answer'] = ds['correct_answer'].astype(str) + '-'\nds = ds.sort_values(['number_of_answers'])\n\nfig = px.bar(\n    ds, \n    x='number_of_answers', \n    y='correct_answer', \n    orientation='h', \n    title='Number of correct answers per group', \n    width=WIDTH,\n    height=300\n)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:56.746489Z","iopub.status.busy":"2020-12-22T06:20:56.744434Z","iopub.status.idle":"2020-12-22T06:20:56.809895Z","shell.execute_reply":"2020-12-22T06:20:56.809109Z"},"papermill":{"duration":0.166265,"end_time":"2020-12-22T06:20:56.810022","exception":false,"start_time":"2020-12-22T06:20:56.643757","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"ds = questions['part'].value_counts().reset_index()\n\nds.columns = [\n    'part', \n    'count'\n]\n\nds['part'] = ds['part'].astype(str) + '-'\nds = ds.sort_values(['count'])\n\nfig = px.bar(\n    ds, \n    x='count', \n    y='part', \n    orientation='h', \n    title='Parts distribution',\n    width=WIDTH,\n    height=400\n)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:57.031901Z","iopub.status.busy":"2020-12-22T06:20:57.016539Z","iopub.status.idle":"2020-12-22T06:20:57.071409Z","shell.execute_reply":"2020-12-22T06:20:57.070741Z"},"papermill":{"duration":0.168595,"end_time":"2020-12-22T06:20:57.071534","exception":false,"start_time":"2020-12-22T06:20:56.902939","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"ds = questions['tags_number'].value_counts().reset_index()\n\nds.columns = [\n    'tags_number', \n    'count'\n]\n\nds['tags_number'] = ds['tags_number'].astype(str) + '-'\nds = ds.sort_values(['tags_number'])\n\nfig = px.bar(\n    ds, \n    x='count', \n    y='tags_number', \n    orientation='h', \n    title='Number tags distribution', \n    width=WIDTH,\n    height=400 \n    )\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:57.308643Z","iopub.status.busy":"2020-12-22T06:20:57.307539Z","iopub.status.idle":"2020-12-22T06:20:57.406712Z","shell.execute_reply":"2020-12-22T06:20:57.405933Z"},"papermill":{"duration":0.241434,"end_time":"2020-12-22T06:20:57.406837","exception":false,"start_time":"2020-12-22T06:20:57.165403","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"\n\ncheck = questions['tags'].str.split(' ').explode('tags').reset_index()\ncheck = check['tags'].value_counts().reset_index()\n\ncheck.columns = [\n    'tag', \n    'count'\n]\n\ncheck['tag'] = check['tag'].astype(str) + '-'\ncheck = check.sort_values(['count']).tail(40)\n\nfig = px.bar(\n    check, \n    x='count', \n    y='tag', \n    orientation='h', \n     title='Top 40 most useful tags', \n    width=WIDTH,\n    height=900 \n)\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.094457,"end_time":"2020-12-22T06:20:57.597471","exception":false,"start_time":"2020-12-22T06:20:57.503014","status":"completed"},"tags":[]},"cell_type":"markdown","source":"\nLECTURES.CSV"},{"metadata":{"papermill":{"duration":0.094632,"end_time":"2020-12-22T06:20:57.785853","exception":false,"start_time":"2020-12-22T06:20:57.691221","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Metadata for the lectures watched by users as they progress in their education.\n\n**lecture_id**: foreign key for the train/test content_id column, when the content type is lecture (1).\n\n**part**: top level category code for the lecture.\n\n**tag**: one tag codes for the lecture. The meaning of the tags will not be provided, but these codes are sufficient for clustering the lectures together.\n\n**type_of**: brief description of the core purpose of the lecture"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:57.988458Z","iopub.status.busy":"2020-12-22T06:20:57.987514Z","iopub.status.idle":"2020-12-22T06:20:57.991896Z","shell.execute_reply":"2020-12-22T06:20:57.992588Z"},"papermill":{"duration":0.112526,"end_time":"2020-12-22T06:20:57.992776","exception":false,"start_time":"2020-12-22T06:20:57.88025","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"lectures.head(10)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:58.193311Z","iopub.status.busy":"2020-12-22T06:20:58.192303Z","iopub.status.idle":"2020-12-22T06:20:58.195968Z","shell.execute_reply":"2020-12-22T06:20:58.196589Z"},"papermill":{"duration":0.108631,"end_time":"2020-12-22T06:20:58.19681","exception":false,"start_time":"2020-12-22T06:20:58.088179","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"print('Part of missing values for every column')\nprint(lectures.isnull().sum() / len(lectures))","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:58.416026Z","iopub.status.busy":"2020-12-22T06:20:58.415004Z","iopub.status.idle":"2020-12-22T06:20:58.419906Z","shell.execute_reply":"2020-12-22T06:20:58.419146Z"},"papermill":{"duration":0.10986,"end_time":"2020-12-22T06:20:58.420037","exception":false,"start_time":"2020-12-22T06:20:58.310177","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"lectures['type_of'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.100595,"end_time":"2020-12-22T06:20:58.618873","exception":false,"start_time":"2020-12-22T06:20:58.518278","status":"completed"},"tags":[]},"cell_type":"markdown","source":"**Test.csv**"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:58.836076Z","iopub.status.busy":"2020-12-22T06:20:58.834952Z","iopub.status.idle":"2020-12-22T06:20:58.840323Z","shell.execute_reply":"2020-12-22T06:20:58.839512Z"},"papermill":{"duration":0.124694,"end_time":"2020-12-22T06:20:58.840455","exception":false,"start_time":"2020-12-22T06:20:58.715761","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.097663,"end_time":"2020-12-22T06:20:59.036072","exception":false,"start_time":"2020-12-22T06:20:58.938409","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.096955,"end_time":"2020-12-22T06:20:59.232485","exception":false,"start_time":"2020-12-22T06:20:59.13553","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.097191,"end_time":"2020-12-22T06:20:59.438988","exception":false,"start_time":"2020-12-22T06:20:59.341797","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# FEATURE ENGINEERING"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:59.651122Z","iopub.status.busy":"2020-12-22T06:20:59.650138Z","iopub.status.idle":"2020-12-22T06:20:59.65523Z","shell.execute_reply":"2020-12-22T06:20:59.654629Z"},"papermill":{"duration":0.119307,"end_time":"2020-12-22T06:20:59.65536","exception":false,"start_time":"2020-12-22T06:20:59.536053","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:20:59.861054Z","iopub.status.busy":"2020-12-22T06:20:59.860184Z","iopub.status.idle":"2020-12-22T06:20:59.864Z","shell.execute_reply":"2020-12-22T06:20:59.863177Z"},"papermill":{"duration":0.109136,"end_time":"2020-12-22T06:20:59.864137","exception":false,"start_time":"2020-12-22T06:20:59.755001","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"features_df = train.iloc[:int(9/10 * len(train))]\ntrain = train.iloc[int(9/10 * len(train)):]","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:00.074142Z","iopub.status.busy":"2020-12-22T06:21:00.072883Z","iopub.status.idle":"2020-12-22T06:21:00.882015Z","shell.execute_reply":"2020-12-22T06:21:00.881353Z"},"papermill":{"duration":0.916997,"end_time":"2020-12-22T06:21:00.882151","exception":false,"start_time":"2020-12-22T06:20:59.965154","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train_questions_only_df = features_df[features_df['answered_correctly']!=-1]\ngrouped_by_user_df = train_questions_only_df.groupby('user_id')\n\nuser_answers_df = grouped_by_user_df.agg(\n    {\n        'answered_correctly': [\n            'mean', \n            'count', \n            'std', \n            'median', \n            'skew'\n        ]\n    }\n).copy()\n\nuser_answers_df.columns = [\n    'mean_user_accuracy',\n    'questions_answered',\n    'std_user_accuracy', \n    'median_user_accuracy', \n    'skew_user_accuracy'\n]","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.098566,"end_time":"2020-12-22T06:21:01.080999","exception":false,"start_time":"2020-12-22T06:21:00.982433","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:01.293289Z","iopub.status.busy":"2020-12-22T06:21:01.291808Z","iopub.status.idle":"2020-12-22T06:21:03.427085Z","shell.execute_reply":"2020-12-22T06:21:03.428009Z"},"papermill":{"duration":2.248647,"end_time":"2020-12-22T06:21:03.428229","exception":false,"start_time":"2020-12-22T06:21:01.179582","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"grouped_by_content_df = train_questions_only_df.groupby('content_id')\ncontent_answers_df = grouped_by_content_df.agg(\n    {\n        'answered_correctly': [\n            'mean', \n            'count', \n            'std', \n            'median', \n            'skew'\n        ]\n    }\n).copy()\n\ncontent_answers_df.columns = [\n    'mean_accuracy', \n    'question_asked', \n    'std_accuracy', \n    'median_accuracy', \n    'skew_accuracy'\n]\ncontent_answers_df","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:03.641701Z","iopub.status.busy":"2020-12-22T06:21:03.640711Z","iopub.status.idle":"2020-12-22T06:21:03.643864Z","shell.execute_reply":"2020-12-22T06:21:03.643106Z"},"papermill":{"duration":0.108285,"end_time":"2020-12-22T06:21:03.643999","exception":false,"start_time":"2020-12-22T06:21:03.535714","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"del features_df\ndel grouped_by_user_df\ndel grouped_by_content_df","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:03.854918Z","iopub.status.busy":"2020-12-22T06:21:03.853858Z","iopub.status.idle":"2020-12-22T06:21:03.856673Z","shell.execute_reply":"2020-12-22T06:21:03.857408Z"},"papermill":{"duration":0.110229,"end_time":"2020-12-22T06:21:03.857561","exception":false,"start_time":"2020-12-22T06:21:03.747332","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"features = [\n    'mean_user_accuracy', \n    'questions_answered',\n    'std_user_accuracy', \n    'median_user_accuracy',\n    'skew_user_accuracy',\n    'mean_accuracy', \n    'question_asked',\n    'std_accuracy', \n    'median_accuracy',\n    'prior_question_elapsed_time', \n    'prior_question_had_explanation',\n    'skew_accuracy'\n]\n\ntarget = 'answered_correctly'","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:04.06481Z","iopub.status.busy":"2020-12-22T06:21:04.063662Z","iopub.status.idle":"2020-12-22T06:21:04.068538Z","shell.execute_reply":"2020-12-22T06:21:04.067961Z"},"papermill":{"duration":0.110198,"end_time":"2020-12-22T06:21:04.068691","exception":false,"start_time":"2020-12-22T06:21:03.958493","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"target","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:04.279734Z","iopub.status.busy":"2020-12-22T06:21:04.278561Z","iopub.status.idle":"2020-12-22T06:21:04.288762Z","shell.execute_reply":"2020-12-22T06:21:04.287988Z"},"papermill":{"duration":0.11842,"end_time":"2020-12-22T06:21:04.288883","exception":false,"start_time":"2020-12-22T06:21:04.170463","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train = train[train[target] != -1]","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:04.496417Z","iopub.status.busy":"2020-12-22T06:21:04.495591Z","iopub.status.idle":"2020-12-22T06:21:04.542563Z","shell.execute_reply":"2020-12-22T06:21:04.541778Z"},"papermill":{"duration":0.153342,"end_time":"2020-12-22T06:21:04.54271","exception":false,"start_time":"2020-12-22T06:21:04.389368","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train = train.merge(user_answers_df, how='left', on='user_id')\ntrain = train.merge(content_answers_df, how='left', on='content_id')","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:04.753939Z","iopub.status.busy":"2020-12-22T06:21:04.753118Z","iopub.status.idle":"2020-12-22T06:21:04.75749Z","shell.execute_reply":"2020-12-22T06:21:04.756902Z"},"papermill":{"duration":0.112168,"end_time":"2020-12-22T06:21:04.757636","exception":false,"start_time":"2020-12-22T06:21:04.645468","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"target","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:04.972709Z","iopub.status.busy":"2020-12-22T06:21:04.971913Z","iopub.status.idle":"2020-12-22T06:21:04.9941Z","shell.execute_reply":"2020-12-22T06:21:04.993398Z"},"papermill":{"duration":0.129506,"end_time":"2020-12-22T06:21:04.994223","exception":false,"start_time":"2020-12-22T06:21:04.864717","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"\ntrain['prior_question_had_explanation'] = train['prior_question_had_explanation'].fillna(value=False).astype(bool)\ndf = train.fillna(value=0.5)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:05.210691Z","iopub.status.busy":"2020-12-22T06:21:05.209795Z","iopub.status.idle":"2020-12-22T06:21:05.218066Z","shell.execute_reply":"2020-12-22T06:21:05.217243Z"},"papermill":{"duration":0.120745,"end_time":"2020-12-22T06:21:05.218202","exception":false,"start_time":"2020-12-22T06:21:05.097457","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"col_to_drop = set(train.columns.values.tolist()).difference(features + [target])\nfor col in col_to_drop:\n    del df[col]","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:05.456663Z","iopub.status.busy":"2020-12-22T06:21:05.455738Z","iopub.status.idle":"2020-12-22T06:21:05.472829Z","shell.execute_reply":"2020-12-22T06:21:05.472123Z"},"papermill":{"duration":0.141937,"end_time":"2020-12-22T06:21:05.472957","exception":false,"start_time":"2020-12-22T06:21:05.33102","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"df = df.replace([np.inf, -np.inf], np.nan)\ndf = df.fillna(0.5)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.102374,"end_time":"2020-12-22T06:21:05.679867","exception":false,"start_time":"2020-12-22T06:21:05.577493","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Modeling"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:05.895926Z","iopub.status.busy":"2020-12-22T06:21:05.894833Z","iopub.status.idle":"2020-12-22T06:21:05.927887Z","shell.execute_reply":"2020-12-22T06:21:05.926901Z"},"papermill":{"duration":0.146003,"end_time":"2020-12-22T06:21:05.928066","exception":false,"start_time":"2020-12-22T06:21:05.782063","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train_df, test_df, y_train, y_test = train_test_split(df[features], df[target], random_state=777, test_size=0.2)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:06.144125Z","iopub.status.busy":"2020-12-22T06:21:06.142997Z","iopub.status.idle":"2020-12-22T06:21:06.145879Z","shell.execute_reply":"2020-12-22T06:21:06.146477Z"},"papermill":{"duration":0.111516,"end_time":"2020-12-22T06:21:06.146662","exception":false,"start_time":"2020-12-22T06:21:06.035146","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# clf = LGBMClassifier(random_state=777)\n\n# params = {\n#     'n_estimators': [50, 150, 300],\n#     'max_depth': [3, 5, 10],\n#     'num_leaves': [5, 15, 30],\n#     'min_data_in_leaf': [5, 50, 100],\n#     'feature_fraction': [0.1, 0.5, 1.],\n#     'lambda': [0., 0.5, 1.],\n# }\n\n# cv = RandomizedSearchCV(clf, param_distributions=params, cv=5, n_iter=50, verbose=2)\n# cv.fit(df[features], df[target])\n\n# print(cv.best_params_)\n# print(cv.best_score_)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:06.362669Z","iopub.status.busy":"2020-12-22T06:21:06.361529Z","iopub.status.idle":"2020-12-22T06:21:06.36502Z","shell.execute_reply":"2020-12-22T06:21:06.364432Z"},"papermill":{"duration":0.112226,"end_time":"2020-12-22T06:21:06.365148","exception":false,"start_time":"2020-12-22T06:21:06.252922","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"params = {\n    'num_leaves': 30, \n    'n_estimators': 300, \n    'min_data_in_leaf': 100, \n    'max_depth': 5, \n    'lambda': 0.0, \n    'feature_fraction': 1.0\n}\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:06.57641Z","iopub.status.busy":"2020-12-22T06:21:06.575434Z","iopub.status.idle":"2020-12-22T06:21:07.782054Z","shell.execute_reply":"2020-12-22T06:21:07.783065Z"},"papermill":{"duration":1.315256,"end_time":"2020-12-22T06:21:07.783253","exception":false,"start_time":"2020-12-22T06:21:06.467997","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"model = LGBMClassifier(**params)\nmodel.fit(train_df, y_train)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:08.032048Z","iopub.status.busy":"2020-12-22T06:21:08.031218Z","iopub.status.idle":"2020-12-22T06:21:08.186655Z","shell.execute_reply":"2020-12-22T06:21:08.187507Z"},"papermill":{"duration":0.275371,"end_time":"2020-12-22T06:21:08.187708","exception":false,"start_time":"2020-12-22T06:21:07.912337","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"print('LGB ROC-AUC score: ', roc_auc_score(y_test.values, model.predict_proba(test_df)[:, 1]))","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:08.456711Z","iopub.status.busy":"2020-12-22T06:21:08.455522Z","iopub.status.idle":"2020-12-22T06:21:08.458893Z","shell.execute_reply":"2020-12-22T06:21:08.458255Z"},"papermill":{"duration":0.12058,"end_time":"2020-12-22T06:21:08.459015","exception":false,"start_time":"2020-12-22T06:21:08.338435","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"params_cat = {\n    'loss_function': 'Logloss',\n    'eval_metric': 'AUC',\n    'task_type': 'GPU' ,\n    'grow_policy': 'Lossguide',\n    'iterations': 2500,\n    'learning_rate': 4e-2,\n    'random_seed': 0,\n    'l2_leaf_reg': 1e-1,\n    'depth': 15,\n    'max_leaves': 10,\n    'border_count': 128,\n    'verbose': 50,\n}","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:08.759558Z","iopub.status.busy":"2020-12-22T06:21:08.744659Z","iopub.status.idle":"2020-12-22T06:21:09.453487Z","shell.execute_reply":"2020-12-22T06:21:09.452713Z"},"papermill":{"duration":0.88937,"end_time":"2020-12-22T06:21:09.453643","exception":false,"start_time":"2020-12-22T06:21:08.564273","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"model_cat =  CatBoostClassifier(**params_cat)\nmodel_cat.fit(train_df, y_train)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:09.704183Z","iopub.status.busy":"2020-12-22T06:21:09.697058Z","iopub.status.idle":"2020-12-22T06:21:09.709784Z","shell.execute_reply":"2020-12-22T06:21:09.709157Z"},"papermill":{"duration":0.151484,"end_time":"2020-12-22T06:21:09.709921","exception":false,"start_time":"2020-12-22T06:21:09.558437","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"print('CatBoost ROC-AUC score: ', roc_auc_score(y_test.values, model_cat.predict_proba(test_df)[:, 1]))","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.105401,"end_time":"2020-12-22T06:21:09.920755","exception":false,"start_time":"2020-12-22T06:21:09.815354","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Random forest classifier"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:10.140534Z","iopub.status.busy":"2020-12-22T06:21:10.139734Z","iopub.status.idle":"2020-12-22T06:21:27.65718Z","shell.execute_reply":"2020-12-22T06:21:27.656375Z"},"papermill":{"duration":17.631367,"end_time":"2020-12-22T06:21:27.657303","exception":false,"start_time":"2020-12-22T06:21:10.025936","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nrfclf = RandomForestClassifier()\nrfclf.fit(train_df,y_train)\npred = rfclf.predict(test_df)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:27.874655Z","iopub.status.busy":"2020-12-22T06:21:27.873898Z","iopub.status.idle":"2020-12-22T06:21:27.883785Z","shell.execute_reply":"2020-12-22T06:21:27.883119Z"},"papermill":{"duration":0.120312,"end_time":"2020-12-22T06:21:27.883917","exception":false,"start_time":"2020-12-22T06:21:27.763605","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\nprint(roc_auc_score(y_test,pred))","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.10712,"end_time":"2020-12-22T06:21:28.108765","exception":false,"start_time":"2020-12-22T06:21:28.001645","status":"completed"},"tags":[]},"cell_type":"markdown","source":"#Xgboost classifier"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:28.332216Z","iopub.status.busy":"2020-12-22T06:21:28.331181Z","iopub.status.idle":"2020-12-22T06:21:31.806217Z","shell.execute_reply":"2020-12-22T06:21:31.807077Z"},"papermill":{"duration":3.590184,"end_time":"2020-12-22T06:21:31.807274","exception":false,"start_time":"2020-12-22T06:21:28.21709","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from xgboost import XGBClassifier\nxgbclf = XGBClassifier()\nxgbclf.fit(train_df,y_train)\nxgb_pred = xgbclf.predict(test_df)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:32.061017Z","iopub.status.busy":"2020-12-22T06:21:32.060191Z","iopub.status.idle":"2020-12-22T06:21:32.07022Z","shell.execute_reply":"2020-12-22T06:21:32.069309Z"},"papermill":{"duration":0.132632,"end_time":"2020-12-22T06:21:32.070392","exception":false,"start_time":"2020-12-22T06:21:31.93776","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"print(roc_auc_score(y_test,xgb_pred))","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:32.292587Z","iopub.status.busy":"2020-12-22T06:21:32.291796Z","iopub.status.idle":"2020-12-22T06:21:46.6918Z","shell.execute_reply":"2020-12-22T06:21:46.690811Z"},"papermill":{"duration":14.513329,"end_time":"2020-12-22T06:21:46.69196","exception":false,"start_time":"2020-12-22T06:21:32.178631","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from sklearn.neural_network import MLPClassifier\nmlp = MLPClassifier()\nmlp.fit(train_df,y_train)\nmlp_pred = mlp.predict(test_df)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:46.918949Z","iopub.status.busy":"2020-12-22T06:21:46.917891Z","iopub.status.idle":"2020-12-22T06:21:46.928237Z","shell.execute_reply":"2020-12-22T06:21:46.927398Z"},"papermill":{"duration":0.124874,"end_time":"2020-12-22T06:21:46.928396","exception":false,"start_time":"2020-12-22T06:21:46.803522","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"print(roc_auc_score(y_test,mlp_pred))","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:47.152509Z","iopub.status.busy":"2020-12-22T06:21:47.150982Z","iopub.status.idle":"2020-12-22T06:21:47.18816Z","shell.execute_reply":"2020-12-22T06:21:47.18739Z"},"papermill":{"duration":0.151426,"end_time":"2020-12-22T06:21:47.18828","exception":false,"start_time":"2020-12-22T06:21:47.036854","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"eli5.show_weights(model, top=20)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:47.411734Z","iopub.status.busy":"2020-12-22T06:21:47.410687Z","iopub.status.idle":"2020-12-22T06:21:47.597354Z","shell.execute_reply":"2020-12-22T06:21:47.597951Z"},"papermill":{"duration":0.30074,"end_time":"2020-12-22T06:21:47.598134","exception":false,"start_time":"2020-12-22T06:21:47.297394","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"lgb.plot_importance(model)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.109834,"end_time":"2020-12-22T06:21:47.819334","exception":false,"start_time":"2020-12-22T06:21:47.7095","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Submission preparation"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:48.046803Z","iopub.status.busy":"2020-12-22T06:21:48.045895Z","iopub.status.idle":"2020-12-22T06:21:48.048912Z","shell.execute_reply":"2020-12-22T06:21:48.048309Z"},"papermill":{"duration":0.11861,"end_time":"2020-12-22T06:21:48.049067","exception":false,"start_time":"2020-12-22T06:21:47.930457","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:48.287526Z","iopub.status.busy":"2020-12-22T06:21:48.286454Z","iopub.status.idle":"2020-12-22T06:21:48.290323Z","shell.execute_reply":"2020-12-22T06:21:48.289528Z"},"papermill":{"duration":0.130163,"end_time":"2020-12-22T06:21:48.290457","exception":false,"start_time":"2020-12-22T06:21:48.160294","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"iter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-22T06:21:48.556203Z","iopub.status.busy":"2020-12-22T06:21:48.55513Z","iopub.status.idle":"2020-12-22T06:21:49.007808Z","shell.execute_reply":"2020-12-22T06:21:49.006686Z"},"papermill":{"duration":0.603026,"end_time":"2020-12-22T06:21:49.007986","exception":false,"start_time":"2020-12-22T06:21:48.40496","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    # merge\n    test_df = test_df.merge(user_answers_df, on = \"user_id\", how = \"left\")\n    #test_df = test_df.merge(task_container_characteristics, on = \"task_container_id\", how = \"left\")\n    test_df = test_df.merge(content_answers_df, on = \"content_id\", how = \"left\")\n    \n    # type transformation\n    test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(value=False).astype(bool)\n    test_df.fillna(value = 0.5, inplace = True)\n    test_df = test_df.replace([np.inf, -np.inf], np.nan)\n    test_df = test_df.fillna(0.5)\n    \n    # preds\n    test_df['answered_correctly'] = model.predict_proba(test_df[features])[:, 1]\n    cols_to_submission = ['row_id', 'answered_correctly', 'group_num']\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.115633,"end_time":"2020-12-22T06:21:49.2514","exception":false,"start_time":"2020-12-22T06:21:49.135767","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}