{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# libraries\nimport riiideducation\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nimport os\nfrom typing import List, Dict, Optional\nimport numpy as np\nfrom sklearn.model_selection import RepeatedKFold\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nimport math\nimport time\nimport random\nimport lightgbm as lgb\nimport gc\nimport os\nfrom sklearn.preprocessing import LabelEncoder\nfrom numba import jit\nfrom sklearn.model_selection import StratifiedKFold, KFold, RepeatedKFold, GroupKFold, GridSearchCV, train_test_split, TimeSeriesSplit\nfrom sklearn import metrics","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt \nimport matplotlib.colors as mcolors\nimport seaborn as sns\nplt.style.use('fivethirtyeight')\n%matplotlib inline\nplt.rcParams['figure.figsize']=10,6\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage(deep=True).sum() / 1024**2\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                c_prec = df[col].apply(lambda x: np.finfo(x).precision).max()\n                if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max and c_prec == np.finfo(np.float32).precision:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    end_mem = df.memory_usage(deep=True).sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n    \n\n@jit\ndef fast_auc(y_true, y_prob):\n    \"\"\"\n    fast roc_auc computation: https://www.kaggle.com/c/microsoft-malware-prediction/discussion/76013\n    \"\"\"\n    y_true = np.asarray(y_true)\n    y_true = y_true[np.argsort(y_prob)]\n    nfalse = 0\n    auc = 0\n    n = len(y_true)\n    for i in range(n):\n        y_i = y_true[i]\n        nfalse += (1 - y_i)\n        auc += y_i * nfalse\n    auc /= (nfalse * (n - nfalse))\n    return auc\n\n\ndef eval_auc(y_true, y_pred):\n    \"\"\"\n    Fast auc eval function for lgb.\n    \"\"\"\n    return 'auc', fast_auc(y_true, y_pred), True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Genersting the pie chart\ndef pie_chart(df,col,path):\n  label = df[col].value_counts().index.tolist()\n  fig = plt.figure(figsize=(10,6))\n  ax = (df[col].value_counts()*100.0 /len(df))\\\n  .plot.pie(startangle=90,autopct='%.1f%%', labels =label, fontsize=12)                                                                           \n  ax.set_title('% '+str(col))\n  # plt.savefig(path+str(col1)+'.png')\n  plt.show()\n    \ndef bar_chart(df,parameter, figsize=(10,6)):\n    target_counts = df[parameter].value_counts()\n    target_perc = target_counts.div(target_counts.sum(), axis=0)\n    plt.figure(figsize=figsize)\n    ax = sns.barplot(x=target_counts.index.values, y=target_counts.values, order=target_counts.index,palette='winter')\n#     plt.xticks(rotation=90)\n    plt.xlabel(f'{parameter}', weight ='bold',fontsize=16)\n    plt.ylabel('# of occurances', weight ='bold',fontsize=16)\n    plt.title(\"Count of \"+f'{parameter}', weight ='bold',fontsize=20)\n\n    rects = ax.patches\n    labels = np.round(target_perc.values*100, 2)\n    for rect, label in zip(rects, labels):\n        height = rect.get_height()\n        ax.text(rect.get_x() + rect.get_width()/2, height + 5, f'{label}%', ha='center', va='bottom')\n    \n    try:\n        labels =target_counts.index.tolist()\n\n    #     labels.sort()\n        labels=[textwrap.fill(text,12) for text in labels]\n        pos = np.arange(len(labels)) \n        plt.xticks(pos, labels,fontsize=12)\n    except:\n        pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '/kaggle/input'\n\ntrain = pd.read_csv(f'{path}/riiid-test-answer-prediction/train.csv', low_memory=False, nrows=5 * (10**5),\n                    usecols=['timestamp', 'user_id', 'content_id', 'content_type_id', 'user_answer', 'answered_correctly',\n                             'prior_question_elapsed_time', 'prior_question_had_explanation'],\n                       dtype={'timestamp': 'int64',\n                              'user_id': 'int32',\n                              'content_id': 'int16',\n                              'content_type_id': 'int8',\n                              'user_answer': 'int8',\n                              'answered_correctly': 'int8',\n                              'prior_question_elapsed_time': 'float32', \n                              'prior_question_had_explanation': 'boolean',\n                             }\n                      )\ntrain = train.sort_values(['timestamp'], ascending=True)\nquestions = pd.read_csv(f'{path}/riiid-test-answer-prediction/questions.csv')\nlectures = pd.read_csv(f'{path}/riiid-test-answer-prediction/lectures.csv')\nprint('Train shapes: ', train.shape)     ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lectures.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check the cardinality of each columns\ncardinality_df = pd.DataFrame()\nl = []\nfor i in range(len(train.columns)):\n#     print(i)\n    distinct_count = train[train.columns[i]].value_counts().count()\n    l.append(distinct_count)\ncol_name = train.columns.tolist()\ncardinality_df[\"Columns_Name\"]= col_name\ncardinality_df['Cardinality_Count'] = l\ncardinality_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(train.prior_question_elapsed_time)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bar_chart(train,'content_type_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bar_chart(train,'user_answer')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bar_chart(train,'answered_correctly')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bar_chart(train,'prior_question_had_explanation')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gr_df=train.groupby(['content_type_id','answered_correctly'])['user_id'].count().to_frame().rename(columns={'user_id':'count'}).reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"g = sns.catplot(x=\"content_type_id\", y=\"count\",col=\"answered_correctly\",\n                data=gr_df, kind=\"bar\",\n                height=6, aspect=.7);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dgen_df = train.groupby(['user_answer'])['answered_correctly'].count().to_frame().reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.catplot(x=\"user_answer\", y=\"answered_correctly\",\n                data=dgen_df, kind=\"bar\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rv_df = train.groupby(['user_answer','answered_correctly'])['user_id'].count().to_frame().rename(columns={'user_id':'count'}).reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"g = sns.catplot(x=\"user_answer\", y=\"count\",col=\"answered_correctly\",\n                data=rv_df, kind=\"bar\",\n                height=6, aspect=.9);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def description(df):\n    summary = pd.DataFrame(df.dtypes,columns=['dtypes'])\n    summary = summary.reset_index()\n    summary['Name'] = summary['index']\n    summary = summary[['Name','dtypes']]\n    summary['Missing'] = df.isnull().sum().values    \n    summary['Uniques'] = df.nunique().values\n    summary['First Value'] = df.iloc[0].values\n    summary['Second Value'] = df.iloc[1].values\n    summary['Third Value'] = df.iloc[2].values\n    return summary","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"description(train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Distribution of Target Variable"},{"metadata":{"trusted":true},"cell_type":"code","source":"pie_chart(train,'answered_correctly',8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"id_col = ['user_id', 'content_id', 'content_type_id', 'prior_question_elapsed_time']\nplt.figure(figsize=(10,6))\nfor i, col in enumerate(id_col):\n    plt.subplot(2, 2, i + 1)\n    sns.distplot(train[col], color='green', \n                 hist_kws={'alpha':1,\"linewidth\": 2},\n                 kde_kws={\"color\": \"red\", \"lw\": 2, 'bw':0.01})\n    plt.title(col)\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"time_col = ['timestamp', 'prior_question_elapsed_time',]\nplt.figure(figsize=(10,6))\nfor i, col in enumerate(time_col):\n    plt.subplot(1, 2, i + 1)\n    train[col].hist(bins = 50,color='red')\n    plt.title(col)\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Timestamp represents the time from the first user interaction to the current one and Prior question elapsed time represents how long it took a user to answer their previous question bundle."},{"metadata":{"trusted":true},"cell_type":"code","source":"col = ['prior_question_had_explanation', 'user_answer',]\n\ntotal = len(train)\nplt.figure(figsize=(12,5), dpi=60)\n\nfor i, col in enumerate(col):\n    plt.subplot(1, 2, i + 1)\n    g=sns.countplot(train[col], palette='winter')\n    sizes=[] # Get highest values in y\n    for p in g.patches:\n        height = p.get_height()\n        sizes.append(height)\n        g.text(p.get_x()+p.get_width()/2.,\n                height + 3,\n                '{:1.2f}%'.format(height/total*100),\n                ha=\"center\", fontsize=14) \n    g.set_ylim(0, max(sizes) * 1.15) # set y limit based on highest heights\n    plt.title(col)\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# thanks to @ilialar \ntrain_only_df = train[train['answered_correctly']!=-1]\ngrouped_by_user_df = train_only_df.groupby('user_id')\nuser_answers_df = grouped_by_user_df.agg({'answered_correctly': ['mean', 'count'] })\n\nfig,ax=plt.subplots(figsize=(15,8), dpi=100)\n\nplt.subplot(2, 2, 1)\ng1=user_answers_df[('answered_correctly','mean')].hist(bins=100, color='blue')\ng1.set_title(\"users correct answer mean dist.\",)\n\nplt.subplot(2, 2, 2)\ng2=user_answers_df[('answered_correctly','count')].hist(bins=100, color='blue')\ng2.set_title('users correct answer count dist.')\n\nplt.subplot(2, 2, 3)\ng3=user_answers_df[user_answers_df[('answered_correctly','count')]<= 100][('answered_correctly','mean')].hist(bins=100, color='blue')\ng3.set_title('users correct answer mean dist. less than 100 question')\n\nplt.subplot(2, 2, 4)\ng4=user_answers_df[user_answers_df[('answered_correctly','count')]>=100][('answered_correctly','mean')].hist(bins=100, color='blue')\ng4.set_title('users correct answer count dist. more than 100 question')\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Average user score is lower than the overall % of correct answers(bottom left graph). It means heavy users have even better scores(bottom right graph)."},{"metadata":{"trusted":true},"cell_type":"code","source":"user_time_answers_df = grouped_by_user_df.agg({'answered_correctly': ['mean', 'count'],\n                                        'timestamp': ['mean', 'count']})\n\nfig,ax=plt.subplots(figsize=(15,6), dpi=50)\n\nplt.subplot(1, 2, 1)\nplt.scatter(x = user_answers_df[('answered_correctly','count')], \n            y = user_answers_df[ ('answered_correctly','mean')], color='green')\nplt.title('relation b/w correct answer mean and count')\n\nplt.subplot(1, 2, 2)\nplt.scatter(x = user_time_answers_df[ ('timestamp','mean')], \n            y = user_time_answers_df[ ('answered_correctly','mean')], color='green')\nplt.title('relation b/w  timestamp mean and correct answer mean')\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> There is relationship between the average score for the active user, and the number of questions answered; there is relation average timestamp and average correct answer can be useful for baseline."},{"metadata":{},"cell_type":"markdown","source":"### Correct Answers by Content"},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_by_content_df = train_only_df.groupby('content_id')\ncontent_answers_df = grouped_by_content_df.agg({'answered_correctly': ['mean', 'count'] })\n\nfig,ax=plt.subplots(figsize=(15,8), dpi=100)\n\nplt.subplot(2, 2, 1)\ng1=content_answers_df[('answered_correctly','mean')].hist(bins=100, color='red')\ng1.set_title(\"content correct answer mean dist.\")\n\nplt.subplot(2, 2, 2)\ng2=content_answers_df[('answered_correctly','count')].hist(bins=100, color='red')\ng2.set_title('content answer count dist.')\n\nplt.subplot(2, 2, 3)\ng3=content_answers_df[content_answers_df[('answered_correctly','count')]<= 100][('answered_correctly','mean')].hist(bins=100, color='red')\ng3.set_title('content correct answer mean dist. less than 100 question')\n\nplt.subplot(2, 2, 4)\ng4=content_answers_df[content_answers_df[('answered_correctly','count')]>=100][('answered_correctly','mean')].hist(bins=100, color='red')\ng4.set_title('content correct answer count dist. more than 100 question')\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_time_content_df = grouped_by_content_df.agg({'answered_correctly': ['mean', 'count'],\n                                        'timestamp': ['mean', 'count']})\n\nfig,ax=plt.subplots(figsize=(15,6), dpi=70)\n\nplt.subplot(1, 2, 1)\nplt.scatter(x = content_answers_df[('answered_correctly','count')], \n            y=content_answers_df[ ('answered_correctly','mean')], color='cyan')\nplt.title('relation b/w correct answer mean and count')\n\nplt.subplot(1, 2, 2)\nplt.scatter(x = user_time_content_df[ ('timestamp','mean')], \n            y = user_time_content_df[ ('answered_correctly','mean')], color='cyan')\nplt.title('relation b/w  timestamp mean and correct answer count')\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Question.csv"},{"metadata":{"trusted":true},"cell_type":"code","source":"description(questions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"id_col = ['question_id', 'bundle_id']\nplt.figure(figsize=(10,6))\nfor i, col in enumerate(id_col):\n    plt.subplot(1, 2, i + 1)\n    sns.distplot(questions[col], color='green',bins=100, \n                 hist_kws={'alpha':1,\"linewidth\": 1},\n                 kde_kws={\"color\": \"red\", \"lw\": 2, 'bw':0.01})\n    plt.title(col)\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df = questions.merge(content_answers_df, left_on = 'question_id', right_on = 'content_id', how = 'left')\nbundle_dict = questions_df['bundle_id'].value_counts().to_dict()\n\nquestions_df['right_answers'] = questions_df[('answered_correctly', 'mean')] * questions_df[('answered_correctly', 'count')]\nquestions_df['bundle_size'] = questions_df['bundle_id'].apply(lambda x: bundle_dict[x])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"col = ['correct_answer', 'part', 'bundle_size']\n\ntotal = len(questions_df)\nplt.figure(figsize=(15,8), dpi=100)\n\nfor i, col in enumerate(col):\n    plt.subplot(2, 2, i + 1)\n    g=sns.countplot(questions_df[col], palette='winter')\n    sizes=[] # Get highest values in y\n    for p in g.patches:\n        height = p.get_height()\n        sizes.append(height)\n        g.text(p.get_x()+p.get_width()/2.,\n                height + 3,\n                '{:1.2f}%'.format(height/total*100),\n                ha=\"center\", fontsize=14) \n    g.set_ylim(0, max(sizes) * 1.15) # set y limit based on highest heights\n    plt.title(col)\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig,ax =plt.subplots(figsize=(15,6),dpi=70)\nplt.subplot(1,2,1)\nplt.scatter(x=questions_df['answered_correctly','count'],\n           y=questions_df['right_answers'],color ='blue')\nplt.title(\"relation b/w right answer and question asked (count)\")\n\nplt.subplot(1,2,2)\nplt.scatter(x=questions_df['right_answers'],\n           y=questions_df[('answered_correctly','mean')],color='blue')\nplt.title(\"relation b/w right_answer and correct answer mean\")\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_by_bundle_df = questions_df.groupby('bundle_id')\nbundle_answers_df = grouped_by_bundle_df.agg({'right_answers': 'sum', ('answered_correctly', 'count'): 'sum'}).copy()\nbundle_answers_df.columns = ['bundle_rignt_answers', 'bundle_questions_asked']\nbundle_answers_df['bundle_accuracy'] = bundle_answers_df['bundle_rignt_answers'] / bundle_answers_df['bundle_questions_asked']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bundle_answers_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig,ax=plt.subplots(figsize=(15,6), dpi=100)\nplt.subplot(1, 2, 1)\nplt.scatter(x = bundle_answers_df['bundle_questions_asked'], \n            y=bundle_answers_df['bundle_accuracy'], color='dodgerblue')\nplt.title('relation b/w bundle_questions_asked and bundle_accuracy')\n\nplt.subplot(1, 2, 2)\nplt.scatter(x = bundle_answers_df['bundle_rignt_answers'], \n            y = bundle_answers_df['bundle_accuracy'], color='dodgerblue')\nplt.title('relation b/w  bundle_rignt_answers and bundle_accuracy')\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_by_part_df = questions_df.groupby('part')\npart_answers_df = grouped_by_part_df.agg({'right_answers': 'sum', ('answered_correctly', 'count'): 'sum'}).copy()\npart_answers_df.columns = ['part_rignt_answers', 'part_questions_asked']\npart_answers_df['part_accuracy'] = part_answers_df['part_rignt_answers'] / part_answers_df['part_questions_asked']\npart_answers_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# del train_df\n# del features_part_df\ndel questions_df\ndel bundle_answers_df\ndel grouped_by_part_df\ndel part_answers_df\ndel grouped_by_user_df\ndel grouped_by_content_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '/kaggle/input'\n\ntrain = pd.read_csv(f'{path}/riiid-test-answer-prediction/train.csv',\n                    usecols=['timestamp', 'user_id', 'content_id', 'content_type_id', 'user_answer', 'answered_correctly',\n                             'prior_question_elapsed_time', 'prior_question_had_explanation'],\n                       dtype={'timestamp': 'int64',\n                              'user_id': 'int32',\n                              'content_id': 'int16',\n                              'content_type_id': 'int8',\n                              'user_answer': 'int8',\n                              'answered_correctly': 'int8',\n                              'prior_question_elapsed_time': 'float32', \n                              'prior_question_had_explanation': 'boolean',\n                             }\n                      )\ntrain = train.sort_values(['timestamp'], ascending=True)\nquestions = pd.read_csv(f'{path}/riiid-test-answer-prediction/questions.csv')\nlectures = pd.read_csv(f'{path}/riiid-test-answer-prediction/lectures.csv')\nprint('Train shapes: ', train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# filter out lectures\ntrain = train.loc[train['answered_correctly'] != -1].reset_index(drop=True)\ntrain = train.drop(['timestamp','content_type_id'], axis=1)\ntrain['prior_question_had_explanation'] = train['prior_question_had_explanation'].fillna(value = False).astype(bool)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_answers_df = train.groupby('user_id').agg({'answered_correctly': ['mean', 'count']}).copy()\nuser_answers_df.columns = ['mean_user_accuracy', 'questions_answered']\n\ncontent_answers_df = train.groupby('content_id').agg({'answered_correctly': ['mean', 'count']}).copy()\ncontent_answers_df.columns = ['mean_accuracy', 'question_asked']\n\nuser_content_answers_df = train.groupby(['user_id', 'content_id']).agg({'answered_correctly': ['mean', 'count']}).copy()\nuser_content_answers_df.columns = ['mean_user_content_accuracy', 'content_questions_answered']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Now we will use only a part of data for training, to avoid leaks and memory error\n\ntrain = train.iloc[90000000:,:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.merge(user_answers_df, how = 'left', on = 'user_id')\ntrain = train.merge(content_answers_df, how = 'left', on = 'content_id')\ntrain = train.merge(user_content_answers_df, how = 'left', on = ['user_id', 'content_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.fillna(value = 0.5, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['mean_diff1'] = train['mean_user_accuracy'] - train['mean_user_content_accuracy']\ntrain['mean_diff2'] = train['mean_accuracy'] - train['mean_user_content_accuracy']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"le = LabelEncoder()\ntrain[\"prior_question_had_explanation\"] = le.fit_transform(train[\"prior_question_had_explanation\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.sort_values(['user_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = train['answered_correctly']\n\ncolumns = ['mean_user_accuracy', 'questions_answered', 'mean_accuracy', 'question_asked',\n           'prior_question_had_explanation', 'mean_diff1', 'mean_diff2', 'mean_user_content_accuracy']\nX = train[columns]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scores = []\nfeature_importance = pd.DataFrame()\nmodels = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params = {'num_leaves': 32,\n          'max_bin': 300,\n          'objective': 'binary',\n          'max_depth': 13,\n          'learning_rate': 0.03,\n          \"boosting_type\": \"gbdt\",\n          \"metric\": 'auc',\n         }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"columns = ['mean_user_accuracy', 'questions_answered', 'mean_accuracy', 'question_asked',\n#            'prior_question_had_explanation', 'mean_diff1', 'mean_diff2'\n          ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"folds = StratifiedKFold(n_splits=5, shuffle=False)\nfor fold_n, (train_index, valid_index) in enumerate(folds.split(X, y)):\n    print(f'Fold {fold_n} started at {time.ctime()}')\n    X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]\n    y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]\n    model = lgb.LGBMClassifier(**params, n_estimators=700, n_jobs = 1)\n    model.fit(X_train, y_train, \n            eval_set=[(X_train, y_train), (X_valid, y_valid)], eval_metric=eval_auc,\n            verbose=1000, early_stopping_rounds=10)\n    score = max(model.evals_result_['valid_1']['auc'])\n    \n    models.append(model)\n    scores.append(score)\n\n    fold_importance = pd.DataFrame()\n    fold_importance[\"feature\"] = columns\n    fold_importance[\"importance\"] = model.feature_importances_\n    fold_importance[\"fold\"] = fold_n + 1\n    feature_importance = pd.concat([feature_importance, fold_importance], axis=0)\n    break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_importance[\"importance\"] /= 1\ncols = feature_importance[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(\n    by=\"importance\", ascending=False)[:50].index\n\nbest_features = feature_importance.loc[feature_importance.feature.isin(cols)]\n\nplt.figure(figsize=(16, 12));\nsns.barplot(x=\"importance\", y=\"feature\", data=best_features.sort_values(by=\"importance\", ascending=False));\nplt.title('LGB Features (avg over folds)');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del X, y","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Making predictions.\nCode is taken from https://www.kaggle.com/sishihara/riiid-lgbm-5cv-benchmark"},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    y_preds = []\n    test_df = test_df.merge(user_answers_df, how = 'left', on = 'user_id')\n    test_df = test_df.merge(content_answers_df, how = 'left', on = 'content_id')\n    test_df = test_df.merge(user_content_answers_df, how = 'left', on = ['user_id', 'content_id'])\n    test_df['mean_diff1'] = test_df['mean_user_accuracy'] - test_df['mean_user_content_accuracy']\n    test_df['mean_diff2'] = test_df['mean_accuracy'] - test_df['mean_user_content_accuracy']\n    test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(value = False).astype(bool)\n    test_df = test_df.loc[test_df['content_type_id'] == 0].reset_index(drop=True)\n    test_df.fillna(value = 0.5, inplace = True)\n    test_df[\"prior_question_had_explanation_enc\"] = le.fit_transform(test_df[\"prior_question_had_explanation\"])\n\n    for model in models:\n        y_pred = model.predict_proba(test_df[columns], num_iteration=model.best_iteration_)[:, 1]\n        y_preds.append(y_pred)\n\n    y_preds = sum(y_preds) / len(y_preds)\n    test_df['answered_correctly'] = y_preds\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Reference Link\n[https://www.kaggle.com/vikassingh1996/riiid-eda-xgb-and-feature-importance](http://)"},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{},"cell_type":"markdown","source":""}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}