{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook is an ensemble of three simple models.LightGBM/XGboost/Catboost.\nAlmost all parameters in these models are default values.\nTo save memory, train.csv is read to only 30M lines. !Update! 30M → 80M\n\nExplanation of unique features\n*     'user_lot' : Length Of Time per users. In fact, this feature may not be important.\n*     'incorrect_answer_var' : Represents the variance of incorrect choices.Indicates if the question is prone to misleading\n*     'user_part_lec' : The number of times the user has been lectured on that part.\n*     'all_tag_mean' : Correct answer rate for each tag included in the question\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.metrics import roc_auc_score\nimport datatable as dt\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom catboost import CatBoostClassifier, Pool\nimport riiideducation\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom pandas.api.types import is_categorical_dtype","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df, use_float16=False):\n    \"\"\"\n    Iterate through all the columns of a dataframe and modify the data type to reduce memory usage.        \n    \"\"\"\n    \n    start_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n    \n    for col in df.columns:\n        if is_datetime(df[col]) or is_categorical_dtype(df[col]):\n            continue\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == \"int\":\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype(\"category\")\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def qtag_mean(df,qtag):\n    qtag_mean = df[[f'qtag_{qtag}','answered_correctly']].groupby([f'qtag_{qtag}']).agg(['mean']).reset_index()\n    qtag_mean.columns = ['qtag',f'qtag_{qtag}_mean']\n    qtag_mean = qtag_mean.astype({'qtag': 'int16'})\n    return qtag_mean","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess(df,qdf,ldf):\n\n    lec = pd.merge(df[df[\"answered_correctly\"]==-1], ldf, on=['content_id'], how=\"left\")    \n    \n    user_part_lec = lec[['part','user_id','content_id']].groupby(['part','user_id']).agg(['count']).reset_index()\n    user_part_lec.columns = ['part','user_id','user_part_lec']\n    \n    del lec\n    \n    df = df[df[\"answered_correctly\"]!=-1]\n    \n    #incorrect_var process...\n    incorrect_answer_count = df[df[\"answered_correctly\"]==0][['content_id','user_answer','answered_correctly']].groupby(['content_id','user_answer']).agg(['count']).reset_index()\n    incorrect_answer_count.columns = ['content_id','user_answer','incorrect_answer_count']\n    \n    incorrect_answer_cmax = incorrect_answer_count[['content_id','incorrect_answer_count']].groupby(['content_id']).agg(['max']).reset_index()\n    incorrect_answer_cmax.columns = ['content_id','incorrect_answer_cmax']\n    incorrect_answer_count = pd.merge(incorrect_answer_count,incorrect_answer_cmax, on=['content_id'], how=\"left\")\n    incorrect_answer_count['scaled_IAC'] = incorrect_answer_count['incorrect_answer_count']/incorrect_answer_count['incorrect_answer_cmax']\n    \n    incorrect_answer_var = incorrect_answer_count[['content_id','scaled_IAC']].groupby(['content_id']).agg(['var']).reset_index()\n    incorrect_answer_var.columns = ['content_id','incorrect_answer_var']\n    del incorrect_answer_count,incorrect_answer_cmax\n    df = df.drop(['user_answer'], axis=1)\n\n    df['prior_question_had_explanation'].fillna(0, inplace=True)\n    df['prior_question_had_explanation'] = df['prior_question_had_explanation'].astype(np.int8)\n    explanation_count = df[['prior_question_had_explanation','user_id']].groupby(['user_id']).agg(['sum']).reset_index()\n    explanation_count.columns = ['user_id', 'explanation_count']\n\n    user_time = df[['timestamp','user_id']].groupby(['user_id']).agg(['min','max']).reset_index()\n    user_time.columns = ['user_id', 'min' ,'max']\n    user_time['user_lot'] = user_time['max'] - user_time['min']\n    user_time = user_time.drop(['max', 'min'], axis=1)\n\n    df = df.drop(['timestamp','prior_question_had_explanation'], axis=1)\n\n    user_cal = df[['user_id','answered_correctly']].groupby(['user_id']).agg(['mean', 'count', 'std', 'sem']).reset_index()\n    user_cal.columns = ['user_id','user_mean', 'user_count', 'user_std', 'user_sem']\n    user_cal = reduce_mem_usage(user_cal, use_float16=True)\n\n    content_cal = df[['content_id','answered_correctly']].groupby(['content_id']).agg(['mean', 'count']).reset_index()\n    content_cal.columns = ['content_id','content_mean', 'content_count']\n    content_cal = reduce_mem_usage(content_cal, use_float16=True)\n    \n    task_cal = df[['task_container_id','answered_correctly']].groupby(['task_container_id']).agg(['count', 'std']).reset_index()\n    task_cal.columns = ['task_container_id', 'task_count', 'task_std']\n    task_cal = reduce_mem_usage(task_cal, use_float16=True)\n\n    df = df.drop(['task_container_id'], axis=1)\n\n    df = pd.merge(df,qdf,on=\"content_id\",how=\"left\")\n\n    part_cal = df[['part','answered_correctly']].groupby(['part']).agg(['mean']).reset_index()\n    part_cal.columns = ['part','part_mean']\n    part_cal = reduce_mem_usage(part_cal, use_float16=True)\n    \n    qtag1_mean = qtag_mean(df,1)\n    qtag2_mean = qtag_mean(df,2)\n    qtag3_mean = qtag_mean(df,3)\n    qtag4_mean = qtag_mean(df,4)\n    qtag5_mean = qtag_mean(df,5)\n    qtag6_mean = qtag_mean(df,6)\n\n    qtag_merge = pd.DataFrame(range(188),columns={'qtag'})\n    qtag_merge = pd.merge(qtag_merge,qtag1_mean,on=\"qtag\",how=\"left\")\n    qtag_merge = pd.merge(qtag_merge,qtag2_mean,on=\"qtag\",how=\"left\")\n    qtag_merge = pd.merge(qtag_merge,qtag3_mean,on=\"qtag\",how=\"left\")\n    qtag_merge = pd.merge(qtag_merge,qtag4_mean,on=\"qtag\",how=\"left\")\n    qtag_merge = pd.merge(qtag_merge,qtag5_mean,on=\"qtag\",how=\"left\")\n    qtag_merge = pd.merge(qtag_merge,qtag6_mean,on=\"qtag\",how=\"left\")\n    qtag_merge['tag_mean'] = qtag_merge.iloc[:,1:6].mean(axis=1)\n    qtag_merge = qtag_merge.drop(['qtag_1_mean', 'qtag_2_mean', 'qtag_3_mean', 'qtag_4_mean', 'qtag_5_mean', 'qtag_6_mean'], axis=1)\n    \n    user_cal = pd.merge(user_cal, explanation_count, on=['user_id'], how=\"left\")\n    user_cal['explanation_rate'] = user_cal['explanation_count']/user_cal['user_count']\n    user_cal = user_cal.drop(['explanation_count'], axis=1)\n    user_cal = pd.merge(user_cal, user_time, on=['user_id'], how=\"left\")\n    content_cal = pd.merge(content_cal, incorrect_answer_var, on=['content_id'], how=\"left\")\n    \n    del explanation_count,user_time, incorrect_answer_var,qtag1_mean,qtag2_mean,qtag3_mean,qtag4_mean,qtag5_mean,qtag6_mean\n    \n    return user_cal, content_cal, part_cal, task_cal, user_part_lec, qtag_merge","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_types_dict = {\n    #'row_id': 'int64',\n    'timestamp': 'int64',\n    'user_id': 'int32',\n    'content_id': 'int16',\n    #'content_type_id': 'int8',\n    'task_container_id': 'int16',\n    'user_answer': 'int8',\n    'answered_correctly': 'int8',\n    'prior_question_elapsed_time': 'float16',\n    'prior_question_had_explanation': 'boolean'\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',\n                   usecols=[1, 2, 3, 5, 6, 7, 8, 9],\n                   nrows=80_000_000, \n                   dtype={'timestamp': 'int64',\n                          'user_id': 'int32',\n                          'content_id': 'int16',\n                          'task_container_id': 'int16',\n                          'user_answer': 'int8',\n                          'answered_correctly':'int8',\n                          'prior_question_elapsed_time': 'float32',\n                          'prior_question_had_explanation': 'boolean'}\n                   )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',usecols=[0,3,4])\nquestions = questions.rename(columns={'question_id': 'content_id'})\ntag = questions[\"tags\"].str.split(\" \", n = 10, expand = True)\ntag = tag.rename(columns=lambda s : 'qtag_' + f'{s+1}')\nquestions = pd.merge(questions, tag, left_index=True,right_index=True)\ndel tag\nquestions = questions.drop(['tags'], axis=1)\n\nquestions = reduce_mem_usage(questions, use_float16=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lectures = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv')\nlectures = lectures.rename(columns={'lecture_id': 'content_id'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.sort_values(['timestamp'], ascending=True).reset_index(drop = True)\ntrain = train_df.groupby('user_id').tail(25)\nfeatures = train_df[~train_df.index.isin(train.index)]\ndel train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = features.drop(['prior_question_elapsed_time'], axis=1)\nuser_cal, content_cal, part_cal, task_cal, user_part_lec, qtag_merge = preprocess(features,questions,lectures)\ndel features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train_df.groupby('user_id').tail(25)\ntrain = train[train[\"answered_correctly\"]!=-1]\ntrain = pd.merge(train,questions,on=\"content_id\",how=\"left\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#'all_tag_mean' process... tags1~6 average\nfor tag in range(1,7):\n    train[f'qtag_{tag}'] = pd.to_numeric(train[f'qtag_{tag}'], errors='coerce')\n    train = pd.merge(train,qtag_merge,left_on=f'qtag_{tag}',right_on='qtag',how=\"left\").drop(columns={'qtag',f'qtag_{tag}'})\n    train = train.rename(columns={'tag_mean': f'tag_mean{tag}'})\n    \ntrain['all_tag_mean'] = train.loc[:,'tag_mean1':'tag_mean6'].mean(axis=1)\ntrain = train.drop(['tag_mean1','tag_mean2','tag_mean3','tag_mean4','tag_mean5','tag_mean6'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.merge(train, user_cal, on=['user_id'], how=\"left\")\ntrain = pd.merge(train, content_cal, on=['content_id'], how=\"left\")\ntrain = pd.merge(train, part_cal, on=['part'], how=\"left\")\ntrain = pd.merge(train, task_cal, on=['task_container_id'], how=\"left\")\ntrain = pd.merge(train, user_part_lec, on=['user_id','part'], how=\"left\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isnull().any()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False).astype('int8')\ntrain['prior_question_elapsed_time'].fillna(train['prior_question_elapsed_time'].median(), inplace=True)\ntrain['user_mean'].fillna(train['user_mean'].median(),  inplace=True)\ntrain['user_count'].fillna(0,  inplace=True)\ntrain['user_std'].fillna(train['user_std'].median(),  inplace=True)\ntrain['user_sem'].fillna(train['user_sem'].median(),  inplace=True)\ntrain['content_mean'].fillna(train['content_mean'].median(),  inplace=True)\ntrain['content_count'].fillna(0,  inplace=True)\ntrain['task_count'].fillna(0,  inplace=True)\ntrain['task_std'].fillna(train['task_std'].median(),  inplace=True)\ntrain['part_mean'].fillna(train['part_mean'].median(),  inplace=True)\ntrain['user_part_lec'].fillna(0,  inplace=True)\ntrain['user_lot'].fillna(0,  inplace=True)\ntrain['explanation_rate'].fillna(train['explanation_rate'].median(),  inplace=True)\ntrain['incorrect_answer_var'].fillna(0,  inplace=True)\ntrain['all_tag_mean'].fillna(train['all_tag_mean'].mean(skipna=True),  inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isnull().any()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"select_features = [\n    'user_mean', \n    'user_count',\n    'user_std', \n    'user_sem', \n    'content_mean',\n    'content_count',\n    'task_count',\n    'task_std', \n    'part_mean',\n    'prior_question_elapsed_time', \n    'prior_question_had_explanation',\n    'explanation_rate',\n    'user_lot',\n    'incorrect_answer_var',\n    'user_part_lec',\n    'all_tag_mean'\n]\n\ntarget = 'answered_correctly'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = train.groupby('user_id').tail(6)\nx = train[~train.index.isin(y.index)]\ndel train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_params = {\n    'loss_function': 'Logloss',\n    'eval_metric': 'AUC',\n    'task_type': 'GPU',\n    'grow_policy': 'Lossguide',\n    'iterations': 10000\n}\n\ncat_train = Pool(x[select_features], label = x[target])\ncat_valid = Pool(y[select_features], label = y[target])\n\ncat_model = CatBoostClassifier(**cat_params)\ncat_model.fit(cat_train, eval_set = cat_valid,verbose_eval=50, early_stopping_rounds=10, use_best_model = True)\ndel cat_train,cat_valid","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_params = {\n    'eval_metric': 'auc',\n    'tree_method': 'gpu_hist',\n    'n_estimators': 5000\n}\n\nxgb_model = xgb.XGBClassifier(**xgb_params)\nxgb_model.fit(x[select_features], x[target], \n        eval_set=[(y[select_features], y[target])],\n        verbose=50, early_stopping_rounds=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_params = {\n    'objective': 'binary',\n    'metric': 'auc',\n}\n\n\nlgb_train = lgb.Dataset(x[select_features], x[target])\nlgb_eval = lgb.Dataset(y[select_features], y[target], reference=lgb_train)\n\nlgb_model = lgb.train(\n    lgb_params, lgb_train,\n    valid_sets=[lgb_train, lgb_eval],\n    verbose_eval=50,\n    num_boost_round=5000,\n    early_stopping_rounds=10\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb.plot_importance(lgb_model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('cat_auc:', roc_auc_score(y[target], cat_model.predict_proba(y[select_features].values)[:,1]))\n\nprint('xgb_auc:', roc_auc_score(y[target], xgb_model.predict_proba(y[select_features])[:,1]))\n\nprint('lgb_auc:', roc_auc_score(y[target], lgb_model.predict(y[select_features])))\n\nprint('ensemble_auc:', roc_auc_score(y[target], np.average([\n    cat_model.predict_proba(y[select_features].values)[:,1],\n    xgb_model.predict_proba(y[select_features])[:,1],\n    lgb_model.predict(y[select_features])\n    ], axis=0)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del x,y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_cal, content_cal, part_cal, task_cal, user_part_lec, qtag_merge = preprocess(train_df,questions,lectures)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n   \n    test_df = pd.merge(test_df,user_cal, on=['user_id'], how=\"left\")\n    test_df = pd.merge(test_df,content_cal, on=['content_id'], how=\"left\")\n    test_df = pd.merge(test_df,task_cal, on=['task_container_id'], how=\"left\")    \n    test_df = pd.merge(test_df,questions, on=['content_id'], how=\"left\")\n    test_df = pd.merge(test_df,part_cal, on=['part'], how=\"left\")\n    test_df = pd.merge(test_df,user_part_lec, on=['user_id','part'], how=\"left\")\n\n    test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False).astype('int8')\n    test_df['prior_question_elapsed_time'].fillna(test_df['prior_question_elapsed_time'].median(), inplace=True)\n\n    for tag in range(1,7):\n        test_df[f'qtag_{tag}'] = pd.to_numeric(test_df[f'qtag_{tag}'], errors='coerce')\n        test_df = pd.merge(test_df,qtag_merge,left_on=f'qtag_{tag}',right_on='qtag',how=\"left\").drop(columns={'qtag',f'qtag_{tag}'})\n        test_df = test_df.rename(columns={'tag_mean': f'tag_mean{tag}'})\n    \n    test_df['all_tag_mean'] = test_df.loc[:,'tag_mean1':'tag_mean6'].mean(axis=1)\n    test_df = test_df.drop(['tag_mean1','tag_mean2','tag_mean3','tag_mean4','tag_mean5','tag_mean6'], axis=1)\n      \n    test_df['user_mean'].fillna(test_df['user_mean'].median(),  inplace=True)\n    test_df['user_count'].fillna(0,  inplace=True)\n    test_df['user_std'].fillna(test_df['user_std'].median(),  inplace=True)\n    test_df['user_sem'].fillna(test_df['user_sem'].median(),  inplace=True)\n\n    test_df['content_mean'].fillna(test_df['content_mean'].median(),  inplace=True)\n    test_df['content_count'].fillna(0,  inplace=True)\n\n    test_df['task_count'].fillna(0,  inplace=True)\n    test_df['task_std'].fillna(test_df['task_std'].median(),  inplace=True)\n    \n    test_df['part_mean'].fillna(test_df['part_mean'].median(),  inplace=True)\n\n    test_df['user_part_lec'].fillna(0,  inplace=True)\n    test_df['user_lot'].fillna(0,  inplace=True)\n    test_df['explanation_rate'].fillna(test_df['explanation_rate'].median(),  inplace=True)\n    test_df['incorrect_answer_var'].fillna(0,  inplace=True)\n    test_df['all_tag_mean'].fillna(test_df['all_tag_mean'].median(),  inplace=True)\n\n    test_df['answered_correctly'] = np.average([        \n        cat_model.predict_proba(test_df[select_features].values)[:,1],\n        xgb_model.predict_proba(test_df[select_features])[:,1],\n        lgb_model.predict(test_df[select_features])\n    ], axis=0)\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}