{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport riiideducation\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nimport os\nfrom typing import List, Dict, Optional\nfrom sklearn.model_selection import RepeatedKFold\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nimport math\nimport time\nimport random\nimport lightgbm as lgb\nimport gc\nfrom sklearn.preprocessing import LabelEncoder\nfrom numba import jit\nfrom sklearn.model_selection import StratifiedKFold #KFold, RepeatedKFold, GroupKFold, \n#GridSearchCV, train_test_split, TimeSeriesSplit\nfrom sklearn import metrics \n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Helper Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage(deep=True).sum() / 1024**2\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                c_prec = df[col].apply(lambda x: np.finfo(x).precision).max()\n                if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max and c_prec == np.finfo(np.float32).precision:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    end_mem = df.memory_usage(deep=True).sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@jit\ndef fast_auc(y_true, y_prob):\n    \"\"\"\n    fast roc_auc computation: https://www.kaggle.com/c/microsoft-malware-prediction/discussion/76013\n    \"\"\"\n    y_true = np.asarray(y_true)\n    y_true = y_true[np.argsort(y_prob)]\n    nfalse = 0\n    auc = 0\n    n = len(y_true)\n    for i in range(n):\n        y_i = y_true[i]\n        nfalse += (1 - y_i)\n        auc += y_i * nfalse\n    auc /= (nfalse * (n - nfalse))\n    return auc\n\ndef eval_auc(y_true, y_pred):\n    \"\"\"\n    Fast auc eval function for lgb.\n    \"\"\"\n    return 'auc', fast_auc(y_true, y_pred), True\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Data Overview"},{"metadata":{"trusted":true},"cell_type":"code","source":"!wc -l /kaggle/input/riiid-test-answer-prediction/train.csv\n!wc -l /kaggle/input/riiid-test-answer-prediction/lectures.csv\n!wc -l /kaggle/input/riiid-test-answer-prediction/questions.csv","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"More than 100m rows in data train"},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '/kaggle/input'\n\ntrain = pd.read_csv(f'{path}/riiid-test-answer-prediction/train.csv',\n                    usecols=['timestamp', 'user_id', 'content_id', 'content_type_id', 'user_answer', 'answered_correctly',\n                             'prior_question_elapsed_time', 'prior_question_had_explanation'],\n                       dtype={'timestamp': 'int64',\n                              'user_id': 'int32',\n                              'content_id': 'int16',\n                              'content_type_id': 'int8',\n                              'user_answer': 'int8',\n                              'answered_correctly': 'int8',\n                              'prior_question_elapsed_time': 'float32', \n                              'prior_question_had_explanation': 'boolean',\n                             }\n                      )\ntrain = train.sort_values(['timestamp'], ascending=True)\nquestions = pd.read_csv(f'{path}/riiid-test-answer-prediction/questions.csv')\nlectures = pd.read_csv(f'{path}/riiid-test-answer-prediction/lectures.csv')\nprint('Train shapes: ', train.shape)\nprint('Questions shape: ', questions.shape)\nprint('Lectures shape: ', lectures.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.sample(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['answered_correctly'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions['correct_answer'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Exploring the Features"},{"metadata":{"trusted":true},"cell_type":"code","source":"_ = plt.hist(train['timestamp'], bins=50)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.groupby(['user_id'])['timestamp'].max().sort_values(ascending=False).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_ = plt.hist(train.groupby(['user_id'])['timestamp'].max().sort_values(ascending=False))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"user_id"},{"metadata":{"trusted":true},"cell_type":"code","source":"train['user_id'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_ = plt.hist(train['user_id'].value_counts())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"content_id"},{"metadata":{"trusted":true},"cell_type":"code","source":"train['content_id'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_ = plt.hist(train['content_id'], bins=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.loc[train['content_id'] == 6116]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.loc[train['content_id'] == 6116, 'user_answer'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions.loc[questions['question_id'] == 6116]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"content_type_id"},{"metadata":{"trusted":true},"cell_type":"code","source":"train['content_type_id'].value_counts()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.loc[train['content_type_id'] == 1, 'user_id'].nunique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Feature engineering\n\nsome code is taken from https://www.kaggle.com/ilialar/simple-eda-and-baseline \nhttps://www.kaggle.com/lgreig/simple-lgbm-baseline"},{"metadata":{"trusted":true},"cell_type":"code","source":"# filter out lectures\ntrain = train.loc[train['answered_correctly'] != -1].reset_index(drop=True)\ntrain = train.drop(['timestamp','content_type_id'], axis = 1)\ntrain['prior_question_had_explanation'] = train['prior_question_had_explanation'].fillna(value = False).astype(bool)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_answers_df = train.groupby('user_id').agg({'answered_correctly': ['mean', 'count']}).copy()\nuser_answers_df.columns = ['mean_user_accuracy', 'questions_answered']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"content_answers_df = train.groupby('content_id').agg({'answered_correctly': ['mean', 'count']}).copy()\ncontent_answers_df.columns = ['mean_accuracy', 'question_asked']\n# user_content_answers_df = train.groupby(['user_id', 'content_id']).agg({'answered_correctly': ['mean', 'count']}).copy()\n# user_content_answers_df.columns = ['mean_user_content_accuracy', 'content_questions_answered']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Now we will use only a part of data for training, to avoid leaks and memory error\n# propotion = 0.15\n# train = train.sample(frac=propotion)\n# train.sample(5)\ntrain = train.iloc[:35000000,:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"na_cols = (train.isna().sum())\nna_cols[na_cols > 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = train\nave_prior = df_train['prior_question_elapsed_time'].mean()\ndf_train['prior_question_elapsed_time'] = df_train['prior_question_elapsed_time'].fillna(ave_prior)\ndf_train['prior_question_elapsed_time'].isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = df_train.merge(user_answers_df, how = 'left', on = 'user_id')\ndf_train = df_train.merge(content_answers_df, how = 'left', on = 'content_id')\n# train = train.merge(user_content_answers_df, how = 'left', on = ['user_id', 'content_id'])\ndf_train.fillna(value = 0.5, inplace = True)\n# train['mean_diff1'] = train['mean_user_accuracy'] - train['mean_user_content_accuracy']\n# train['mean_diff2'] = train['mean_accuracy'] - train['mean_user_content_accuracy']\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"le = LabelEncoder()\ndf_train[\"prior_question_had_explanation\"] = le.fit_transform(df_train[\"prior_question_had_explanation\"])\ndf_train = df_train.sort_values(['user_id'])\ny = df_train['answered_correctly']\n\ncolumns = ['mean_user_accuracy', 'questions_answered', 'mean_accuracy', 'question_asked', 'prior_question_elapsed_time',\n           'prior_question_had_explanation',# 'mean_diff1', 'mean_diff2', 'mean_user_content_accuracy'\n          ]\nX = df_train[columns]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del df_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scores = []\nfeature_importance = pd.DataFrame()\nmodels = []\nparams = {'num_leaves': 450,\n          'max_bin': 700,\n          'objective': 'binary',\n          'max_depth': -1,\n          'learning_rate': 0.05,\n          \"boosting_type\": \"gbdt\",\n          \"metric\": 'auc',\n          'bagging_seed': 11,\n          'random_state': 50\n         }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"columns = ['mean_user_accuracy', 'questions_answered', 'mean_accuracy', 'question_asked',\n           'prior_question_had_explanation', 'prior_question_elapsed_time' #'mean_diff1', 'mean_diff2'\n          ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold\n#from sklearn.model_selection import RepeatedKFold\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import TimeSeriesSplit","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"folds = StratifiedKFold(n_splits=5, shuffle=False)\nfor fold_n, (train_index, valid_index) in enumerate(folds.split(X, y)):\n    print(f'Fold {fold_n} started at {time.ctime()}')\n    X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]\n    y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]\n    model = lgb.LGBMClassifier(**params, n_estimators=1000, n_jobs = 1)\n    model.fit(X_train, y_train, \n            eval_set=[(X_train, y_train), (X_valid, y_valid)], eval_metric=eval_auc,\n            verbose=1000, early_stopping_rounds=10)\n    score = max(model.evals_result_['valid_1']['auc'])\n    \n    models.append(model)\n    scores.append(score)\n\n    fold_importance = pd.DataFrame()\n    fold_importance[\"feature\"] = columns\n    fold_importance[\"importance\"] = model.feature_importances_\n    fold_importance[\"fold\"] = fold_n + 1\n    feature_importance = pd.concat([feature_importance, fold_importance], axis=0)\n    break\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_importance[\"importance\"] /= 1\ncols = feature_importance[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(\n    by=\"importance\", ascending=False)[:50].index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_features = feature_importance.loc[feature_importance.feature.isin(cols)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(16, 12));\nsns.barplot(x=\"importance\", y=\"feature\", data=best_features.sort_values(by=\"importance\", ascending=False));\nplt.title('LGB Features (avg over folds)');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del X, y","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Making Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    y_preds = []\n    test_df = test_df.merge(user_answers_df, how = 'left', on = 'user_id')\n    test_df = test_df.merge(content_answers_df, how = 'left', on = 'content_id')\n#     test_df = test_df.merge(user_content_answers_df, how = 'left', on = ['user_id', 'content_id'])\n#     test_df['mean_diff1'] = test_df['mean_user_accuracy'] - test_df['mean_user_content_accuracy']\n#     test_df['mean_diff2'] = test_df['mean_accuracy'] - test_df['mean_user_content_accuracy']\n    test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(value = False).astype(bool)\n    test_df = test_df.loc[test_df['content_type_id'] == 0].reset_index(drop=True)\n    test_df.fillna(value = 0.5, inplace = True)\n    test_df[\"prior_question_had_explanation_enc\"] = le.fit_transform(test_df[\"prior_question_had_explanation\"])\n\n    for model in models:\n        y_pred = model.predict_proba(test_df[columns], num_iteration=model.best_iteration_)[:, 1]\n        y_preds.append(y_pred)\n\n    y_preds = sum(y_preds) / len(y_preds)\n    test_df['answered_correctly'] = y_preds\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}