{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Riiid! Answer Correctness Prediction with help lightgbm classifier with futures and parameters tuning. Beginners guide."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import optuna\nfrom sklearn.feature_selection import RFECV\nfrom lightgbm import LGBMClassifier\nimport pandas as pd\nimport joblib\nimport numpy as np\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import train_test_split\nimport optuna.integration.lightgbm as lgb\nfrom optuna.samplers import TPESampler\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1. Data preprocessing."},{"metadata":{},"cell_type":"markdown","source":"### 1.1 \n### Let's read the data from attached files(train.csv, questions.csv) into the specified columns."},{"metadata":{"trusted":true},"cell_type":"code","source":"used_data_types_dict = {\n    'timestamp': 'int64',\n    'user_id': 'int32',\n    'content_id': 'int16',\n    'answered_correctly': 'int8',\n    'prior_question_elapsed_time': 'float32',\n    'prior_question_had_explanation': 'boolean'\n}\n\ntrain_df = pd.read_csv(\n    '../input/riiid-test-answer-prediction/train.csv',\n    usecols=used_data_types_dict.keys(),\n    dtype=used_data_types_dict\n)\n\nquestions_df = pd.read_csv(\n    '../input/riiid-test-answer-prediction/questions.csv',\n    usecols=[0, 3],\n    dtype={'question_id': 'int16', 'part': 'int8'}\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 1.2 \n### Attach data frame from *questions_df* to *train_df* using pandas method *merge*. Joining occurs on the specified columns: *content_id* in *train_df* and *question_id* in *questions_df*."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.merge(train_df, questions_df, left_on='content_id', right_on='question_id', how='left')\ntrain_df.drop(columns=['question_id'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 1.3 \n### Fill NaN with mean values for numerical features and the most common for boolean ones. To find out the most common value in columns or distribution of values, we use  *matplotlib.pyplot* methods: *pie*(categorial/boolean) or *hist*(numerical) or something else.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['prior_question_had_explanation'].fillna(bool(True), inplace=True)\ntrain_df = train_df.replace([-np.inf, np.inf], np.nan)\ntrain_df = train_df.fillna(train_df.mean())\ntrain_df['prior_question_had_explanation'] = train_df['prior_question_had_explanation'].astype(bool)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = [bool(False), bool(True)]\nsizes = [list(train_df['prior_question_had_explanation']).count(labels[0]),\n        list(train_df['prior_question_had_explanation']).count(labels[1])]\n\nplt.pie(sizes, labels=labels, shadow=True, startangle=90, autopct='%1.1f%%')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 1.4 \n### Сreate all possible features based by *'answered_correctly'*, grouping values first by *'user_id'*, then by *'content_id'* with using *dataframe* method *group_by* and genetating features with *agg* functions.\n\n### Write the resulting data for convenience in a file with help *read_csv*."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df[train_df['answered_correctly'] != -1]\ntrain_questions_only_df = train_df\n\ngrouped_by_user_df = train_questions_only_df.groupby('user_id')\nuser_answers_df = grouped_by_user_df.agg({'answered_correctly': ['mean', 'count', 'skew',\n                                                                 'std', 'var', 'sem',\n                                                                               'sum']}).copy()\nuser_answers_df.columns = [\n    'mean_user_accuracy',\n    'questions_answered',\n    'questions_skew',\n    'questions_std',\n    'questions_var',\n    'questions_sem',\n    'questions_sum'\n]\nuser_answers_df.to_csv('user.csv')\n\ngrouped_by_content_df = train_questions_only_df.groupby('content_id')\ncontent_answers_df = grouped_by_content_df.agg({'answered_correctly': ['mean', 'count', 'skew',\n                                                                       'std', 'var', 'sem',\n                                                                                     'sum']}).copy()\ncontent_answers_df.columns = [\n    'content_mean',\n    'question_asked',\n    'content_skew',\n    'content_std',\n    'content_var',\n    'content_sem',\n    'content_sum'\n]\ncontent_answers_df.to_csv('content.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Repeat 1.2 to join the data from the previous point to the main data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.merge(user_answers_df, how='left', on='user_id')\ntrain_df = train_df.merge(content_answers_df, how='left', on='content_id')\ntrain_df = train_df.replace([-np.inf, np.inf], np.nan)\ntrain_df = train_df.fillna(train_df.mean()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 1.5\n### Separating the training features from the target"},{"metadata":{"trusted":true},"cell_type":"code","source":"features = [\n    'timestamp',\n    'user_id',\n    'content_id',\n    'prior_question_elapsed_time',\n    'prior_question_had_explanation',\n\n    'part',\n\n    'mean_user_accuracy',\n    'questions_skew',\n    'questions_std',\n    'questions_var',\n    'questions_sem',\n\n    'content_mean',\n    'content_skew',\n    'content_std',\n    'content_var',\n    'content_sem'\n]\n\ntarget = 'answered_correctly'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2\n## Features selection with RFECV."},{"metadata":{},"cell_type":"markdown","source":"### 2.1\n### Let's create the simplest classifier for recursive selection of features using *sklearn.feature_selection.RFECV*. Train selector on our data."},{"metadata":{"trusted":true},"cell_type":"code","source":"selection_df = train_df[:20_000_000]\n\nselect_model = LGBMClassifier()\nselector = RFECV(select_model, step=1, cv=3, n_jobs=12, verbose=10, min_features_to_select=6)\nselector.fit(selection_df[features], selection_df[target])\n\njoblib.dump(selector, 'Selector.joblib') # save the selector on disk","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2.2\n### Using a logical mask(*selectoe.support_*), we select the most useful features. You can also find out the rank assigned to each feature(*selector.ranking_*)."},{"metadata":{"trusted":true},"cell_type":"code","source":"features = [features[i] for i in range(len(selector.support_)) if selector.support_[i] == True]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3\n## Parameters selection with optuna."},{"metadata":{},"cell_type":"markdown","source":"### 3.1 \n### Create train and test data with *sklearn.model_selection.train_test_split*."},{"metadata":{"trusted":true},"cell_type":"code","source":"optuna_df = train_df[:20_000_000]\n\nXt, Xv, Yt, Yv = train_test_split(optuna_df[features], optuna_df[target], test_size=0.3, shuffle=True)\nlgb_train = lgb.Dataset(Xt, Yt)\nlgb_eval = lgb.Dataset(Xv, Yv)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 3.2\n### Let's define functions that return a model with a list of LGBMClassifier parameters(*create_model*) and its accuracy on test data(*objective*)."},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(trial):\n    params = {\n        'num_leaves': trial.suggest_int('num_leaves', 32, 512),\n        'boosting_type': 'gbdt',\n        'objective': 'binary',\n        'metric': 'auc',\n        'learning_rate': trial.suggest_uniform('learning_rate', 0.05, 0.5),\n        'max_depth': trial.suggest_int('max_depth', 3, 18),\n        'min_child_weight': trial.suggest_int('min_child_weight', 1, 20),\n        'feature_fraction': trial.suggest_uniform('feature_fraction', 0.4, 1.0),\n        'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),\n        'bagging_freq': trial.suggest_int('bagging_freq', 1, 8),\n        'min_child_samples': trial.suggest_int('min_child_samples', 4, 80),\n        'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-8, 1.0),\n        'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-8, 1.0),\n        'n_estimators': trial.suggest_int('n_estimators', 200, 600),\n        'random_state': 42\n    }\n    \n    model = LGBMClassifier(**params)\n    return model\n\ndef objective(trial):\n    model = create_model(trial)\n    model.fit(Xt, Yt)\n    preds = model.predict_proba(Xv)[:, 1]\n    score = roc_auc_score(Yv, preds)\n    return score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 3.3 \n### Finally, let's create a optuna model that iterates through the parameters from the given list and selects the best ones."},{"metadata":{"trusted":true},"cell_type":"code","source":"sampler = TPESampler(seed=42)\nstudy = optuna.create_study(direction='maximize', sampler=sampler)\nstudy.optimize(objective, n_trials=50) # n_trials - number of parameter sets and model\n\nparams = study.best_params # get the best set of parameters from these n_trials sets\n\njoblib.dump(study, 'Study_optuna.joblib')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### The parameters I got:"},{"metadata":{"trusted":true},"cell_type":"code","source":"best_params = {'num_leaves': 392, \n               'learning_rate': 0.14812766987568138, \n               'max_depth': 11, \n               'min_child_weight': 13, \n               'feature_fraction': 0.9829084591151024, \n               'bagging_fraction': 0.9793416187075863, \n               'bagging_freq': 5, \n               'min_child_samples': 22, \n               'reg_alpha': 0.8989695252132637,\n               'reg_lambda': 0.024084559071289695, \n               'n_estimators': 397\n              }","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 4\n## LGBMClassifier with chosen parameters and features training."},{"metadata":{},"cell_type":"markdown","source":"### 4.1\n### Repeat 3.1, but on all dataset."},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(\n    train_df[features], train_df[target], test_size=0.2, random_state=42\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 4.2\n### Prepare the model with the selected parameters and train it. "},{"metadata":{"trusted":true},"cell_type":"code","source":"model = LGBMClassifier(**params)\nmodel.fit(X_train, y_train)\n\njoblib.dump(model, 'Final_lgb.joblib')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 5\n## Submitting result."},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### For speed, let's execute all the above code on the local computer and load the resulting models and data on kaggle."},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df = pd.read_csv(\n    '../input/riiid-test-answer-prediction/questions.csv',\n    usecols=[0, 3],\n    dtype={'question_id': 'int16', 'part': 'int8'}\n)\n\nmodel = joblib.load()\nselector = joblib.load('../input/selector/Selector.joblib')\n\nuser_answers_df = read_csv('../input/preprocessingcontentuser/user.csv')\ncontent_answers_df = read_csv('../input/preprocessingcontentuser/content.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    \n    test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')\n    test_df.drop(columns=['question_id'], inplace=True)\n    test_df['prior_question_had_explanation'].fillna(bool(True), inplace=True)\n    test_df = test_df.replace([-np.inf, np.inf], np.nan)\n    test_df = test_df.fillna(test_df.mean())\n    \n    test_df = test_df[test_df['content_type_id'] != 1]\n    \n    test_df = test_df.merge(user_answers_df, how='left', on='user_id')\n    test_df = test_df.merge(content_answers_df, how='left', on='content_id')\n    test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].astype(bool)\n    test_df = test_df.replace([-np.inf, np.inf], np.nan)\n    test_df = test_df.fillna(test_df.mean())\n\n    \n    test_df['answered_correctly'] = model.predict_proba(test_df[features])[:,1]\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### That's all."},{"metadata":{},"cell_type":"markdown","source":"### The accuracy i got: 0.753"},{"metadata":{},"cell_type":"markdown","source":"### Good luck to all!"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}