{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Beating the benchmark notebook\n## Author: Thomas SELECK\n## Date: 05/10/2020\n\nThe purpose of that notebook is to provide a small piece of code that can be used to beat the benchmark of this competition."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nimport lightgbm as lgb\nimport gc\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\n\nimport riiideducation","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"markdown","source":"# 1. Loading the data\n\nHere, we'll load each file and print few rows to see how the data looks like."},{"metadata":{"trusted":true},"cell_type":"code","source":"st = time.time()\ntraining_set_df = pd.read_csv(\"/kaggle/input/riiid-test-answer-prediction/train.csv\", dtype = {\"row_id\": \"int64\", \"timestamp\": \"int64\", \"user_id\": \"int32\", \"content_id\": \"int16\", \n                                                                                               \"content_type_id\": \"int8\", \"task_container_id\": \"int16\", \"user_answer\": \"int8\", \n                                                                                               \"answered_correctly\": \"int8\", \"prior_question_elapsed_time\": \"float32\", \n                                                                                               \"prior_question_had_explanation\": \"boolean\"}, nrows = 5 * 10 ** 6)\nquestions_df = pd.read_csv(\"/kaggle/input/riiid-test-answer-prediction/questions.csv\", dtype = {\"question_id\": \"int16\", \"bundle_id\": \"int16\", \"correct_answer\": \"int8\", \"part\": \"int8\"})\nlectures_df = pd.read_csv(\"/kaggle/input/riiid-test-answer-prediction/lectures.csv\", dtype = {\"lecture_id\": \"int16\", \"tag\": \"int16\", \"part\": \"int8\"})\ngc.collect()\n\nprint(\"Loaded data in:\", round(time.time() - st, 3), \"secs\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"training_set_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lectures_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Data preprocessing\n\nHere, we'll preprocess the data to make it usable by the predictive model."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Drop index for train\ntraining_set_df.drop(\"row_id\", axis = 1, inplace = True)\n\n# Cast boolean column\ntraining_set_df[\"prior_question_had_explanation\"] = training_set_df[\"prior_question_had_explanation\"].fillna(0).astype(np.int8)\n\n# Extract the target\noriginal_shape = training_set_df.shape[0]\ntraining_set_df = training_set_df.loc[training_set_df[\"answered_correctly\"] != -1]\nprint(\"Deleted\", original_shape - training_set_df.shape[0], \"rows where the target was missing.\")\ntarget_sr = training_set_df[\"answered_correctly\"]\ntraining_set_df.drop([\"answered_correctly\", \"user_answer\"], axis = 1, inplace = True) # Remove also 'user_answer' to avoid leakage","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Prepare 'questions_df' and 'lectures_df' tables for merging with the main table\nquestions_df[\"content_type_id\"] = 0\nquestions_df[\"content_type_id\"] = questions_df[\"content_type_id\"].astype(np.int8)\nquestions_df = questions_df.rename(columns = {\"question_id\": \"content_id\"})\nlectures_df[\"content_type_id\"] = 1\nlectures_df[\"content_type_id\"] = lectures_df[\"content_type_id\"].astype(np.int8)\nlectures_df = lectures_df.rename(columns = {\"lecture_id\": \"content_id\"})\n\n# Merge 'questions_df' and 'lectures_df' tables to the main dataset\ntraining_set_df = training_set_df.merge(questions_df, how = \"left\", on = [\"content_id\", \"content_type_id\"])\ntraining_set_df = training_set_df.merge(lectures_df, how = \"left\", on = [\"content_id\", \"content_type_id\"])\ntraining_set_df[\"part\"] = (training_set_df[\"part_x\"].fillna(0) + training_set_df[\"part_y\"].fillna(0)).astype(np.int8)\ntraining_set_df.drop([\"part_x\", \"part_y\"], axis = 1, inplace = True)\ntraining_set_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Remove constant features\ntmp = training_set_df.nunique()\nconstant_features_lst = tmp.loc[tmp < 2].index.tolist()\nif len(constant_features_lst) > 0:\n    print(\"Found\", len(constant_features_lst), \"constant features:\")\n    for f in constant_features_lst:\n        print(\"  - \" + f)\n        \n    training_set_df.drop(constant_features_lst, axis = 1, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Drop \"tags\" feature\ntraining_set_df.drop(\"tags\", axis = 1, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Encode categorical features\ncategorical_features_lst = [\"task_container_id\", \"prior_question_had_explanation\", \"bundle_id\", \"correct_answer\", \"part\"] # \"user_id\"\nlabel_encoders_dict = {}\n\nfor col in categorical_features_lst:\n    le = LabelEncoder()\n    training_set_df[col] = le.fit_transform(training_set_df[col])\n    label_encoders_dict[col] = le","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Train LightGBM model\n\nHere, we'll train a single LightGBM model."},{"metadata":{"trusted":true},"cell_type":"code","source":"st = time.time()\n\n# Hyperparameters for LightGBM\nlgb_params = {\n    \"boosting_type\": \"gbdt\",\n    \"metric\": \"auc\",\n    \"objective\": \"binary\",\n    \"n_jobs\": 4,\n    \"seed\": 42,\n    \"learning_rate\": 0.03,\n    \"subsample\": 0.75,\n    \"bagging_freq\": 1,\n    \"colsample_bytree\": 0.77,\n    \"max_depth\": -1,\n    \"num_leaves\": 40,\n    \"reg_alpha\": 0.05,\n    \"reg_lambda\": 0.05,\n    \"verbosity\": -1\n}\n\n# Split training data into training and validation datasets\nX_train, X_valid, y_train, y_valid = train_test_split(training_set_df, target_sr, test_size = 0.20, random_state = 42)\ndel training_set_df\ngc.collect()\n\n# Generate LightGBM datasets\nlgb_train = lgb.Dataset(X_train, y_train, categorical_feature = categorical_features_lst)\nlgb_eval = lgb.Dataset(X_valid, y_valid, categorical_feature = categorical_features_lst)\n\n# Try to save some memory\ngc.collect()\n\nlgb_model = lgb.train(lgb_params, lgb_train, valid_sets = [lgb_train, lgb_eval], verbose_eval = 10, num_boost_round = 500, early_stopping_rounds = 50)\ngc.collect()\n\n# Try to save some memory\ngc.collect()\n\nprint(\"Trained LightGBM in:\", round(time.time() - st, 3), \"secs\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Free some memory by deletng training set\ndel X_train, X_valid, y_train, y_valid, lgb_train, lgb_eval\n\n# Try to save some memory\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. Make predictions\n\nHere, we'll make predictions for the test set using the Kaggle API."},{"metadata":{"trusted":true},"cell_type":"code","source":"st = time.time()\n\n# Generate the submission environment\nsubmission_env = riiideducation.make_env()\n\n# Actually make predictions\nfor (testing_set_df, sample_prediction_df) in submission_env.iter_test():    \n    X_test = testing_set_df.drop([\"prior_group_answers_correct\", \"prior_group_responses\", \"row_id\"], axis = 1)\n        \n    # Cast boolean column\n    X_test[\"prior_question_had_explanation\"] = X_test[\"prior_question_had_explanation\"].fillna(0).astype(np.int8)\n    \n    # Merge 'questions_df' and 'lectures_df' tables to the main dataset\n    X_test = X_test.merge(questions_df, how = \"left\", on = [\"content_id\", \"content_type_id\"])\n    X_test = X_test.merge(lectures_df, how = \"left\", on = [\"content_id\", \"content_type_id\"])\n    X_test[\"part\"] = (X_test[\"part_x\"].fillna(0) + X_test[\"part_y\"].fillna(0)).astype(np.int8)\n    X_test.drop([\"part_x\", \"part_y\"], axis = 1, inplace = True)\n    \n    # Drop constant features\n    if len(constant_features_lst) > 0:\n        X_test.drop(constant_features_lst, axis = 1, inplace = True)\n        \n    # Drop \"tags\" feature\n    X_test.drop(\"tags\", axis = 1, inplace = True)\n    \n    # Encode categorical features\n    for i, col in enumerate(categorical_features_lst):\n        X_test[col] = label_encoders_dict[col].transform(X_test[col])\n    \n    # Make predictions using LightGBM\n    predictions_npa = lgb_model.predict(X_test, num_iteration = lgb_model.best_iteration)\n    testing_set_df[\"answered_correctly\"] = predictions_npa\n    submission_env.predict(testing_set_df.loc[testing_set_df[\"content_type_id\"] == 0, [\"row_id\", \"answered_correctly\"]])\n    \n    # Try to save some memory\n    gc.collect()\n    \nprint(\"Made predictions in:\", round(time.time() - st, 3), \"secs\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5. Get LightGBM feature importance"},{"metadata":{"trusted":true},"cell_type":"code","source":"importance = lgb_model.feature_importance(importance_type = \"gain\")\nfeatures_names = lgb_model.feature_name()\nfeature_importance_df = pd.DataFrame({\"feature\": features_names, \"importance\": importance}).sort_values(by = \"importance\", ascending = False).reset_index(drop = True)\nfeature_importance_df.to_csv(\"lgb_feature_importance.csv\", index = False)\nfeature_importance_df.head(30)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}