{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt # plots\nimport seaborn as sns # plots\nimport gc\nimport riiideducation\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\n!cp ../input/rapids/rapids.0.17.0 /opt/conda/envs/rapids.tar.gz\n!cd /opt/conda/envs/ && tar -xzvf rapids.tar.gz > /dev/null\nsys.path = [\"/opt/conda/envs/rapids/lib/python3.7/site-packages\"] + sys.path\nsys.path = [\"/opt/conda/envs/rapids/lib/python3.7\"] + sys.path\nsys.path = [\"/opt/conda/envs/rapids/lib\"] + sys.path \n!cp /opt/conda/envs/rapids/lib/libxgboost.so /opt/conda/lib/","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Riiid answer prediction - XGBoost\n\n## Steps\n1. Load\n2. Process\n3. Model\n4. Evaluate"},{"metadata":{},"cell_type":"markdown","source":"## 1. Load"},{"metadata":{},"cell_type":"markdown","source":"### Issue    : Data volume  \n### Solution : RAPIDS library & Kaggle GPU (39H/week)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Rapids Imports\nimport cudf\nimport cupy # CuPy is an open-source array library accelerated with NVIDIA CUDA.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data : *train.csv*"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# Read in data\ndtypes = {\n    \"row_id\": \"int64\",\n    \"timestamp\": \"int64\",\n    \"user_id\": \"int32\",\n    \"content_id\": \"int16\",\n    \"content_type_id\": \"int8\",\n    \"task_container_id\": \"int16\",\n    \"user_answer\": \"int8\",\n    \"answered_correctly\": \"int8\",\n    \"prior_question_elapsed_time\": \"float32\", \n    \"prior_question_had_explanation\": \"int8\"\n}\n\ntrain = cudf.read_csv('../input/riiid-test-answer-prediction/train.csv', dtype=dtypes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Fill in missing values with \"-1\"\ntrain[\"prior_question_elapsed_time\"] = train[\"prior_question_elapsed_time\"].fillna(-1)\ntrain[\"prior_question_had_explanation\"] = train[\"prior_question_had_explanation\"].fillna(-1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"___"},{"metadata":{},"cell_type":"markdown","source":"# 2. Process"},{"metadata":{"trusted":true},"cell_type":"code","source":"def aggregations(frame, target):\n    \"\"\"\n    Thus function create aggregations data.\n    \"\"\"\n    cols = ['user_id']  # Columns to aggregate\n    \n    aggs = ['mean']  # List of aggregation functions\n    \n    aggs_dfs = []  # List of aggregated DataFrame\n    \n    for col in cols:  # Loop over the columns to aggregate\n        \n        df = frame.groupby(col).agg({target : aggs})\n        \n        df.columns = [ col[0] + new_column for new_column in df.columns.droplevel()]\n        \n        df[col] = df.index  # Add the index as column for the merge\n        \n        frame = frame.merge(df, on=col)  # Merge based on the same column\n        \n        aggs_dfs.append(df)\n        \n    return frame, aggs_dfs\n\ndef preprocess_frame(frame, features, target):\n    \"\"\"\n    This function do the preprocessing on the dataframe and the feature\n    engineering.\n    \"\"\"\n    \n    frame = frame[features]  # Working only on features\n        \n    frame, aggs_dfs = aggregations(frame, target)  # Do the aggregations\n\n    \n    return frame, aggs_dfs","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"cudf.set_allocator(\"managed\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# Let's exclude all observations where (content_type_id = 1) & (answered_correctly = -1)\ntrain = train[train['content_type_id'] != 1]\ntrain = train[train['answered_correctly'] != -1].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# RAPIDS roc_auc_score is 16x faster than sklearn. - cdeotte\nfrom cuml.metrics import roc_auc_score\nfrom cuml.preprocessing.model_selection import train_test_split\nimport xgboost\nimport pickle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_xgb_model(X_train, X_test, y_train, y_test, params, prints=True):\n    '''Trains an XGB and returns the trained model + ROC value.'''\n    # Create DMatrix - is optimized for both memory efficiency and training speed.\n    train_matrix = xgboost.DMatrix(data = X_train, label = y_train)\n    \n    # Create & Train the model\n    model = xgboost.train(params, dtrain = train_matrix)\n\n    # Make prediction\n    predicts = model.predict(xgboost.DMatrix(X_test))\n    roc = roc_auc_score(y_test.astype('int32'), predicts)\n\n    if prints:\n        print(\"ROC: {:.5}\".format(roc))\n    \n    return model, roc\n\n\ndef param_tuning_graph(param_values, roc_values):\n    '''Represents visually the ROC results for the speciffic parameter tune.'''\n    \n    plt.figure(figsize=(18, 3))\n    ax = sns.barplot(x=param_values, y=roc_values, palette=custom_colors)\n\n    for p in ax.patches:\n        width = p.get_width()\n        height = p.get_height()\n        x, y = p.get_xy() \n        ax.annotate(f'{height:.5%}', (x + width/2, y + height*1.02), ha='center')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntarget = 'answered_correctly'\n\n# Preprocessing\ntrain_proc, u_aggs = preprocess_frame(train, train.columns.tolist(), target)\n#train_proc = train\n\nfeatures = train_proc.columns.tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features.remove('answered_correctly')\nfeatures.remove('user_answer')\nfeatures.remove('row_id')\nfeatures.remove('user_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# Features, target and train/test split\nX = train_proc[features]\ny = train_proc[target]\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, shuffle=True, stratify=y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params1 = {\n    'max_depth' : 4,\n    'max_leaves' : 2**4,\n    'tree_method' : 'gpu_hist',\n    'objective' : 'reg:logistic',\n    'grow_policy' : 'lossguide',\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model, roc = train_xgb_model(X_train, X_test, y_train, y_test, params1, prints=True)\n\nversion = \"xgb_v7\"\n#model.save_model(version)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = xboost.load_model(version)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. Hyperparameter tuning"},{"metadata":{},"cell_type":"markdown","source":"___"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_from(model, Xs, threshold=0.6):\n    \"\"\"\n    This function get the predictions from a given pandas dataframe format\n    in need to be converted to the model specifics.\n    \"\"\"\n    \n    dmatrix = xgboost.DMatrix(Xs)  # Convert DataFrame column to DMatrix\n    \n    predictions_probas = model.predict(dmatrix)  # Get the probas of predictions\n    \n    predictions = predictions_probas > threshold  # Get True or False\n    \n    return predictions.astype(int)  # Predictions with 1 for True and 0 for False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def link_to_aggs(Xs, aggs, col):\n    \n    Xs = cudf.from_pandas(Xs)  # Convert pandas to cudf\n        \n    Xs = Xs.merge(aggs, how='left', on=col)  # Merge cudf DataFrames\n    \n    return Xs.to_pandas()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5. Predict and Submit"},{"metadata":{"trusted":true},"cell_type":"code","source":"features_submission = ['row_id', 'timestamp', 'user_id', 'content_id', 'content_type_id',\n       'task_container_id', 'prior_question_elapsed_time',\n       'prior_question_had_explanation', 'prior_group_answers_correct',\n       'prior_group_responses']\n\nf_sub = set(features_submission)  # Features available for submission\n\ndtypes_sub = {\n    \"timestamp\": \"int64\",\n    \"user_id\": \"int32\",\n    \"content_id\": \"int16\",\n    \"content_type_id\": \"int8\",\n    \"task_container_id\": \"int16\",\n    \"prior_question_elapsed_time\": \"float32\", \n    \"prior_question_had_explanation\": \"int8\"\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f_train = set(features)  # Features used for training","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(\"Intersection :\", f_sub & f_train)\n# print(\"Difference   :\", f_sub - f_train)\n# print(\"Difference   :\", f_train - f_sub)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"___"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# Create the env\nenv = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# Create the iterator\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# Iter and predict\nfor (test_df, sample_prediction_df) in iter_test:\n    \n    X = test_df.copy()\n    \n    X = X.merge(u_aggs[0].to_pandas().reset_index(drop=True), how='left', on='user_id')  # Add the aggregated data\n    # X = link_to_aggs(X, c_aggs, 'content_id')  # Add the aggregated data\n    \n    # Preprocessing block\n    X[\"prior_question_elapsed_time\"]    = X[\"prior_question_elapsed_time\"].fillna(-1)\n    X[\"prior_question_had_explanation\"] = X[\"prior_question_had_explanation\"].fillna(False)\n    \n    X = X.astype(dtypes_sub)  # Only take defined features\n        \n    predictions = predict_from(model, X[features], 0.65)  # Get predictions\n    \n    test_df['answered_correctly'] = predictions  # Assign predictions\n    \n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"___"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}