{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import gc\nfrom sklearn.metrics import roc_auc_score\nfrom collections import defaultdict\nfrom tqdm.notebook import tqdm\nimport xgboost\nfrom xgboost import XGBClassifier\nfrom xgboost import plot_importance\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nfrom tabulate import tabulate\nimport missingno as msno \nfrom IPython.display import display_html\nfrom PIL import Image\nimport gc\nimport cv2\nfrom scipy.stats import pearsonr\nimport tqdm\nimport seaborn as sns\n\nfrom skimage.transform import resize\nimport copy\nimport re\n\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\nwarnings.filterwarnings(\"ignore\", category=UserWarning)\nwarnings.filterwarnings(\"ignore\", category=FutureWarning)\n\n# Color Palette\ncustom_colors = ['#00FFE2', '#00FDFF', '#00BCFF', '#0082FF', '#8000FF', '#B300FF', '#F400FF']\nsns.palplot(sns.color_palette(custom_colors))\n\n# Set Style\nsns.set_style(\"whitegrid\")\nsns.despine(left=True, bottom=True)\n\n# Set tick size\nplt.rc('xtick',labelsize=12)\nplt.rc('ytick',labelsize=12)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check CUDA/cuDNN Version\n!nvcc -V && which nvcc\n!nvidia-smi","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install GPUtil\n\nimport torch\nfrom GPUtil import showUtilization as gpu_usage\nfrom numba import cuda\n\ndef free_gpu_cache():\n    print(\"Initial GPU Usage\")\n    gpu_usage()                             \n    torch.cuda.empty_cache()\n    cuda.select_device(0)\n    cuda.close()\n    cuda.select_device(0)\n    print(\"GPU Usage after emptying the cache\")\n    gpu_usage()\n\nfree_gpu_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport cudf\nimport dask_cudf\n\ntrain = dask_cudf.from_cudf(cudf.read_csv('../input/riiid-test-answer-prediction/train.csv',\n                          dtype={'row_id': 'int64',\n                          'timestamp': 'int64',\n                          'user_id': 'int32',\n                          'content_id': 'int16',\n                          'content_type_id': 'int8',\n                          'task_container_id': 'int16',\n                          'user_answer': 'int8',\n                          'answered_correctly':'int8',\n                          'prior_question_elapsed_time': 'float32',\n                          'prior_question_had_explanation': 'boolean'}\n                   ),npartitions=6).compute()\nprint(train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.head())\nprint(train.isnull().sum())\ncols = train.columns\nfor col in cols: \n    print('Unique values :',{col,train[col].nunique()})\n\ntrain.describe().T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Data Information\nprint(\"Rows: {:,}\".format(len(train)), \"\\n\" +\n      \"Columns: {}\".format(len(train.columns)))\n\n# Find Missing Data if any\ntotal = len(train)\n\n# Fill in missing values with \"-1\"\ntrain[\"prior_question_elapsed_time\"] = train[\"prior_question_elapsed_time\"].fillna(-1)\ntrain[\"prior_question_had_explanation\"] = train[\"prior_question_had_explanation\"].fillna(0)\n\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def distplot_features(df, feature, title, color = custom_colors[4], categorical=True):\n    '''Takes a column from the GPU dataframe and plots the distribution (after count).'''\n    \n    if categorical:\n        values = cupy.asnumpy(df[feature].value_counts().values)\n    else:\n        values = cupy.asnumpy(df[feature].values)\n        \n    print('Mean: {:,}'.format(np.mean(values)), \"\\n\"\n          'Median: {:,}'.format(np.median(values)), \"\\n\"\n          'Max: {:,}'.format(np.max(values)))\n\n    \n    plt.figure(figsize = (18, 3))\n    \n    if categorical:\n        sns.distplot(values, hist=False, color = color, kde_kws = {'lw':3})\n    else:\n        # To speed up the process\n        sns.distplot(values[::250000], hist=False, color = color, kde_kws = {'lw':3})\n    \n    plt.title(title, fontsize=15)\n    plt.show();\n    \n    del values\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def barplot_features(df, feature, title, palette = custom_colors[2:]):\n    '''Takes the numerical columns (with less than 10 categories) and plots the barplot.'''\n    \n    # We need to extract both the name of the category and the no. of appearences\n    index = cupy.asnumpy(df[feature].value_counts().reset_index()[\"index\"].values)\n    values = cupy.asnumpy(df[feature].value_counts().reset_index()[feature].values) \n\n    plt.figure(figsize = (18, 3))\n    sns.barplot(x = index, y = values, palette = custom_colors[2:])\n    plt.title(title, fontsize=15)\n    plt.show();\n    \n    del index, values\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cupy\nnumerical_features = ['timestamp', 'prior_question_elapsed_time']\n\nfor feature in numerical_features:\n    distplot_features(train, feature=feature, title = feature + \" distribution\", color = custom_colors[1], categorical=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"categorical_features = ['user_id', 'content_id', 'task_container_id']\n\nfor feature in categorical_features:\n    distplot_features(train, feature=feature, title = feature + \" countplot distribution\", color = custom_colors[4], categorical=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"categorical_for_bar = ['content_type_id', 'user_answer', \n                       'answered_correctly', 'prior_question_had_explanation']\n\nfor feature in categorical_for_bar:\n    barplot_features(train, feature=feature, title = feature + \" barplot\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions = cudf.read_csv('../input/riiid-test-answer-prediction/questions.csv')\n\n# Data Information\nprint(\"Rows: {:,}\".format(len(questions)), \"\\n\" +\n      \"Columns: {}\".format(len(questions.columns)))\n\n# Find Missing Data if any\ntotal = len(questions)\n\nfor column in questions.columns:\n    if questions[column].isna().sum() != 0:\n        print(\"{} has: {:,} ({:.2}%) missing values.\".format(column, questions[column].isna().sum(), \n                                                             (questions[column].isna().sum()/total)*100))\n        \n        \n# Fill in missing values with \"-1\"\nquestions[\"tags\"] = questions[\"tags\"].fillna(-1)\n\nquestions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for feature in ['part', 'correct_answer']:\n    barplot_features(questions, feature=feature, title=feature + \" - barplot distribution\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"distplot_features(questions, 'tags', title = \"Tags - Count Distribution\", color = custom_colors[0], categorical=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"free_gpu_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#choosing important attributes\ntrain = dask_cudf.from_cudf(cudf.read_csv('../input/riiid-test-answer-prediction/train.csv',\n                          dtype={'row_id': 'int64',\n                          'timestamp': 'int64',\n                          'user_id': 'int32',\n                          'content_id': 'int16',\n                          'content_type_id': 'int8',\n                          'task_container_id': 'int16',\n                          'user_answer': 'int8',\n                          'answered_correctly':'int8',\n                          'prior_question_elapsed_time': 'float32',\n                          'prior_question_had_explanation': 'boolean'}\n                   ),npartitions=6).compute()\nprint(train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_grouped = train.groupby('user_id')\ntrain_grouped.nth(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#split based on user_id and timestamp\ntrain = train[train.content_type_id == False].sort_values('timestamp').reset_index(drop = True)\nprint(train.shape)\ntrain_pd = train.to_pandas()\nvalid_split = train_pd.groupby('user_id').tail(5)\ndel(train_pd)\nvalid_split1 = cudf.from_pandas(valid_split)\ndel(valid_split)\ntrain_split1 = train[~train.row_id.isin(valid_split1.row_id)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del(train_split1)\ndel(valid_split1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !python -m pip --version","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip3 install --upgrade pip","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !git clone https://github.com/rapidsai/rapidsai-csp-utils.git\n# !bash rapidsai-csp-utils/colab/rapids-colab.sh stable\n\n# import sys, os\n\n# # dist_package_index = sys.path.index('/usr/local/lib/python3.7/dist-packages')\n# # sys.path = sys.path[:dist_package_index] + ['/usr/local/lib/python3.7/site-packages'] + sys.path[dist_package_index:]\n# # sys.path\n# exec(open('rapidsai-csp-utils/colab/update_modules.py').read(), globals())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# #split training and testing set\n# #created 5 cross validation sets\n\n# val_size = 2500000\n\n# for cv in range(5):\n#     valid = train[-val_size:]\n#     train = train[:-val_size]\n#     # check new users and new contents\n#     new_users = len(valid[~valid.user_id.isin(train.user_id)].user_id.unique())\n#     valid_question = valid[valid.content_type_id == 0]\n#     train_question = train[train.content_type_id == 0]\n#     new_contents = len(valid_question[~valid_question.content_id.isin(train_question.content_id)].content_id.unique())    \n#     print('cv{cv} {train_question.answered_correctly.mean():.3f} {valid_question.answered_correctly.mean():.3f} {new_users} {new_contents}')\n#     valid.to_pickle(f'cv{cv+1}_valid.pickle')\n#     train.to_pickle(f'cv{cv+1}_train.pickle') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\n \ntrain_pickle ='../input/pickle-files-generated-xgboost/train_pickle.p'\nvalid_pickle ='../input/pickle-files-generated-xgboost/valid_pickle.p'\n \ntrain = pd.read_pickle(train_pickle)\nvalid = pd.read_pickle(valid_pickle)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n!apt update\n!apt-get install -y cmake build-essential\n!git clone https://github.com/dmlc/xgboost.git /tmp/xgboost\n!pushd /tmp/xgboost/python-package\n!git submodule init\n!git submodule update\n!python setup.py install\n!popd\n\nfrom sklearn.metrics import roc_auc_score\n \nFEATS = ['row_id', 'user_id', 'content_id', 'content_type_id',\n       'prior_question_elapsed_time',\n       'prior_question_had_explanation', 'answered_correctly_sum_u', 'count_u',\n       'answered_correctly_avg_u', 'answered_correctly_avg_c',\n       'prior_question_elapsed_time_mean', 'question_id', 'part']\n \nTARGET = ['answered_correctly']\n \nX_train = train[FEATS]\ny_train = train[TARGET]\nX_valid = valid[FEATS]\ny_valid = valid[TARGET]\n\n\nimport xgboost\nfrom xgboost import XGBClassifier\nfrom xgboost import plot_importance\nimport matplotlib.pyplot as plt\n\nparams = {\n    'max_depth' : 12,\n    'max_leaves' : 2**8,\n    'eta':0.6,  # Step size shrinkage used in update to prevents overfitting\n                # After each boosting step, we can directly get the weights of \n    # new features, and eta shrinks the feature weights to make the boosting \n    # process more conservative.\n    'alpha':0.1,     # L1 regularization term on weights. \n    'lambda' : 0.2,  # L2 regularization term on weights. \n                     # Increasing this value will make model more conservative.\n    'min_child_weight':1,\n    'subsample':0.8,  # default = 1, Subsample ratio of the training instances. \n                      # Setting it to 0.5 means that XGBoost would randomly \n                      # sample half of the training data prior to growing trees.\n                      # l prevent overfitting. \n                      # Subsampling will occur once in every boosting iteration.\n    'tree_method' : 'gpu_hist',\n    'learning_rate': 0.5, #default = 0.3,\n    'colsample_bytree':0.7, # is the subsample ratio of columns when constructing each tree. \n                            # Subsampling occurs once for every tree constructed.\n    'eval_metric':'auc', \n    'objective' : 'binary:logistic',\n    'sample_type': 'weighted',\n    'grow_policy' : 'lossguide',\n    'n_estimators': 200,\n    'normalize_type': 'tree',\n    'rate_drop': 0.2,    # dropout rate \n    'skip_drop': 0.1,    # probability of skipping dropout (If a dropout is \n                         # skipped, new trees are added in the same manner as gbtree.)\n    'feature_selector':'thrifty',  # This operation is multithreaded and is a \n    # linear complexity approximation of the quadratic greedy selection. \n    'deterministic_histogram': 'true',  # Histogram building is not deterministic \n    # due to the non-associative aspect of floating point summation. We employ a \n    # pre-rounding routine to mitigate the issue, which may lead to slightly lower \n    # accuracy\n    # 'booster' : 'dart'\n    'single_precision_histogram':'true', # single precision to build histograms instead of double precision\n    'sampling_method':'gradient_based', # only supported when tree_method is set to gpu_hist    \n    'predictor': 'gpu_predictor'\n\n}\n\nnum_round = 50\ntrain_matrix = xgboost.DMatrix(data = train[FEATS], label =  train[TARGET])\ntest_matrix = xgboost.DMatrix(data = valid[FEATS])\nxgb = xgboost.train(params, dtrain = train_matrix)\n\npredicts = xgb.predict(test_matrix, ntree_limit=num_round)\nroc = roc_auc_score(valid[TARGET].astype('int32'), predicts)\nprint('ROC for XGBoost model')\nprint(roc)\nplot_importance(xgb)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}