{"cells":[{"metadata":{},"cell_type":"markdown","source":"XGBoost has reigned the predictions on tabular data for quite sometime and the ML/ AI community is itching to get past the baseline it sets. Recently, have been introduced to TabNet architecture. In this Notebook, i do a quick & plain vanilla comparison between the 2 algorithms on 5M sample. I have barely made much changes to default parameters and here, i just intend to see if both have comparable results and if eventually an ensemble could result into a better result."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom sklearn.metrics import roc_auc_score, accuracy_score, f1_score,roc_curve,classification_report\n\nimport xgboost as xgb\nfrom xgboost import plot_importance\nfrom xgboost.sklearn import XGBClassifier\nimport riiideducation\n\nimport torch\nfrom pytorch_tabnet.tab_model import TabNetClassifier\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pip install pytorch_tabnet","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',\n                   usecols=[1, 2, 3, 4, 5, 7, 8, 9],\n                   dtype={'timestamp': 'int64',\n                          'user_id': 'int32',\n                          'content_id': 'int16',\n                          'content_type_id': 'int8',\n                          'task_container_id': 'int16',\n                          'answered_correctly':'int8',\n                          'prior_question_elapsed_time': 'float32',\n                          'prior_question_had_explanation': 'boolean'}\n                   )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Remove lectures and additional processing\ntrain = train[train.content_type_id == False]\n\ntrain = train.sort_values(['timestamp'], ascending=True)\ntrain.drop(['timestamp','content_type_id'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Read Questions and Lectures\nquestions = pd.read_csv('../input/riiid-test-answer-prediction/questions.csv')\nlectures = pd.read_csv('../input/riiid-test-answer-prediction/lectures.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Merge train with Questions\ntrain = pd.merge(train, questions, left_on = 'content_id', right_on = 'question_id', how = 'left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Indicator for first question in a batch\ntrain['firstQindicator'] = np.where(train['prior_question_elapsed_time'].isnull(),1,0)\ntrain['prior_question_elapsed_time'] = np.where(train['prior_question_elapsed_time'].isnull(),\n                                                0,train['prior_question_elapsed_time'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Remove unused columns\ndel train['question_id']\ndel train['bundle_id']\ndel train['correct_answer']\ndel train['tags']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.prior_question_had_explanation = np.where(train.prior_question_had_explanation=='True',1,0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Sample 5M records\ntrain = train.sample(n=5000000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train test split\nxtrain, xvalid, ytrain, yvalid = train_test_split(train.drop(['answered_correctly'],axis=1), \n                                                  train['answered_correctly'],\n                                                  random_state=42, \n                                                  test_size=0.2, \n                                                  shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train XGB Classifier\nclf_xgb = xgb.XGBClassifier()\nclf_xgb.fit(xtrain, ytrain)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Predict using XGB Classifier\npred_xgb = clf_xgb.predict(xvalid)\nprint('\\t\\t\\tCLASSIFICATIION METRICS: XGBOOST\\n')\nprint(metrics.classification_report(yvalid, pred_xgb))\nscore = roc_auc_score(yvalid, pred_xgb)\nprint('ROC value is: {}'.format(score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Tabnet object\nclf_tabnet = TabNetClassifier()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Fit TabNet model\nclf_tabnet.fit(\n    X_train=xtrain.values, y_train=ytrain.values,\n    eval_set=[(xvalid.values, yvalid.values)]\n    \n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Predict using TabNet\npred_tabnet = clf_tabnet.predict(xvalid.values)\nprint('\\t\\t\\tCLASSIFICATIION METRICS: TabNet\\n')\nprint(metrics.classification_report(yvalid, pred_tabnet))\nscore = roc_auc_score(yvalid, pred_tabnet)\nprint('ROC value is: {}'.format(score))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I think XGBoost still has an edge. As the model is done on partial data, only minimal feature engineering is done, and no hyper-parameter tuning is done, we see the results much less than what the baselines from other Kernels are providing. As the next step, my focus is going to be along the lines and see how much each algo can stretch to. **Stay tuned...**"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}