{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Iteration 4\n\n1) Used Raddar's dataset https://www.kaggle.com/datasets/raddar/amex-data-integer-dtypes-parquet-format\n\n2) Based on the work by Chris https://www.kaggle.com/code/cdeotte/xgboost-starter-0-793\n\n3) Chris' work highlights the importance of working with aggregated data, partitioning large datasets when predicting and the use of the cudf library which drastically improves performance\n\n\n","metadata":{}},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2022-09-06T20:27:17.790458Z","iopub.execute_input":"2022-09-06T20:27:17.791171Z","iopub.status.idle":"2022-09-06T20:27:17.804933Z","shell.execute_reply.started":"2022-09-06T20:27:17.791052Z","shell.execute_reply":"2022-09-06T20:27:17.803964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LOAD LIBRARIES\nimport pandas as pd, numpy as np # CPU libraries\nimport cudf # GPU libraries\nimport gc\n","metadata":{"execution":{"iopub.status.busy":"2022-09-06T20:27:17.806371Z","iopub.execute_input":"2022-09-06T20:27:17.806847Z","iopub.status.idle":"2022-09-06T20:27:19.200778Z","shell.execute_reply.started":"2022-09-06T20:27:17.806811Z","shell.execute_reply":"2022-09-06T20:27:19.199780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\npartTrain = cudf.read_parquet('../input/amex-data-integer-dtypes-parquet-format/train.parquet')\npartTrain = partTrain.iloc[0:5531451, :]\n#partTrain = partTrain.iloc[0:5000:2, :]\n\nprint(partTrain.shape)","metadata":{"execution":{"iopub.status.busy":"2022-09-06T20:27:19.202062Z","iopub.execute_input":"2022-09-06T20:27:19.202453Z","iopub.status.idle":"2022-09-06T20:27:21.827276Z","shell.execute_reply.started":"2022-09-06T20:27:19.202415Z","shell.execute_reply":"2022-09-06T20:27:21.826115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#List non-numeric columns\nallCols = partTrain.columns.tolist()\nnumericCols = partTrain.select_dtypes(include=np.number).columns.tolist()\n\nprint(\"all\", len(allCols))\nprint(\"numeric\", len(numericCols))\n\n\nnonNumeric = list(set(allCols) - set(numericCols))\nprint(\"non numeric\", len(nonNumeric))\n# converting pandas \"categorical\" dtype to numeric\n#partTrain[nonNumeric] = partTrain[nonNumeric].apply(pd.to_numeric, errors='coerce')\n\n\ndel allCols\ngc.collect()\n\n# remove target and customer id\nwhile True:\n    try:\n        numericCols.remove(\"target\")\n    except ValueError:\n        break\n        \n        \nwhile True:\n    try:\n        nonNumeric.remove(\"customer_ID\")\n    except ValueError:\n        break\n\n        \n# remove target and customer id\nwhile True:\n    try:\n        nonNumeric.remove(\"target\")\n    except ValueError:\n        break","metadata":{"execution":{"iopub.status.busy":"2022-09-06T20:27:21.830810Z","iopub.execute_input":"2022-09-06T20:27:21.831170Z","iopub.status.idle":"2022-09-06T20:27:21.984026Z","shell.execute_reply.started":"2022-09-06T20:27:21.831125Z","shell.execute_reply":"2022-09-06T20:27:21.982570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# feature engineering \n# https://www.kaggle.com/code/cdeotte/xgboost-starter-0-793\n# Add agg columns\n\nprint(partTrain.head(3))\nnumericCols.append(\"customer_ID\")\n\ntest_num_agg = partTrain.groupby(\"customer_ID\")[numericCols].agg(['mean', 'std', 'min', 'max', 'last'])\ntest_num_agg.columns = ['_'.join(x) for x in test_num_agg.columns]\nprint(test_num_agg.head())\n\nnonNumeric.append(\"customer_ID\")\ntest_cat_agg = partTrain.groupby(\"customer_ID\")[nonNumeric].agg(['count', 'nunique'])\ntest_cat_agg.columns = ['_'.join(x) for x in test_cat_agg.columns]\nprint(test_cat_agg.head())\n","metadata":{"execution":{"iopub.status.busy":"2022-09-06T20:27:21.986309Z","iopub.execute_input":"2022-09-06T20:27:21.986688Z","iopub.status.idle":"2022-09-06T20:27:23.743558Z","shell.execute_reply.started":"2022-09-06T20:27:21.986652Z","shell.execute_reply":"2022-09-06T20:27:23.742317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_num_agg.reset_index(inplace=True)\ntest_cat_agg.reset_index(inplace=True)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-09-06T20:27:23.745131Z","iopub.execute_input":"2022-09-06T20:27:23.746134Z","iopub.status.idle":"2022-09-06T20:27:23.753740Z","shell.execute_reply.started":"2022-09-06T20:27:23.746100Z","shell.execute_reply":"2022-09-06T20:27:23.752504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# target variable\ntargets = cudf.read_csv('../input/amex-default-prediction/train_labels.csv')\n#print(targets.head())\n#print(targets.shape)\n\ntest_num_agg = test_num_agg.join(targets, how='left', lsuffix='_left', rsuffix='_right')\ntest_num_agg = test_num_agg.drop_duplicates(['customer_ID_right'])\n\nprint(test_num_agg.sample(3))\ntest_num_agg.reset_index(inplace=True)\nprint(test_num_agg.sample(3))\n","metadata":{"execution":{"iopub.status.busy":"2022-09-06T20:27:23.755761Z","iopub.execute_input":"2022-09-06T20:27:23.756245Z","iopub.status.idle":"2022-09-06T20:27:24.997760Z","shell.execute_reply.started":"2022-09-06T20:27:23.756202Z","shell.execute_reply":"2022-09-06T20:27:24.996571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nX = test_num_agg.drop(columns=['customer_ID_right', 'customer_ID_left',  'target', 'index'],axis=1)\ny = test_num_agg[['customer_ID_right', 'target']]\ny = y.drop(columns=[\"customer_ID_right\"],axis=1)\n\nprint(X.shape)\nprint(y.shape)\nrows = int(np.round(X.shape[0]*.7))\nprint(rows)\n\nimport xgboost as xgb\nX_train = X.iloc[0:rows, :]\nX_test = X.iloc[rows:, :]\ny_train = y.iloc[0:rows, :]\ny_test = y.iloc[rows:, :]\nprint(X_train.shape,y_train.shape )\n\nprint(X_train.head(3))\n\nd_train = xgb.DMatrix(data=X_train, label=y_train)\nd_test = xgb.DMatrix(data=X_test, label=y_test)","metadata":{"execution":{"iopub.status.busy":"2022-09-06T20:27:24.999700Z","iopub.execute_input":"2022-09-06T20:27:25.000115Z","iopub.status.idle":"2022-09-06T20:27:28.160874Z","shell.execute_reply.started":"2022-09-06T20:27:25.000073Z","shell.execute_reply":"2022-09-06T20:27:28.159865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {  \"objective\":\"binary:logistic\", \n            'max_depth':4, \n            'learning_rate':0.05, \n            'subsample':0.8,\n            'colsample_bytree':0.6, \n            'tree_method':'gpu_hist',\n            'predictor':'gpu_predictor',\n            \"eval_metric\":'aucpr', # updated to make use of the aucpr option\n}\n\nprint(\"===================== TRAIN==========================================\")\nxgb_clf = xgb.train(\n    params,\n    d_train,\n    num_boost_round=10,\n    evals=[(d_train, 'train'),(d_test, 'test')],\n    early_stopping_rounds=10,\n    verbose_eval=0\n)\n\n\nprint(\"===================== PREDICT TRAIN==========================================\")\ny_pred = xgb_clf.predict(d_test)\nprint(y_pred)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-06T20:27:28.166255Z","iopub.execute_input":"2022-09-06T20:27:28.167092Z","iopub.status.idle":"2022-09-06T20:27:29.314780Z","shell.execute_reply.started":"2022-09-06T20:27:28.166918Z","shell.execute_reply":"2022-09-06T20:27:29.313693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(xgb_clf)","metadata":{"execution":{"iopub.status.busy":"2022-09-06T20:27:29.316234Z","iopub.execute_input":"2022-09-06T20:27:29.316890Z","iopub.status.idle":"2022-09-06T20:27:29.322365Z","shell.execute_reply.started":"2022-09-06T20:27:29.316851Z","shell.execute_reply":"2022-09-06T20:27:29.321340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# delete vars that are not needed anymore\n\ndel y_pred, X_train, y_train, X_test, y_test, d_train, d_test, test_num_agg, partTrain\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-09-06T20:27:29.323821Z","iopub.execute_input":"2022-09-06T20:27:29.324620Z","iopub.status.idle":"2022-09-06T20:27:29.730694Z","shell.execute_reply.started":"2022-09-06T20:27:29.324567Z","shell.execute_reply":"2022-09-06T20:27:29.729648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(xgb_clf)","metadata":{"execution":{"iopub.status.busy":"2022-09-06T20:27:29.732331Z","iopub.execute_input":"2022-09-06T20:27:29.732679Z","iopub.status.idle":"2022-09-06T20:27:29.740134Z","shell.execute_reply.started":"2022-09-06T20:27:29.732643Z","shell.execute_reply":"2022-09-06T20:27:29.738986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# generate output\nprint(\"===================== READ TEST FILE ==========================================\")\n\npartTest = cudf.read_parquet('../input/amex-data-integer-dtypes-parquet-format/test.parquet')\npartTest = partTest.sort_values('customer_ID')\n\nprint(\"===================== CREATE CHUNKS ON THE TEST FILE ==========================================\")\nrows_1 = int(np.round(partTest.shape[0]*.33))\nrows_2 = int(np.round(partTest.shape[0]*.66))\n\npartTest_1 = partTest.iloc[0:rows_1, :]\npartTest_2 = partTest.iloc[rows_1:rows_2, :]\npartTest_3 = partTest.iloc[rows_2:, :]\n\ndel partTest\ngc.collect()\n\nprint(partTest_1.shape)\nprint(partTest_2.shape)\nprint(partTest_3.shape)\n\n\n\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-09-06T20:27:29.745274Z","iopub.execute_input":"2022-09-06T20:27:29.745855Z","iopub.status.idle":"2022-09-06T20:27:36.170697Z","shell.execute_reply.started":"2022-09-06T20:27:29.745800Z","shell.execute_reply":"2022-09-06T20:27:36.169491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"===================== AGG THE TEST FILE 1 ==========================================\")\n\ntest_num_agg_1 = partTest_1.groupby(\"customer_ID\")[numericCols].agg(['mean', 'std', 'min', 'max', 'last'])\ndel partTest_1\ngc.collect()\n\ntest_num_agg_1.columns = ['_'.join(x) for x in test_num_agg_1.columns]\ntest_num_agg_1.reset_index(inplace=True)\nprint(test_num_agg_1.head())\n\nprint(test_num_agg_1.shape)\n\nC_1 = test_num_agg_1[['customer_ID']]\nX_1 = test_num_agg_1.drop(columns=['customer_ID'],axis=1)\n\ndel test_num_agg_1\ngc.collect()\n\n\nd_test_1 = xgb.DMatrix(data=X_1)\ndel X_1\ngc.collect()\n\nprint(\"===================== TRAIN FINAL==========================================\")\ny_pred_prob_1 = xgb_clf.predict(d_test_1)\nprint(y_pred_prob_1.shape)\n\ndel d_test_1\ngc.collect()\n","metadata":{"execution":{"iopub.status.busy":"2022-09-06T20:27:36.172419Z","iopub.execute_input":"2022-09-06T20:27:36.172804Z","iopub.status.idle":"2022-09-06T20:27:41.221253Z","shell.execute_reply.started":"2022-09-06T20:27:36.172767Z","shell.execute_reply":"2022-09-06T20:27:41.220208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"===================== AGG THE TEST FILE 1 ==========================================\")\n\ntest_num_agg_2 = partTest_2.groupby(\"customer_ID\")[numericCols].agg(['mean', 'std', 'min', 'max', 'last'])\ndel partTest_2\ngc.collect()\n\ntest_num_agg_2.columns = ['_'.join(x) for x in test_num_agg_2.columns]\ntest_num_agg_2.reset_index(inplace=True)\nprint(test_num_agg_2.head())\n\nprint(test_num_agg_2.shape)\n\n\nC_2 = test_num_agg_2[['customer_ID']]\nX_2 = test_num_agg_2.drop(columns=['customer_ID'],axis=1)\n\ndel test_num_agg_2\ngc.collect()\n\n\nd_test_2 = xgb.DMatrix(data=X_2)\ndel X_2\ngc.collect()\n\nprint(\"===================== TRAIN FINAL 2==========================================\")\ny_pred_prob_2 = xgb_clf.predict(d_test_2)\nprint(y_pred_prob_2.shape)\n\ndel d_test_2\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-09-06T20:27:41.222802Z","iopub.execute_input":"2022-09-06T20:27:41.223701Z","iopub.status.idle":"2022-09-06T20:27:46.172596Z","shell.execute_reply.started":"2022-09-06T20:27:41.223660Z","shell.execute_reply":"2022-09-06T20:27:46.171572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"===================== AGG THE TEST FILE 3 ==========================================\")\n\ntest_num_agg_3 = partTest_3.groupby(\"customer_ID\")[numericCols].agg(['mean', 'std', 'min', 'max', 'last'])\ndel partTest_3\ngc.collect()\n\n\ntest_num_agg_3.columns = ['_'.join(x) for x in test_num_agg_3.columns]\ntest_num_agg_3.reset_index(inplace=True)\nprint(test_num_agg_3.head())\n\nprint(test_num_agg_3.shape)\n\n\nC_3 = test_num_agg_3[['customer_ID']]\nX_3 = test_num_agg_3.drop(columns=['customer_ID'],axis=1)\n\ndel test_num_agg_3\ngc.collect()\n\n\nd_test_3 = xgb.DMatrix(data=X_3)\ndel X_3\ngc.collect()\n\nprint(\"===================== TRAIN FINAL 3==========================================\")\ny_pred_prob_3 = xgb_clf.predict(d_test_3)\nprint(y_pred_prob_3.shape)\n\ndel d_test_3\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-09-06T20:27:46.174292Z","iopub.execute_input":"2022-09-06T20:27:46.174678Z","iopub.status.idle":"2022-09-06T20:27:51.410317Z","shell.execute_reply.started":"2022-09-06T20:27:46.174638Z","shell.execute_reply":"2022-09-06T20:27:51.409294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del xgb_clf\ngc.collect()\n","metadata":{"execution":{"iopub.status.busy":"2022-09-06T20:27:51.411734Z","iopub.execute_input":"2022-09-06T20:27:51.412606Z","iopub.status.idle":"2022-09-06T20:27:51.533092Z","shell.execute_reply.started":"2022-09-06T20:27:51.412567Z","shell.execute_reply":"2022-09-06T20:27:51.531917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"C_1[\"pred\"] = y_pred_prob_1\nC_2[\"pred\"] = y_pred_prob_2\nC_3[\"pred\"] = y_pred_prob_3\n\ndel y_pred_prob_1, y_pred_prob_2, y_pred_prob_3\ngc.collect()\n\n\nprint(C_1.shape)\nprint(C_2.shape)\nprint(C_3.shape)\n\nfinal_C = C_1.merge(C_2, on='customer_ID', how='outer', suffixes=('left', 'right'))\nfinal_C = final_C.merge(C_3, on='customer_ID', how='outer', suffixes=('left', 'right'))\nprint(final_C.shape)\n\nprint(final_C.head)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-06T20:27:51.534544Z","iopub.execute_input":"2022-09-06T20:27:51.535030Z","iopub.status.idle":"2022-09-06T20:27:51.720162Z","shell.execute_reply.started":"2022-09-06T20:27:51.534993Z","shell.execute_reply":"2022-09-06T20:27:51.719113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_C.fillna(0, inplace=True)\nfinal_C['final_output'] = final_C['predleft'] + final_C['predright'] + final_C['pred']\nfinal_C = final_C[['customer_ID', 'final_output' ]]\nprint(final_C.head)","metadata":{"execution":{"iopub.status.busy":"2022-09-06T20:27:51.721588Z","iopub.execute_input":"2022-09-06T20:27:51.722183Z","iopub.status.idle":"2022-09-06T20:27:51.756351Z","shell.execute_reply.started":"2022-09-06T20:27:51.722116Z","shell.execute_reply":"2022-09-06T20:27:51.755324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##Create Submission\n#print(\"===================== CREATE SUBMISSION FILE ==========================================\")\n\nsubmission = cudf.read_csv('../input/amex-default-prediction/sample_submission.csv')\nsubmission = submission.drop_duplicates(['customer_ID'])\nprint(submission.shape)\nsubmission = submission.merge(final_C, on='customer_ID', how='left', suffixes=('left', 'right'))\nprint(submission.shape)\nsubmission = submission[['customer_ID', \"final_output\"]]\nsubmission.rename(columns={\"final_output\": \"prediction\"}, inplace=True)\nprint(submission.head, submission.shape)\nsubmission.to_csv('submission.csv',index=False)\nprint('Submission file shape is', submission.shape )","metadata":{"execution":{"iopub.status.busy":"2022-09-06T20:27:51.757799Z","iopub.execute_input":"2022-09-06T20:27:51.758164Z","iopub.status.idle":"2022-09-06T20:27:52.144142Z","shell.execute_reply.started":"2022-09-06T20:27:51.758113Z","shell.execute_reply":"2022-09-06T20:27:52.143118Z"},"trusted":true},"execution_count":null,"outputs":[]}]}