{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport cupy, cudf\nimport gc,os\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import KFold\nfrom xgboost import XGBClassifier","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-08T09:09:16.665674Z","iopub.execute_input":"2022-06-08T09:09:16.666497Z","iopub.status.idle":"2022-06-08T09:09:18.415908Z","shell.execute_reply.started":"2022-06-08T09:09:16.666387Z","shell.execute_reply":"2022-06-08T09:09:18.414993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = cudf.read_parquet(\"../input/amex-data-integer-dtypes-parquet-format/train.parquet\")\ntrain_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-08T08:41:30.794398Z","iopub.execute_input":"2022-06-08T08:41:30.795003Z","iopub.status.idle":"2022-06-08T08:41:52.814194Z","shell.execute_reply.started":"2022-06-08T08:41:30.794962Z","shell.execute_reply":"2022-06-08T08:41:52.813288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-08T08:41:52.815721Z","iopub.execute_input":"2022-06-08T08:41:52.816077Z","iopub.status.idle":"2022-06-08T08:41:52.825169Z","shell.execute_reply.started":"2022-06-08T08:41:52.816040Z","shell.execute_reply":"2022-06-08T08:41:52.823504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_and_feature_engineer(df):\n    # FEATURE ENGINEERING FROM \n    # https://www.kaggle.com/code/huseyincot/amex-agg-data-how-it-created\n    df[\"S_2\"] = cudf.to_datetime(df[\"S_2\"])\n    df[\"S_2_month\"] = df[\"S_2\"].dt.month\n    df[\"S_2_year\"] = df[\"S_2\"].dt.year\n    df[\"S_2_day\"] = df[\"S_2\"].dt.day  \n    \n    all_cols = [c for c in list(df.columns) if c not in ['customer_ID','S_2']]\n    cat_features = [\"B_30\",\"B_38\",\"D_114\",\"D_116\",\"D_117\",\"D_120\",\"D_126\",\"D_63\",\"D_64\",\"D_66\",\"D_68\"]\n    num_features = [col for col in all_cols if col not in cat_features]\n\n    test_num_agg = df.groupby(\"customer_ID\")[num_features].agg(['mean', 'std', 'min', 'max', 'last'])\n    test_num_agg.columns = ['_'.join(x) for x in test_num_agg.columns]\n\n    test_cat_agg = df.groupby(\"customer_ID\")[cat_features].agg(['count', 'last', 'nunique'])\n    test_cat_agg.columns = ['_'.join(x) for x in test_cat_agg.columns]\n\n    df = cudf.concat([test_num_agg, test_cat_agg], axis=1)\n    del test_num_agg, test_cat_agg\n    print('shape after engineering', df.shape )\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-06-08T09:09:23.936688Z","iopub.execute_input":"2022-06-08T09:09:23.937689Z","iopub.status.idle":"2022-06-08T09:09:23.947575Z","shell.execute_reply.started":"2022-06-08T09:09:23.937641Z","shell.execute_reply":"2022-06-08T09:09:23.946834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = process_and_feature_engineer(train_data)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T08:41:52.841745Z","iopub.execute_input":"2022-06-08T08:41:52.842231Z","iopub.status.idle":"2022-06-08T08:41:57.173162Z","shell.execute_reply.started":"2022-06-08T08:41:52.842169Z","shell.execute_reply":"2022-06-08T08:41:57.172328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = cudf.read_csv(\"../input/amex-default-prediction/train_labels.csv\")\ntargets.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-08T08:41:57.174319Z","iopub.execute_input":"2022-06-08T08:41:57.175373Z","iopub.status.idle":"2022-06-08T08:41:57.628050Z","shell.execute_reply.started":"2022-06-08T08:41:57.175334Z","shell.execute_reply":"2022-06-08T08:41:57.627335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-08T08:41:57.629394Z","iopub.execute_input":"2022-06-08T08:41:57.629742Z","iopub.status.idle":"2022-06-08T08:41:57.635244Z","shell.execute_reply.started":"2022-06-08T08:41:57.629707Z","shell.execute_reply":"2022-06-08T08:41:57.634328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.fillna(-999,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T08:41:57.636932Z","iopub.execute_input":"2022-06-08T08:41:57.637617Z","iopub.status.idle":"2022-06-08T08:41:58.599265Z","shell.execute_reply.started":"2022-06-08T08:41:57.637575Z","shell.execute_reply":"2022-06-08T08:41:58.598277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.reset_index(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T08:41:58.601869Z","iopub.execute_input":"2022-06-08T08:41:58.602257Z","iopub.status.idle":"2022-06-08T08:41:58.610044Z","shell.execute_reply.started":"2022-06-08T08:41:58.602220Z","shell.execute_reply":"2022-06-08T08:41:58.609157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = cudf.merge(train_data,targets,on=\"custmer_ID\",left_index=True,right_index=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T08:41:58.615115Z","iopub.execute_input":"2022-06-08T08:41:58.615489Z","iopub.status.idle":"2022-06-08T08:41:58.976741Z","shell.execute_reply.started":"2022-06-08T08:41:58.615465Z","shell.execute_reply":"2022-06-08T08:41:58.975774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.drop([\"customer_ID_x\",\"customer_ID_y\"],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T08:41:58.978377Z","iopub.execute_input":"2022-06-08T08:41:58.978743Z","iopub.status.idle":"2022-06-08T08:41:58.988153Z","shell.execute_reply.started":"2022-06-08T08:41:58.978706Z","shell.execute_reply":"2022-06-08T08:41:58.987181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-08T08:41:58.989757Z","iopub.execute_input":"2022-06-08T08:41:58.990287Z","iopub.status.idle":"2022-06-08T08:41:59.808971Z","shell.execute_reply.started":"2022-06-08T08:41:58.990234Z","shell.execute_reply":"2022-06-08T08:41:59.808072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def amex_metric_mod(y_true, y_pred):\n\n    labels     = np.transpose(np.array([y_true, y_pred]))\n    labels     = labels[labels[:, 1].argsort()[::-1]]\n    weights    = np.where(labels[:,0]==0, 20, 1)\n    cut_vals   = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four   = np.sum(cut_vals[:,0]) / np.sum(labels[:,0])\n\n    gini = [0,0]\n    for i in [1,0]:\n        labels         = np.transpose(np.array([y_true, y_pred]))\n        labels         = labels[labels[:, i].argsort()[::-1]]\n        weight         = np.where(labels[:,0]==0, 20, 1)\n        weight_random  = np.cumsum(weight / np.sum(weight))\n        total_pos      = np.sum(labels[:, 0] *  weight)\n        cum_pos_found  = np.cumsum(labels[:, 0] * weight)\n        lorentz        = cum_pos_found / total_pos\n        gini[i]        = np.sum((lorentz - weight_random) * weight)\n\n    return 0.5 * (gini[1]/gini[0] + top_four)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T08:41:59.810491Z","iopub.execute_input":"2022-06-08T08:41:59.810869Z","iopub.status.idle":"2022-06-08T08:41:59.819894Z","shell.execute_reply.started":"2022-06-08T08:41:59.810834Z","shell.execute_reply":"2022-06-08T08:41:59.819052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"useful_features = [feature for feature in train_data.columns if feature != \"target\"]","metadata":{"execution":{"iopub.status.busy":"2022-06-08T08:41:59.821104Z","iopub.execute_input":"2022-06-08T08:41:59.821624Z","iopub.status.idle":"2022-06-08T08:41:59.832495Z","shell.execute_reply.started":"2022-06-08T08:41:59.821586Z","shell.execute_reply":"2022-06-08T08:41:59.831710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kfold = KFold(n_splits=5,shuffle=True,random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T08:41:59.834035Z","iopub.execute_input":"2022-06-08T08:41:59.834540Z","iopub.status.idle":"2022-06-08T08:41:59.840829Z","shell.execute_reply.started":"2022-06-08T08:41:59.834505Z","shell.execute_reply":"2022-06-08T08:41:59.839860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"XGBOOST_PARAMS = {'learning_rate': 0.0170074900458309, \n'reg_lambda': 5.9599291346341776e-05, \n'reg_alpha': 0.015370240971015697,\n 'subsample': 0.7027704916274289, \n'colsample_bytree': 0.5705441270528481,\n 'max_depth': 4, \n'n_estimators': 6542}","metadata":{"execution":{"iopub.status.busy":"2022-06-08T08:41:59.842278Z","iopub.execute_input":"2022-06-08T08:41:59.842692Z","iopub.status.idle":"2022-06-08T08:41:59.848670Z","shell.execute_reply.started":"2022-06-08T08:41:59.842653Z","shell.execute_reply":"2022-06-08T08:41:59.847712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = train_data.to_pandas()\nfor fold,(train_idx,valid_idx) in enumerate(kfold.split(train_data, train_data.target)):\n    \n    X_train = train_data.iloc[train_idx][useful_features]\n    X_valid = train_data.iloc[valid_idx][useful_features]\n    y_train = train_data.iloc[train_idx][\"target\"]\n    y_valid = train_data.iloc[valid_idx][\"target\"]\n    \n    model = XGBClassifier( \n        random_state=fold,\n        objective='binary:logistic',\n        tree_method='gpu_hist',  \n        gpu_id=0,\n        predictor='gpu_predictor',\n        n_jobs = -1,\n        **XGBOOST_PARAMS\n    )\n    \n    model.fit(X_train, y_train,\n              eval_set=[(X_valid, y_valid)],\n              verbose=0)\n    \n    preds_valid = model.predict_proba(X_valid)[:,1]\n        \n    print(f\"the kaggle metric score after {fold} fold is : {amex_metric_mod(y_valid, preds_valid)}\")    \n    \n    model.save_model(f'XGB_fold{fold}.xgb')\n    \n    print(f\"Model saved for {fold} fold\")\n    \n    del X_train, X_valid, y_train, y_valid, model\n    \n    _ =  gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-08T08:41:59.850173Z","iopub.execute_input":"2022-06-08T08:41:59.850854Z","iopub.status.idle":"2022-06-08T09:04:45.544081Z","shell.execute_reply.started":"2022-06-08T08:41:59.850816Z","shell.execute_reply":"2022-06-08T09:04:45.543139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_data","metadata":{"execution":{"iopub.status.busy":"2022-06-08T09:07:16.620678Z","iopub.execute_input":"2022-06-08T09:07:16.621353Z","iopub.status.idle":"2022-06-08T09:07:16.633359Z","shell.execute_reply.started":"2022-06-08T09:07:16.621315Z","shell.execute_reply":"2022-06-08T09:07:16.632329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from numba import cuda\ncuda.select_device(0)\ncuda.close()\ncuda.select_device(0)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T09:07:25.423493Z","iopub.execute_input":"2022-06-08T09:07:25.423851Z","iopub.status.idle":"2022-06-08T09:07:25.956733Z","shell.execute_reply.started":"2022-06-08T09:07:25.423820Z","shell.execute_reply":"2022-06-08T09:07:25.955990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_preprocess_test():\n    \n    df = cudf.read_parquet(\"../input/amex-data-integer-dtypes-parquet-format/test.parquet\")\n    \n    df = process_and_feature_engineer(df)\n    \n    df.fillna(-999,inplace=True)\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-06-08T09:09:39.330065Z","iopub.execute_input":"2022-06-08T09:09:39.330877Z","iopub.status.idle":"2022-06-08T09:09:39.335934Z","shell.execute_reply.started":"2022-06-08T09:09:39.330842Z","shell.execute_reply":"2022-06-08T09:09:39.334898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = load_preprocess_test()","metadata":{"execution":{"iopub.status.busy":"2022-06-08T09:09:39.588990Z","iopub.execute_input":"2022-06-08T09:09:39.589520Z","iopub.status.idle":"2022-06-08T09:09:52.664509Z","shell.execute_reply.started":"2022-06-08T09:09:39.589485Z","shell.execute_reply":"2022-06-08T09:09:52.663646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.reset_index(inplace=True)\ntest_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-08T09:10:10.457670Z","iopub.execute_input":"2022-06-08T09:10:10.458057Z","iopub.status.idle":"2022-06-08T09:10:11.215100Z","shell.execute_reply.started":"2022-06-08T09:10:10.458026Z","shell.execute_reply":"2022-06-08T09:10:11.214189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer_id = test_data[\"customer_ID\"]","metadata":{"execution":{"iopub.status.busy":"2022-06-08T09:10:11.216812Z","iopub.execute_input":"2022-06-08T09:10:11.217308Z","iopub.status.idle":"2022-06-08T09:10:11.222372Z","shell.execute_reply.started":"2022-06-08T09:10:11.217269Z","shell.execute_reply":"2022-06-08T09:10:11.221161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.drop(\"customer_ID\",axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T09:10:12.589776Z","iopub.execute_input":"2022-06-08T09:10:12.590161Z","iopub.status.idle":"2022-06-08T09:10:12.594890Z","shell.execute_reply.started":"2022-06-08T09:10:12.590110Z","shell.execute_reply":"2022-06-08T09:10:12.593997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-08T09:10:13.652700Z","iopub.execute_input":"2022-06-08T09:10:13.653506Z","iopub.status.idle":"2022-06-08T09:10:14.387821Z","shell.execute_reply.started":"2022-06-08T09:10:13.653470Z","shell.execute_reply":"2022-06-08T09:10:14.387014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_predictions = []\nfor fold in range(5):\n    \n    model = XGBClassifier(random_state=fold,\n                        objective='binary:logistic',\n                        tree_method='gpu_hist',  \n                        gpu_id=0,\n                        predictor='gpu_predictor',\n                        n_jobs = -1)\n    \n    model.load_model(f'XGB_fold{fold}.xgb')\n    \n    test_preds = model.predict_proba(test_data)[:,1]\n    \n    print(f\"{fold} fold is completed and prediction is appended into final prediction list\")\n    \n    del model\n    \n    final_predictions.append(test_preds)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T09:10:17.614448Z","iopub.execute_input":"2022-06-08T09:10:17.614814Z","iopub.status.idle":"2022-06-08T09:18:02.464387Z","shell.execute_reply.started":"2022-06-08T09:10:17.614783Z","shell.execute_reply":"2022-06-08T09:18:02.463453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_predictions","metadata":{"execution":{"iopub.status.busy":"2022-06-08T09:18:12.109289Z","iopub.execute_input":"2022-06-08T09:18:12.109866Z","iopub.status.idle":"2022-06-08T09:18:12.117957Z","shell.execute_reply.started":"2022-06-08T09:18:12.109832Z","shell.execute_reply":"2022-06-08T09:18:12.116715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = np.mean(np.column_stack(final_predictions), axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T09:18:16.866151Z","iopub.execute_input":"2022-06-08T09:18:16.866522Z","iopub.status.idle":"2022-06-08T09:18:16.903805Z","shell.execute_reply.started":"2022-06-08T09:18:16.866493Z","shell.execute_reply":"2022-06-08T09:18:16.902787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target","metadata":{"execution":{"iopub.status.busy":"2022-06-08T09:18:17.175623Z","iopub.execute_input":"2022-06-08T09:18:17.176445Z","iopub.status.idle":"2022-06-08T09:18:17.186905Z","shell.execute_reply.started":"2022-06-08T09:18:17.176402Z","shell.execute_reply":"2022-06-08T09:18:17.185767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_data = pd.DataFrame({\"customer_ID\":customer_id.to_pandas(),\"prediction\":target})\noutput_data","metadata":{"execution":{"iopub.status.busy":"2022-06-08T09:18:21.178776Z","iopub.execute_input":"2022-06-08T09:18:21.179163Z","iopub.status.idle":"2022-06-08T09:18:21.547923Z","shell.execute_reply.started":"2022-06-08T09:18:21.179111Z","shell.execute_reply":"2022-06-08T09:18:21.547038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_data.to_csv(\"third_submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T09:18:29.062054Z","iopub.execute_input":"2022-06-08T09:18:29.062443Z","iopub.status.idle":"2022-06-08T09:18:33.579867Z","shell.execute_reply.started":"2022-06-08T09:18:29.062405Z","shell.execute_reply":"2022-06-08T09:18:33.579056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}