{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# import libraries\nimport numpy as np\nimport pandas as pd\nimport cupy, cudf # GPU libraries\nimport matplotlib.pyplot as plt\nimport gc","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-28T04:31:06.994723Z","iopub.execute_input":"2022-07-28T04:31:06.995483Z","iopub.status.idle":"2022-07-28T04:31:08.410836Z","shell.execute_reply.started":"2022-07-28T04:31:06.995367Z","shell.execute_reply":"2022-07-28T04:31:08.409795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load training data\ntrain = cudf.read_parquet(\"../input/amex-data-integer-dtypes-parquet-format/train.parquet\")\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-28T04:31:08.412842Z","iopub.execute_input":"2022-07-28T04:31:08.413172Z","iopub.status.idle":"2022-07-28T04:31:30.858697Z","shell.execute_reply.started":"2022-07-28T04:31:08.413145Z","shell.execute_reply":"2022-07-28T04:31:30.857644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# REDUCE DTYPE FOR CUSTOMER AND DATE\ntrain['customer_ID'] = train['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\ntrain.S_2 = cudf.to_datetime( train.S_2 )","metadata":{"execution":{"iopub.status.busy":"2022-07-28T04:31:30.860160Z","iopub.execute_input":"2022-07-28T04:31:30.860722Z","iopub.status.idle":"2022-07-28T04:31:30.959368Z","shell.execute_reply.started":"2022-07-28T04:31:30.860682Z","shell.execute_reply":"2022-07-28T04:31:30.958499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check categorical variables\ncat_features = [\"B_30\",\"B_38\",\"D_114\",\"D_116\",\"D_117\",\"D_120\",\"D_126\",\"D_63\",\"D_64\",\"D_66\",\"D_68\"]\n    \nfor feat in cat_features:\n    print(feat,train[feat].unique())","metadata":{"execution":{"iopub.status.busy":"2022-07-28T04:31:30.961938Z","iopub.execute_input":"2022-07-28T04:31:30.962492Z","iopub.status.idle":"2022-07-28T04:31:31.043573Z","shell.execute_reply.started":"2022-07-28T04:31:30.962444Z","shell.execute_reply":"2022-07-28T04:31:31.042667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data preprocessing\ndef preprocessing(df,training_data=False):\n    # keep only the last record for each customer\n    df.drop_duplicates('customer_ID',keep='last',inplace=True)\n    \n    if training_data:\n        labels = cudf.read_csv(\"../input/amex-default-prediction/train_labels.csv\")\n        labels['customer_ID'] = labels['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\n       # merge labels and train datasets to get the labelled data for training\n        df = cudf.merge(df,labels,on=\"customer_ID\",how='left')\n        df.drop(['customer_ID'],1,inplace=True)\n        del labels\n        gc.collect()\n         \n        \n    # convert into dummies\n    dummies = cudf.get_dummies(df[cat_features])\n\n    # drop categorical variables \n    print(\"dropping cat features\")\n    df.drop(cat_features,1,inplace = True)\n    # concat dummy variables with X\n    df = cudf.concat([df, dummies], axis=1)\n\n    df.drop(['S_2'],1,inplace=True)\n    \n    df = df.fillna(-127) \n    \n    del dummies\n    gc.collect()\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-07-28T04:31:31.045731Z","iopub.execute_input":"2022-07-28T04:31:31.046360Z","iopub.status.idle":"2022-07-28T04:31:31.055268Z","shell.execute_reply.started":"2022-07-28T04:31:31.046319Z","shell.execute_reply":"2022-07-28T04:31:31.054351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# perform preprocessing\ntrain = preprocessing(train,training_data=True)\nx_target = train.target.values\ntrain.drop('target',1,inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T04:31:31.056877Z","iopub.execute_input":"2022-07-28T04:31:31.057244Z","iopub.status.idle":"2022-07-28T04:31:32.175583Z","shell.execute_reply.started":"2022-07-28T04:31:31.057207Z","shell.execute_reply":"2022-07-28T04:31:32.174529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.to_pandas() # free GPU memory\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T04:31:32.176919Z","iopub.execute_input":"2022-07-28T04:31:32.177782Z","iopub.status.idle":"2022-07-28T04:31:32.827452Z","shell.execute_reply.started":"2022-07-28T04:31:32.177741Z","shell.execute_reply":"2022-07-28T04:31:32.826378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Improting the PCA module\nfrom sklearn.decomposition import PCA\npca = PCA(svd_solver='randomized', random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T04:31:32.828803Z","iopub.execute_input":"2022-07-28T04:31:32.829817Z","iopub.status.idle":"2022-07-28T04:31:32.835129Z","shell.execute_reply.started":"2022-07-28T04:31:32.829776Z","shell.execute_reply":"2022-07-28T04:31:32.833740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#let's apply PCA\npca.fit(train)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T04:31:32.836632Z","iopub.execute_input":"2022-07-28T04:31:32.837429Z","iopub.status.idle":"2022-07-28T04:31:52.187630Z","shell.execute_reply.started":"2022-07-28T04:31:32.837389Z","shell.execute_reply":"2022-07-28T04:31:52.186447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#List of PCA components.It would be the same as the number of variables\npca.components_","metadata":{"execution":{"iopub.status.busy":"2022-07-28T04:31:52.191980Z","iopub.execute_input":"2022-07-28T04:31:52.192274Z","iopub.status.idle":"2022-07-28T04:31:52.200117Z","shell.execute_reply.started":"2022-07-28T04:31:52.192247Z","shell.execute_reply":"2022-07-28T04:31:52.198884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Plotting the scree plot\n#Making the screeplot - plotting the cumulative variance against the number of components\n%matplotlib inline\nfig = plt.figure(figsize = (12,8))\nplt.plot(np.cumsum(pca.explained_variance_ratio_))\nplt.xlabel('number of components')\nplt.ylabel('cumulative explained variance')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T04:31:52.201570Z","iopub.execute_input":"2022-07-28T04:31:52.202181Z","iopub.status.idle":"2022-07-28T04:31:52.415501Z","shell.execute_reply.started":"2022-07-28T04:31:52.202143Z","shell.execute_reply":"2022-07-28T04:31:52.414550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Using incremental PCA for efficiency - saves a lot of time on larger datasets\nfrom sklearn.decomposition import IncrementalPCA\npca_final = IncrementalPCA(n_components=30)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T04:31:52.417174Z","iopub.execute_input":"2022-07-28T04:31:52.417507Z","iopub.status.idle":"2022-07-28T04:31:52.424196Z","shell.execute_reply.started":"2022-07-28T04:31:52.417471Z","shell.execute_reply":"2022-07-28T04:31:52.423255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# performing pca on the final principal components\ndf_pca = cudf.DataFrame(pca_final.fit_transform(train))\ndf_pca['target'] = x_target\ndf_pca.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-28T04:31:52.427510Z","iopub.execute_input":"2022-07-28T04:31:52.427813Z","iopub.status.idle":"2022-07-28T04:32:08.042544Z","shell.execute_reply.started":"2022-07-28T04:31:52.427785Z","shell.execute_reply":"2022-07-28T04:32:08.041530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# free memory\ndel train,x_target\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T04:32:08.044143Z","iopub.execute_input":"2022-07-28T04:32:08.044803Z","iopub.status.idle":"2022-07-28T04:32:08.198178Z","shell.execute_reply.started":"2022-07-28T04:32:08.044761Z","shell.execute_reply":"2022-07-28T04:32:08.196650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# FEATURES\nFEATURES = df_pca.columns\nprint(f'There are {len(FEATURES)} features!')","metadata":{"execution":{"iopub.status.busy":"2022-07-28T04:32:08.199720Z","iopub.execute_input":"2022-07-28T04:32:08.200231Z","iopub.status.idle":"2022-07-28T04:32:08.209785Z","shell.execute_reply.started":"2022-07-28T04:32:08.200190Z","shell.execute_reply":"2022-07-28T04:32:08.208666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LOAD XGB LIBRARY\nfrom sklearn.model_selection import KFold\nimport xgboost as xgb\nprint('XGB Version',xgb.__version__)\n\n# XGB MODEL PARAMETERS\nxgb_parms = { \n    'max_depth':4, \n    'learning_rate':0.05, \n    'subsample':0.8,\n    'colsample_bytree':0.6, \n    'eval_metric':'logloss',\n    'objective':'binary:logistic',\n    'tree_method':'gpu_hist',\n    'predictor':'gpu_predictor',\n    'random_state':42\n}","metadata":{"execution":{"iopub.status.busy":"2022-07-28T04:32:08.211558Z","iopub.execute_input":"2022-07-28T04:32:08.211988Z","iopub.status.idle":"2022-07-28T04:32:08.300234Z","shell.execute_reply.started":"2022-07-28T04:32:08.211915Z","shell.execute_reply":"2022-07-28T04:32:08.299227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# NEEDED WITH DeviceQuantileDMatrix BELOW\nclass IterLoadForDMatrix(xgb.core.DataIter):\n    def __init__(self, df=None, features=None, target=None, batch_size=256*1024):\n        self.features = features\n        self.target = target\n        self.df = df\n        self.it = 0 # set iterator to 0\n        self.batch_size = batch_size\n        self.batches = int( np.ceil( len(df) / self.batch_size ) )\n        super().__init__()\n\n    def reset(self):\n        '''Reset the iterator'''\n        self.it = 0\n\n    def next(self, input_data):\n        '''Yield next batch of data.'''\n        if self.it == self.batches:\n            return 0 # Return 0 when there's no more batch.\n        \n        a = self.it * self.batch_size\n        b = min( (self.it + 1) * self.batch_size, len(self.df) )\n        dt = cudf.DataFrame(self.df.iloc[a:b])\n        input_data(data=dt[self.features], label=dt[self.target]) #, weight=dt['weight'])\n        self.it += 1\n        return 1","metadata":{"execution":{"iopub.status.busy":"2022-07-28T04:32:08.301666Z","iopub.execute_input":"2022-07-28T04:32:08.302002Z","iopub.status.idle":"2022-07-28T04:32:08.312992Z","shell.execute_reply.started":"2022-07-28T04:32:08.301967Z","shell.execute_reply":"2022-07-28T04:32:08.311904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def amex_metric_mod(y_true, y_pred):\n\n    labels     = np.transpose(np.array([y_true, y_pred]))\n    labels     = labels[labels[:, 1].argsort()[::-1]]\n    weights    = np.where(labels[:,0]==0, 20, 1)\n    cut_vals   = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four   = np.sum(cut_vals[:,0]) / np.sum(labels[:,0])\n\n    gini = [0,0]\n    for i in [1,0]:\n        labels         = np.transpose(np.array([y_true, y_pred]))\n        labels         = labels[labels[:, i].argsort()[::-1]]\n        weight         = np.where(labels[:,0]==0, 20, 1)\n        weight_random  = np.cumsum(weight / np.sum(weight))\n        total_pos      = np.sum(labels[:, 0] *  weight)\n        cum_pos_found  = np.cumsum(labels[:, 0] * weight)\n        lorentz        = cum_pos_found / total_pos\n        gini[i]        = np.sum((lorentz - weight_random) * weight)\n\n    return 0.5 * (gini[1]/gini[0] + top_four)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T04:32:08.314677Z","iopub.execute_input":"2022-07-28T04:32:08.315056Z","iopub.status.idle":"2022-07-28T04:32:08.327246Z","shell.execute_reply.started":"2022-07-28T04:32:08.315017Z","shell.execute_reply":"2022-07-28T04:32:08.326332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# VERSION NAME FOR SAVED MODEL FILES\nVER = 1\n\nskf = KFold(n_splits=5, shuffle=True, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T04:32:08.329712Z","iopub.execute_input":"2022-07-28T04:32:08.330381Z","iopub.status.idle":"2022-07-28T04:32:08.337646Z","shell.execute_reply.started":"2022-07-28T04:32:08.330345Z","shell.execute_reply":"2022-07-28T04:32:08.336668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# build and save models\n\nfor fold,(train_idx,valid_idx) in enumerate(skf.split(df_pca,df_pca.target)):\n    print('#'*25)\n    print('### Fold',fold+1)\n    print('### Train size',len(train_idx),'Valid size',len(valid_idx))\n    print('#'*25)\n    \n    # TRAIN, VALID, TEST FOR FOLD K\n    Xy_train = IterLoadForDMatrix(df_pca.loc[train_idx], FEATURES, 'target')\n    X_valid = df_pca.loc[valid_idx, FEATURES]\n    y_valid = df_pca.loc[valid_idx, 'target']\n    \n    dtrain = xgb.DeviceQuantileDMatrix(Xy_train, max_bin=256)\n    dvalid = xgb.DMatrix(data=X_valid, label=y_valid)\n\n    # TRAIN MODEL FOLD K\n    model = xgb.train(xgb_parms, \n                dtrain=dtrain,\n                evals=[(dtrain,'train'),(dvalid,'valid')],\n                num_boost_round=9999,\n                early_stopping_rounds=100,\n                verbose_eval=100) \n    model.save_model(f'XGB_v{VER}_fold{fold}.xgb')","metadata":{"execution":{"iopub.status.busy":"2022-07-28T04:32:08.340700Z","iopub.execute_input":"2022-07-28T04:32:08.340974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CLEAN RAM\ndel df_pca\n_ = gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load test data\ntest = cudf.read_parquet(\"../input/amex-data-integer-dtypes-parquet-format/test.parquet\")\ntest = preprocessing(test,training_data=False)\n\ntest = test.to_pandas()\ntest_customers = test.customer_ID\ntest.drop(['customer_ID'],1,inplace=True)\ntest = pd.DataFrame(pca_final.fit_transform(test))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make predictions\nFOLDS = 5\ntest_preds = []\n\ndtest = xgb.DMatrix(data=test)\ndel test\ngc.collect()\n\n# INFER XGB MODELS ON TEST DATA\nmodel = xgb.Booster()\nmodel.load_model(f'XGB_v{VER}_fold0.xgb')\npreds = model.predict(dtest)\nfor f in range(1,FOLDS):\n    model.load_model(f'XGB_v{VER}_fold{f}.xgb')\n    preds += model.predict(dtest)\npreds /= FOLDS\ntest_preds.append(preds)\n\n# CLEAN MEMORY\ndel dtest, model\n_ = gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lets prepare for the prediction submission\ntest_preds = np.concatenate(test_preds)\nsub = pd.DataFrame()\nsub['customer_ID'] = test_customers\nsub['prediction'] = test_preds\nsub.to_csv('submission.csv',index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}