{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Learning to Rank.\nWe are going to test the hypothesis, that there are time-related features in the dataset, which have a strong correlation with time. For this we will start with importing Some Basic Libraries Like:\n1. os\n2. gc\n3. glob \n4. tqdm \n5. NumPy\n6. Pandas\n**NumPy**:\n           NumPy is a Python library used for working with arrays. It also has functions for working in domain of linear algebra, fourier transform, and matrices.\n           \n **Pandas**:\n           Pandas is used to analyze data. Learning by Reading.\n  ","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport glob\nimport tqdm\nimport numpy as np\nimport pandas as pd\n\nAhmed = 30\nnp.random.seed(Ahmed)\nos.environ['PYTHON'] = str(Ahmed)","metadata":{"execution":{"iopub.status.busy":"2022-06-17T20:07:04.245041Z","iopub.execute_input":"2022-06-17T20:07:04.245889Z","iopub.status.idle":"2022-06-17T20:07:04.278826Z","shell.execute_reply.started":"2022-06-17T20:07:04.245786Z","shell.execute_reply":"2022-06-17T20:07:04.277921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = pd.read_csv('../input/amex-default-prediction/train_labels.csv')\ntrain_labels['customer_ID'] = train_labels['customer_ID'].apply(lambda x: int(x[-16:], 16)).astype(np.int64)\ntrain_labels = train_labels.set_axis(train_labels['customer_ID'])\ntrain_labels = train_labels.drop(['customer_ID'], axis=1)\n\ntrain_pkls = sorted(glob.glob('../input/amex-default-prediction/train_data.csv'))\ntest_pkls = sorted(glob.glob('../input/amex-default-prediction/test_data.csv'))","metadata":{"execution":{"iopub.status.busy":"2022-06-17T20:14:26.336586Z","iopub.execute_input":"2022-06-17T20:14:26.337217Z","iopub.status.idle":"2022-06-17T20:14:27.317791Z","shell.execute_reply.started":"2022-06-17T20:14:26.33718Z","shell.execute_reply":"2022-06-17T20:14:27.316724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CPU Libraries\nimport pandas as pd, numpy as np \n# GPU libraries\nimport cupy, cudf \n\n# Standard library\nimport os\nfrom functools import reduce\n\n# Machine learning\nfrom sklearn.model_selection import KFold\nimport xgboost as xgb\n\n# Plotting\nimport matplotlib.pyplot as plt, gc, os\n\n","metadata":{"execution":{"iopub.status.busy":"2022-06-17T20:18:11.078217Z","iopub.execute_input":"2022-06-17T20:18:11.078932Z","iopub.status.idle":"2022-06-17T20:18:14.766065Z","shell.execute_reply.started":"2022-06-17T20:18:11.078895Z","shell.execute_reply":"2022-06-17T20:18:14.765156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"VER = 1\n\n# Random seed\nSEED = 12\n\n# Fill nan values with int8\nNAN_VALUE = -127 \n\n# Training data path\nTRAIN_PATH = '../input/amex-default-prediction/train_data.csv'\n\n# Target path\nLABEL_PATH = '../input/amex-default-prediction/train_labels.csv'\n\n# Test data path\nTEST_PATH = '../input/amex-default-prediction/test_data.csv'\n\n# Folds per model\nFOLDS = 5\n\n# Train with 100% of the samples each fold or a random subset if < 1.0\nTRAIN_SUBSAMPLE = 1.0\n\n# Top x important to visualize\nTOP_NUM_FEATURES = 20\n\n# Partition for test data\nNUM_PARTS = 4","metadata":{"execution":{"iopub.status.busy":"2022-06-17T20:19:58.779972Z","iopub.execute_input":"2022-06-17T20:19:58.780754Z","iopub.status.idle":"2022-06-17T20:19:58.786951Z","shell.execute_reply.started":"2022-06-17T20:19:58.780689Z","shell.execute_reply":"2022-06-17T20:19:58.78581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def defaultmodel(y_true, y_pred):\n\n    labels     = np.transpose(np.array([y_true, y_pred]))\n    labels     = labels[labels[:, 1].argsort()[::-1]]\n    weights    = np.where(labels[:, 0] == 0, 20, 1)\n    cut_vals   = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four   = np.sum(cut_vals[:, 0]) / np.sum(labels[:, 0])\n\n    gini = [0,0]\n    for i in [1,0]:\n        labels         = np.transpose(np.array([y_true, y_pred]))\n        labels         = labels[labels[:, i].argsort()[::-1]]\n        weight         = np.where(labels[:,0]==0, 20, 1)\n        weight_random  = np.cumsum(weight / np.sum(weight))\n        total_pos      = np.sum(labels[:, 0] *  weight)\n        cum_pos_found  = np.cumsum(labels[:, 0] * weight)\n        lorentz        = cum_pos_found / total_pos\n        gini[i]        = np.sum((lorentz - weight_random) * weight)\n\n    return 0.5 * (gini[1] / gini[0] + top_four)","metadata":{"execution":{"iopub.status.busy":"2022-06-17T20:20:27.359709Z","iopub.execute_input":"2022-06-17T20:20:27.360583Z","iopub.status.idle":"2022-06-17T20:20:27.372342Z","shell.execute_reply.started":"2022-06-17T20:20:27.360541Z","shell.execute_reply":"2022-06-17T20:20:27.371112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class IterLoadForDMatrix(xgb.core.DataIter):\n    # Higher batch sizes leads to lower asymptotic test accuracy according to https://medium.com/mini-distill/effect-of-batch-size-on-training-dynamics-21c14f7a716e\n    def __init__(self, df=None, features=None, target=None, batch_size=256*1024):\n        self.features = features\n        self.target = target\n        self.df = df\n        self.it = 0 # Set iterator to 0\n        self.batch_size = batch_size\n        self.batches = int( np.ceil( len(df) / self.batch_size ) )\n        super().__init__()\n\n    def reset(self):\n        '''Reset the iterator'''\n        self.it = 0\n\n    def next(self, input_data):\n        '''Return the next batch of data.'''\n        # Return 0 when there's no more data\n        if self.it == self.batches:\n            return 0 \n        \n        # Starting row index for the current batch\n        a = self.it * self.batch_size\n        # Ending row index for the current batch\n        # Once (self.it + 1) * self.batch_size exceeds the max number of rows, that means we have arrived at the end of the frame, and last row is the ending index \n        b = min( (self.it + 1) * self.batch_size, len(self.df) )\n        # Current batch of training samples\n        dt = cudf.DataFrame(self.df.iloc[a:b])\n        # The input_data function is passed in by XGBoost who has the similar signature to the ``DMatrix`` constructor\n        input_data(data=dt[self.features], label=dt[self.target])\n        self.it += 1\n        return 1","metadata":{"execution":{"iopub.status.busy":"2022-06-17T20:21:42.838289Z","iopub.execute_input":"2022-06-17T20:21:42.838676Z","iopub.status.idle":"2022-06-17T20:21:42.849255Z","shell.execute_reply.started":"2022-06-17T20:21:42.838645Z","shell.execute_reply":"2022-06-17T20:21:42.848288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_file(path = '', usecols = None):\n    # Load dataframe with user specified columns\n    if usecols is not None: \n        df = cudf.read_parquet(path, columns=usecols)\n    else: \n        # Read all columns\n        df = cudf.read_parquet(path)\n    # Reduce Dtype for customer and date\n    # Obtained the last 16 characters and obtain the integer value represented by each customer_ID, finally converting to 'int64' dtype\n    df['customer_ID'] = df['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\n    # Convert date column from 'object' to datetime\n    df.S_2 = cudf.to_datetime(df.S_2)\n    # Sort by customer and date (so agg('last') works correctly)\n    df = df.sort_values(['customer_ID','S_2'])\n    df = df.reset_index(drop=True)\n    # Fill nan\n    df = df.fillna(NAN_VALUE) \n    print('shape of data:', df.shape)\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-06-17T20:22:10.052862Z","iopub.execute_input":"2022-06-17T20:22:10.053326Z","iopub.status.idle":"2022-06-17T20:22:10.062797Z","shell.execute_reply.started":"2022-06-17T20:22:10.053285Z","shell.execute_reply":"2022-06-17T20:22:10.061784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# Lets Get Started.......","metadata":{}},{"cell_type":"code","source":"def process_and_feature_engineer(df):\n    # Feature engineering from https://www.kaggle.com/code/huseyincot/amex-agg-data-how-it-created\n    all_cols = [c for c in list(df.columns) if c not in ['customer_ID','S_2']]\n    # Categorical features\n    cat_features = [\"B_30\",\"B_38\",\"D_114\",\"D_116\",\"D_117\",\"D_120\",\"D_126\",\"D_63\",\"D_64\",\"D_66\",\"D_68\"]\n    # Numerical features\n    num_features = [col for col in all_cols if col not in cat_features]\n\n    # Group by customer id and generate new columns ['mean', 'std', 'min', 'max', 'last'] for each col in 'num_features' (5 x len(num_features) total)\n    # Could also use df.groupby(\"customer_ID\")[num_features].describe(), but that includes quantiles as well so it may more more wasteful\n    test_num_agg = df.groupby(\"customer_ID\")[num_features].agg(['mean', 'std', 'min', 'max', 'last'])\n    # The test_num_agg.columns map object returns tuples of the form ('parent_col', 'child_col') e.g., ('D_39', 'mean'), ('D_39', 'std'), ...\n    # Join the elements of this tuple using an underscore\n    test_num_agg.columns = ['_'.join(x) for x in test_num_agg.columns]\n\n    # Create features for categorical columns of the form ('B_30_count'), ('B_30_last'), ..., ('D_117', 'nunique'), ...\n    test_cat_agg = df.groupby(\"customer_ID\")[cat_features].agg(['count', 'last', 'nunique'])\n    test_cat_agg.columns = ['_'.join(x) for x in test_cat_agg.columns]\n\n    # Column bind\n    df = cudf.concat([test_num_agg, test_cat_agg], axis=1)\n    del test_num_agg, test_cat_agg\n    print('shape after engineering', df.shape )\n    \n    return df","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-17T20:22:42.38423Z","iopub.execute_input":"2022-06-17T20:22:42.384589Z","iopub.status.idle":"2022-06-17T20:22:42.394684Z","shell.execute_reply.started":"2022-06-17T20:22:42.38456Z","shell.execute_reply":"2022-06-17T20:22:42.393925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_rows(customers, test, NUM_PARTS, verbose = ''):\n    # Split customers_ID into 4 parts, each part contains 'chunk' number of ID's at least (floor division rounds down to the nearest integer)\n    chunk = len(customers) // NUM_PARTS\n    \n    # Report to user\n    if verbose != '':\n        print(f'We will process {verbose} data as {NUM_PARTS} separate parts.')\n        print(f'There will be {chunk} customers in each part (except the last part).')\n        print('Below are number of rows in each part:')\n        \n    # Instantiate output container\n    rows = []\n\n    for k in range(NUM_PARTS):\n        # Once we reach the last chunk\n        if k == NUM_PARTS-1: \n            # The starting row index will be k*chunk, so we subset the 'customers' series from that position\n            cc = customers[k*chunk:]\n        else: \n            # Chunks before we arrive at the last chunk, (k+1) ensures we include the last row of each chunk\n            cc = customers[k*chunk:(k+1)*chunk]\n        \n        # The expression test.loc[test.customer_ID.isin(cc)] returns the rows from 'test' whose 'customer_ID' are in the set of  'customer_ID' as defined in 'cc'\n        s = test.loc[test.customer_ID.isin(cc)].shape[0]\n        # Append number of rows to container\n        rows.append(s)\n        \n    if verbose != '': \n        print(rows)\n\n    return rows, chunk","metadata":{"execution":{"iopub.status.busy":"2022-06-17T20:23:12.632704Z","iopub.execute_input":"2022-06-17T20:23:12.63311Z","iopub.status.idle":"2022-06-17T20:23:12.642207Z","shell.execute_reply.started":"2022-06-17T20:23:12.633079Z","shell.execute_reply":"2022-06-17T20:23:12.641108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read in data\nprint('Reading train data...')\ntrain = pd.read_csv('../input/amex-default-prediction/train_data.csv')\n","metadata":{"execution":{"iopub.status.busy":"2022-06-17T20:36:02.918681Z","iopub.execute_input":"2022-06-17T20:36:02.919377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = process_and_feature_engineer(train)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-06-15T20:27:42.951204Z","iopub.execute_input":"2022-06-15T20:27:42.951575Z","iopub.status.idle":"2022-06-15T20:27:45.439628Z","shell.execute_reply.started":"2022-06-15T20:27:42.95154Z","shell.execute_reply":"2022-06-15T20:27:45.438805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Add Targets\ntargets = cudf.read_csv(LABEL_PATH)\n# Reduce datatype\ntargets['customer_ID'] = targets['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\n# Set index for joining\ntargets = targets.set_index('customer_ID')\n# Left join target onto train using 'customer_ID'\ntrain = train.merge(targets, left_index=True, right_index=True, how='left')\n# Reduce dtype\ntrain.target = train.target.astype('int8')\ndel targets\n\n# Needed to make CV deterministic (cudf merge above randomly shuffles rows)\ntrain = train.sort_index().reset_index(inplace=False)\n\n# Features (this is needed for the modeling)\nFEATURES = train.columns[1:-1]\nprint(f'There are {len(FEATURES)} features')","metadata":{"execution":{"iopub.status.busy":"2022-06-15T20:28:57.829928Z","iopub.execute_input":"2022-06-15T20:28:57.830275Z","iopub.status.idle":"2022-06-15T20:28:59.110238Z","shell.execute_reply.started":"2022-06-15T20:28:57.830247Z","shell.execute_reply":"2022-06-15T20:28:59.109223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.iloc[:5, :]","metadata":{"execution":{"iopub.status.busy":"2022-06-15T20:29:34.552073Z","iopub.execute_input":"2022-06-15T20:29:34.552531Z","iopub.status.idle":"2022-06-15T20:29:34.582914Z","shell.execute_reply.started":"2022-06-15T20:29:34.552484Z","shell.execute_reply":"2022-06-15T20:29:34.582095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# XGB model parameters\nxgb_parms = { \n    # Depth of trees\n    'max_depth':4, \n    # Step size for boosting\n    'learning_rate':0.05, \n    # Randomly sample 80% of instances for each boosting iteration (growing each tree)\n    'subsample':0.8,\n    # Randomly sample 60% (about 550) of all features for each boosting iteration (growing each tree)\n    'colsample_bytree':0.6, \n    # Negative log-likelihood\n    'eval_metric':'logloss',\n    # Loss function with output as probabilities\n    'objective':'binary:logistic',\n    'tree_method':'gpu_hist',\n    'predictor':'gpu_predictor',\n    'random_state': SEED\n}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Feature importance output container (list of pandas data frames)\nimportances = []\n# Out-of-fold prediction container\noof = []\n# Free GPU memory\ntrain = train.to_pandas() \n# Free resources\ngc.collect()\n\nskf = KFold(n_splits=FOLDS, shuffle=True, random_state=SEED)\n# Fives rounds of fitting\nfor fold,(train_idx, valid_idx) in enumerate(skf.split(train, train.target)):\n    \n    # Train with subsample of train fold data\n    if TRAIN_SUBSAMPLE < 1.0:\n        np.random.seed(SEED)\n        # Randomly obtain (int(len(train_idx) * TRAIN_SUBSAMPLE) number of indices from the train fold data\n        train_idx = np.random.choice(train_idx, int(len(train_idx) * TRAIN_SUBSAMPLE), replace=False)\n        np.random.seed(None)\n    \n    # Print messages to console\n    print('#'* 60)\n    print('# Fold',fold + 1)\n    print('# Training set size', len(train_idx), 'Validation set size', len(valid_idx))\n    print(f'# Training with {int(TRAIN_SUBSAMPLE * 100)}% of the samples of the current fold...')\n    print('#'* 60)\n    \n    # Train and validate for the Kth fold\n    Xy_train = IterLoadForDMatrix(df=train.loc[train_idx], features=FEATURES, target='target')\n    X_valid = train.loc[valid_idx, FEATURES]\n    y_valid = train.loc[valid_idx, 'target']\n    \n    # Pass the batch of training samples to DeviceQuantileDMatrix with 'max_bin' set to control the number of bins during quantization\n    dtrain = xgb.DeviceQuantileDMatrix(Xy_train, max_bin=256)\n    dvalid = xgb.DMatrix(data=X_valid, label=y_valid)\n    \n    # Train model for the kth fold\n    model = xgb.train(\n        # Booster params\n        xgb_parms, \n        # Training data\n        dtrain=dtrain,\n        num_boost_round=9999,\n        # List of validation sets for which metrics will evaluated during training\n        evals=[(dtrain,'train'),(dvalid,'validate')],\n        # Validation metric needs to improve at least once in every 100 rounds of boosting before early stopping\n        early_stopping_rounds=100,\n        # Print evaluation metric on the validation set every 100 rounds of boosting\n        verbose_eval=100\n    ) \n    model.save_model(f'XGB_v{VER}_fold{fold}.xgb')\n    \n    # Feature importance for the kth fold\n    # 'Weight' is the number of times a feature is used to split the data across all trees\n    dd = model.get_score(importance_type='weight')\n    df = pd.DataFrame({'feature': dd.keys(), f'importance_{fold}': dd.values()})\n    importances.append(df)\n            \n    # Out-of-fold predictions for the kth fold\n    # The predict method returns numpy array\n    oof_preds = model.predict(dvalid)\n    score = amex_metric_mod(y_true=y_valid.values, y_pred=oof_preds)\n    print(f'Metric for fold {fold + 1} = ', score, '\\n')\n    \n    # Store predictions\n    df = train.loc[valid_idx, ['customer_ID','target'] ].copy()\n    df['oof_pred'] = oof_preds\n    oof.append(df)\n    \n    # Clean resources\n    del dtrain, Xy_train, dd, df\n    del X_valid, y_valid, dvalid, model\n    _ = gc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Feature importance\nimportances[0]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof[0]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Row bind 5 sets of out-of-fold predictions (each with 91783 rows) into a single set of prediction frame (458,915 rows)\noof = pd.concat(oof, axis=0, ignore_index=True).set_index('customer_ID')\n# Overall metric score\nscore = amex_metric_mod(y_true=oof.target.values, y_pred=oof.oof_pred.values)\nprint('Overall CV Metric Score =', score)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train\n_ = gc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read in 'customer_ID' from training data\noof_xgb = pd.read_parquet(TRAIN_PATH, columns=['customer_ID']).drop_duplicates()\n\n# Reduce customer_ID data size\noof_xgb['customer_ID_hash'] = oof_xgb['customer_ID'].apply(lambda x: int(x[-16:],16) ).astype('int64')\noof_xgb = oof_xgb.set_index('customer_ID_hash')\noof_xgb = oof_xgb.merge(oof, left_index=True, right_index=True)\noof_xgb = oof_xgb.sort_index().reset_index(drop=True)\n\n# Write predictions to disk\noof_xgb.to_csv(f'oof_xgb_v{VER}.csv',index=False)\noof_xgb.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot predictions\nplt.hist(oof_xgb.oof_pred.values, bins=100)\nplt.title('OOF Predictions')\nplt.xlabel('Probabilities of Default')\nplt.ylabel('Frequencies')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del oof_xgb, oof\n_ = gc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feat_imp = reduce(lambda x, y: x.merge(y, on='feature', how='left'), importances)\nfeat_imp","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The features go down and the folds go across\n# Exclude the first column, which are the feature names, compute the means across\nfeat_imp['importance'] = feat_imp.iloc[:,1:].mean(axis=1)\n# Sort descending\nfeat_imp = feat_imp.sort_values('importance',ascending=False)\nfeat_imp","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Write to disk\nfeat_imp.to_csv(f'xgb_feature_importance_v{VER}.csv',index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Size\nplt.figure(figsize=(10, 5 * TOP_NUM_FEATURES // 10))\n\n# Horiziontal bar chart\nplt.barh(y=np.arange(TOP_NUM_FEATURES, 0, -1), width=feat_imp.importance.values[:TOP_NUM_FEATURES])\n\n# Yticks labels are the feature names\nplt.yticks(np.arange(TOP_NUM_FEATURES, 0, -1), feat_imp.feature.values[:TOP_NUM_FEATURES])\nplt.xlabel('Weight (The number of times a feature is used to split the data across all trees)')\n\n# Title\nplt.title(f'XGB Feature Importance - Top {TOP_NUM_FEATURES}')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Reading test data with columns `customer_ID` and `S_2` ...')\ntest = read_file(path = TEST_PATH, usecols = ['customer_ID','S_2'])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Return a copy of the array underlying 'customer_ID' collapsed into one dimension\n# No duplicates\ncustomers = test[['customer_ID']].drop_duplicates().sort_index().values.flatten()\nrows, num_cust = get_rows(customers, test[['customer_ID']], NUM_PARTS = NUM_PARTS, verbose = 'test')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate one part at a time\n# Initialize indices\nskip_rows = 0\nskip_cust = 0\n# Containers\ntest_preds = []\n\nfor k in range(NUM_PARTS):\n    \n    # Read in one partition of the test data\n    print(f'\\nReading partition {k + 1} of test data...')\n    \n    test = read_file(path = TEST_PATH)\n    # The 'rows' list contains the numbers of rows for each partition\n    test = test.iloc[skip_rows:skip_rows + rows[k]]\n    # Increment the starting index by the numbers of rows for each partition\n    skip_rows += rows[k]\n    \n    print(f'=> Test partition {k + 1} has shape', test.shape )\n    \n    # Process and feature engineering for each partition of the test data\n    test = process_and_feature_engineer(test)\n    \n    if k == NUM_PARTS-1: \n        # Special handling once we arrive at the lask chunk by simple getting the rest of the remaining rows\n        test = test.loc[customers[skip_cust:]]\n    else: \n        test = test.loc[customers[skip_cust:skip_cust + num_cust]]\n        \n    # Increment the ending index by 'num_cust', as each partision has at least 'num_cust' number of unique id's\n    skip_cust += num_cust\n    \n    # Load test data into a dmatrix\n    X_test = test[FEATURES]\n    dtest = xgb.DMatrix(data=X_test)\n    # Reduce memory\n    test = test[['P_2_mean']] \n    del X_test\n    gc.collect()\n\n    # Initializations\n    model = xgb.Booster()\n    model.load_model(f'XGB_v{VER}_fold0.xgb')\n    # This is an numpy array, so we use += below for element-wise addition\n    preds = model.predict(dtest)\n    \n    for f in range(1, FOLDS):\n        model.load_model(f'XGB_v{VER}_fold{f}.xgb')\n        # Take the sum of all predictions\n        preds += model.predict(dtest)\n    # Divide by the number of folds to obtain average prediction for each customer_ID over cv 5 folds\n    preds /= FOLDS\n    # Append prediction for partition k to container\n    test_preds.append(preds)\n\n    # CLEAN MEMORY\n    del dtest, model\n    _ = gc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Join the 4 partitions of preditions (231155 * 3 + 231156 predictions)\ntest_preds = np.concatenate(test_preds)\n\n# Create data frame with predictions and customer_ID as row indices\ntest = cudf.DataFrame(index=customers,data={'prediction':test_preds})\n\n# Read 'customer_ID' column from the sample submission\nsub = cudf.read_csv('../input/amex-default-prediction/sample_submission.csv')[['customer_ID']]\n\n# Processing\nsub['customer_ID_hash'] = sub['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\nsub = sub.set_index('customer_ID_hash')\n\n# Left join processed sample submission 'customer_ID' onto prediction data frame using 'customer_ID' as the key\nsub = sub.merge(test[['prediction']], left_index=True, right_index=True, how='left')\nsub = sub.reset_index(drop=True)\n\n# Write preditions to file \nsub.to_csv(f'test_predictions_xgb_v{VER}.csv',index=False)\nprint('Prediction file has shape', sub.shape )\nsub.head()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize \nplt.hist(sub.to_pandas().prediction, bins=100)\nplt.title('Test Predictions')\nplt.xlabel('Probabilities of Default')\nplt.ylabel('Frequencies')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# If You Like Please Upvote.\n# Ahmed Ali","metadata":{}}]}