{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# American Express - Default Prediction\n# By Mohamed Eltayeb","metadata":{}},{"cell_type":"markdown","source":"## Note: Please Enable the GPU and upvote if you find this notebook useful.","metadata":{}},{"cell_type":"markdown","source":"# Import libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport cudf \nimport cupy\nimport gc\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\n\n\nfrom sklearn.model_selection import KFold\nfrom sklearn import base\n\nfrom catboost import CatBoostClassifier\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nplt.rcParams[\"figure.figsize\"] = (12, 8)","metadata":{"execution":{"iopub.execute_input":"2022-08-23T11:08:08.784945Z","iopub.status.busy":"2022-08-23T11:08:08.783916Z","iopub.status.idle":"2022-08-23T11:08:13.751338Z","shell.execute_reply":"2022-08-23T11:08:13.750326Z","shell.execute_reply.started":"2022-08-23T11:08:08.784820Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Functions","metadata":{}},{"cell_type":"code","source":"#Competition Metric\ndef amex_metric_mod(y_true, y_pred):\n\n    labels     = np.transpose(np.array([y_true, y_pred]))\n    labels     = labels[labels[:, 1].argsort()[::-1]]\n    weights    = np.where(labels[:,0]==0, 20, 1)\n    cut_vals   = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four   = np.sum(cut_vals[:,0]) / np.sum(labels[:,0])\n\n    gini = [0,0]\n    for i in [1,0]:\n        labels         = np.transpose(np.array([y_true, y_pred]))\n        labels         = labels[labels[:, i].argsort()[::-1]]\n        weight         = np.where(labels[:,0]==0, 20, 1)\n        weight_random  = np.cumsum(weight / np.sum(weight))\n        total_pos      = np.sum(labels[:, 0] *  weight)\n        cum_pos_found  = np.cumsum(labels[:, 0] * weight)\n        lorentz        = cum_pos_found / total_pos\n        gini[i]        = np.sum((lorentz - weight_random) * weight)\n\n    return [0.5 * (gini[1]/gini[0] + top_four),top_four, gini[1]/gini[0]]","metadata":{"execution":{"iopub.execute_input":"2022-08-23T11:08:13.754342Z","iopub.status.busy":"2022-08-23T11:08:13.753566Z","iopub.status.idle":"2022-08-23T11:08:13.764938Z","shell.execute_reply":"2022-08-23T11:08:13.763723Z","shell.execute_reply.started":"2022-08-23T11:08:13.754302Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Plot the LGBM Features Importances\ndef ShowImp(features, importances, num = 20, fig_size = (2*num, num)):\n    feature_imp = cudf.DataFrame({'Value':importances,'Feature':features})\n    feature_imp = feature_imp.sort_values(by=\"Value\",ascending=False)[:num]\n    print(feature_imp)","metadata":{"execution":{"iopub.execute_input":"2022-08-23T11:08:13.767646Z","iopub.status.busy":"2022-08-23T11:08:13.767158Z","iopub.status.idle":"2022-08-23T11:08:13.776608Z","shell.execute_reply":"2022-08-23T11:08:13.775605Z","shell.execute_reply.started":"2022-08-23T11:08:13.767605Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Reduce Memory Usage\ndef reduce_memory_usage(df):\n    \n    for col in df.columns:\n        col_type = df[col].dtype.name\n        if ((col_type != 'datetime64[ns]') & (col_type != 'category')):\n            if (col_type != 'object'):\n                c_min = df[col].min()\n                c_max = df[col].max()\n\n                if str(col_type)[:3] == 'int':\n                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                        df[col] = df[col].astype(np.int8)\n                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                        df[col] = df[col].astype(np.int16)\n                    elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                        df[col] = df[col].astype(np.int32)\n                    elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                        df[col] = df[col].astype(np.int64)\n\n                else:\n\n                    if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                        df[col] = df[col].astype(np.float32)\n                    else:\n                        pass\n    \n    return df","metadata":{"execution":{"iopub.execute_input":"2022-08-23T11:08:13.779907Z","iopub.status.busy":"2022-08-23T11:08:13.779484Z","iopub.status.idle":"2022-08-23T11:08:13.790714Z","shell.execute_reply":"2022-08-23T11:08:13.789737Z","shell.execute_reply.started":"2022-08-23T11:08:13.779870Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#READING DATA\ndef read_file(path = '', usecols = None):\n    \n    # LOAD DATAFRAME\n    if usecols is not None: df = cudf.read_parquet(path, columns=usecols)\n    else: df = cudf.read_parquet(path)\n        \n    # REDUCE DTYPE FOR CUSTOMER AND DATE\n    df['customer_ID'] = df['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\n    df.S_2 = cudf.to_datetime( df.S_2 )\n    \n    # SORT BY CUSTOMER AND DATE\n    df = df.sort_values(['customer_ID','S_2'])\n    df = df.reset_index(drop=True)\n    \n    #REDUCE MEMORY USAGE\n    df = reduce_memory_usage(df)\n    print('shape of data:', df.shape)\n    \n    return df","metadata":{"execution":{"iopub.execute_input":"2022-08-23T11:08:13.792784Z","iopub.status.busy":"2022-08-23T11:08:13.792277Z","iopub.status.idle":"2022-08-23T11:08:13.802915Z","shell.execute_reply":"2022-08-23T11:08:13.801877Z","shell.execute_reply.started":"2022-08-23T11:08:13.792746Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Data Preparation","metadata":{}},{"cell_type":"code","source":"print('Reading train data...')\nTRAIN_PATH = '../input/amex-data-integer-dtypes-parquet-format/train.parquet'\ntrain_df = read_file(path = TRAIN_PATH)","metadata":{"execution":{"iopub.execute_input":"2022-08-23T10:20:09.121080Z","iopub.status.busy":"2022-08-23T10:20:09.120352Z","iopub.status.idle":"2022-08-23T10:20:47.503685Z","shell.execute_reply":"2022-08-23T10:20:47.502601Z","shell.execute_reply.started":"2022-08-23T10:20:09.121045Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Temporary Dataset","metadata":{}},{"cell_type":"code","source":"train_df_new = cudf.DataFrame(train_df['customer_ID'].drop_duplicates(),columns=['customer_ID'])","metadata":{"execution":{"iopub.execute_input":"2022-08-23T10:20:47.505775Z","iopub.status.busy":"2022-08-23T10:20:47.505325Z","iopub.status.idle":"2022-08-23T10:20:47.524777Z","shell.execute_reply":"2022-08-23T10:20:47.523943Z","shell.execute_reply.started":"2022-08-23T10:20:47.505739Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Polar Coordinates of P_2 and The Other Features","metadata":{}},{"cell_type":"code","source":"def Polar(X,y, a = 9.9, b = 9.9): # a and b represnt the center\n    r = np.sqrt((X-a)**2 + (y-b)**2)\n    phi = np.arctan2((y-a), (X-b))\n    return r, phi\n\nfeats = ['B_18','B_2','B_33','D_62','D_77','D_47','P_3','D_45','D_51','R_27','S_25','D_112',\n         'D_121','D_128','D_52','D_115','D_114','D_127','D_68','D_118','D_119','D_122','D_54',\n         'D_129','D_134','S_8','D_92','R_12','D_91','D_76','D_56','D_117','B_42','S_6','D_73',\n         'S_13','D_142','D_126','D_140','R_16','B_41','R_21','B_32','B_24','D_133','R_17','D_120',\n         'D_46','D_145','D_139','D_143','D_138','D_136','R_6','D_135','B_25','D_137','R_26','D_141',\n         'R_13','D_79','D_39','D_89','D_113','R_20','R_15','S_15','R_8','D_130','D_131','D_64','D_59',\n         'R_24','R_5','B_28','D_88','R_10','P_4','D_78','D_43','R_3','R_9','D_41','D_70','R_4','D_81',\n         'D_84','S_3','B_17','D_72','B_11','B_30','S_7','B_37','B_1','B_22','B_8','R_2','B_19','D_53',\n         'B_4','D_61','B_38','B_3','B_20','R_1','D_42','B_16','B_23','B_7','D_74','D_44','D_75','D_58',\n         'B_9','D_55','D_48']\n\nfor feat in tqdm(feats):\n    \n    train_df[f'P_2_{feat}_R'], train_df[f'P_2_{feat}_Phi'] = Polar(train_df[\"P_2\"].fillna(127).to_array(),train_df[feat].fillna(127).to_array())\n    \n    train_df_new[f'P_2_{feat}_R_mean'] = train_df_new['customer_ID'].map(train_df.groupby('customer_ID')[f'P_2_{feat}_R'].mean())\n    train_df_new[f'P_2_{feat}_R_last'] = train_df_new['customer_ID'].map(train_df.groupby('customer_ID')[f'P_2_{feat}_R'].last())\n    train_df_new[f'P_2_{feat}_Phi_mean'] = train_df_new['customer_ID'].map(train_df.groupby('customer_ID')[f'P_2_{feat}_Phi'].mean())\n    train_df_new[f'P_2_{feat}_Phi_last'] = train_df_new['customer_ID'].map(train_df.groupby('customer_ID')[f'P_2_{feat}_Phi'].last())\n    \n    train_df.drop([f'P_2_{feat}_R',f'P_2_{feat}_Phi'],inplace=True,axis=1)\n    train_df_new = reduce_memory_usage(train_df_new)","metadata":{"execution":{"iopub.execute_input":"2022-08-23T10:20:47.527120Z","iopub.status.busy":"2022-08-23T10:20:47.526054Z","iopub.status.idle":"2022-08-23T10:21:57.039785Z","shell.execute_reply":"2022-08-23T10:21:57.038584Z","shell.execute_reply.started":"2022-08-23T10:20:47.527085Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Adding Bins for P_2, D_39, B_1","metadata":{}},{"cell_type":"code","source":"for feat in ['P_2','B_1']:\n    percentile = {}\n    for i, per in enumerate([0.0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0]):\n        percentile[i] = train_df[feat].quantile(per)\n    train_df[f'{feat}_bins'] = cudf.cut(train_df[feat],[percentile[x] for x in range(0,11)], labels=False).fillna(-127).astype(int)\n    \ntrain_df['D_39_bins'] = cudf.cut(train_df['D_39'],[0,31,61,91,121,151,181,np.inf], labels=False).fillna(-127).astype(int)","metadata":{"execution":{"iopub.execute_input":"2022-08-23T10:21:57.044086Z","iopub.status.busy":"2022-08-23T10:21:57.043791Z","iopub.status.idle":"2022-08-23T10:21:57.645118Z","shell.execute_reply":"2022-08-23T10:21:57.644145Z","shell.execute_reply.started":"2022-08-23T10:21:57.044058Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Features Groups (Have similar mean or high correlation with each other)","metadata":{}},{"cell_type":"code","source":"feats_groups = []\nfeats_groups.append(['D_111','D_108','D_87','D_136','D_138','D_135','D_137'])\nfeats_groups.append(['D_136','D_138','D_135','D_137'])\nfeats_groups.append(['R_28','R_23','R_18'])\nfeats_groups.append(['R_25','R_22'])\nfeats_groups.append(['R_17','R_4','R_19','R_21','R_15','R_13','R_24'])\nfeats_groups.append(['R_10','R_5','R_6'])\nfeats_groups.append(['B_14','B_13'])\nfeats_groups.append(['B_11','B_42'])\nfeats_groups.append(['B_11','B_25'])\nfeats_groups.append(['B_37','B_1'])\nfeats_groups.append(['B_37','B_1','B_11'])\nfeats_groups.append(['D_139','D_143'])\nfeats_groups.append(['D_43','D_69'])\nfeats_groups.append(['D_102','B_9','D_62'])\nfeats_groups.append(['D_56','B_40'])\nfeats_groups.append(['S_3','S_7'])\nfeats_groups.append(['S_12','S_6'])\nfeats_groups.append(['D_115','D_119','D_118'])\nfeats_groups.append(['D_47','D_129','D_49'])\nfeats_groups.append(['P_3','P_2'])\nfeats_groups.append(['D_63','D_75'])\nfeats_groups.append(['S_8','S_13'])\nfeats_groups.append(['B_4','B_19'])\nfeats_groups.append(['B_2','B_18'])\nfeats_groups.append(['B_2','B_33'])\nfeats_groups.append(['D_75','D_58','D_74'])\nfeats_groups.append(['D_48','P_2','D_55','D_61'])\nfeats_groups.append(['D_75','D_55','D_58'])\nfeats_groups.append(['D_48','P_2'])\nfeats_groups.append(['D_48','D_61','D_55'])\nfeats_groups.append(['D_48','D_58'])\nfeats_groups.append(['B_9','D_75'])\nfeats_groups.append(['D_69','D_65'])\nfeats_groups.append(['D_106','D_39'])\nfeats_groups.append(['B_10','B_16'])","metadata":{"execution":{"iopub.execute_input":"2022-08-23T10:21:57.647080Z","iopub.status.busy":"2022-08-23T10:21:57.646507Z","iopub.status.idle":"2022-08-23T10:21:57.659777Z","shell.execute_reply":"2022-08-23T10:21:57.658602Z","shell.execute_reply.started":"2022-08-23T10:21:57.647040Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i,k in enumerate(feats_groups):\n    \n    train_df[f'Feats_Group{i}_Mean'] = train_df[k].mean(axis=1)\n    train_df_new[f'Feats_Group{i}_Mean_mean'] = train_df_new['customer_ID'].map(train_df.groupby('customer_ID')[f'Feats_Group{i}_Mean'].mean())\n    train_df_new[f'Feats_Group{i}_Mean_last'] = train_df_new['customer_ID'].map(train_df.groupby('customer_ID')[f'Feats_Group{i}_Mean'].last())\n    \n    train_df[f'Feats_Group{i}_Sum'] = train_df[k].sum(axis=1)\n    train_df_new[f'Feats_Group{i}_Sum_mean'] = train_df_new['customer_ID'].map(train_df.groupby('customer_ID')[f'Feats_Group{i}_Sum'].mean())\n    train_df_new[f'Feats_Group{i}_Sum_last'] = train_df_new['customer_ID'].map(train_df.groupby('customer_ID')[f'Feats_Group{i}_Sum'].last())\n    \n    train_df.drop([f'Feats_Group{i}_Mean',f'Feats_Group{i}_Sum'],inplace=True,axis=1)\n    train_df_new = reduce_memory_usage(train_df_new)","metadata":{"execution":{"iopub.execute_input":"2022-08-23T10:21:57.661747Z","iopub.status.busy":"2022-08-23T10:21:57.661176Z","iopub.status.idle":"2022-08-23T10:22:22.340430Z","shell.execute_reply":"2022-08-23T10:22:22.339447Z","shell.execute_reply.started":"2022-08-23T10:21:57.661710Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Add Some Interactions Between D_106 and D_39","metadata":{}},{"cell_type":"code","source":"train_df['D_106_D_39_sum'] = train_df['D_106']+ train_df['D_39']\ntrain_df['D_106_D_39_mean'] = train_df['D_106']+ train_df['D_39']\ntrain_df['D_106_D_39_ratio'] = train_df['D_106'] / train_df['D_39']","metadata":{"execution":{"iopub.execute_input":"2022-08-23T10:22:22.342094Z","iopub.status.busy":"2022-08-23T10:22:22.341719Z","iopub.status.idle":"2022-08-23T10:22:22.350338Z","shell.execute_reply":"2022-08-23T10:22:22.349408Z","shell.execute_reply.started":"2022-08-23T10:22:22.342058Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Target Encoding","metadata":{}},{"cell_type":"code","source":"#Target Encoding For Training Data\nclass KFoldTargetEncoderTrain(base.BaseEstimator, base.TransformerMixin):\n\n    def __init__(self,colnames,targetName,n_fold=5,verbosity=True,discardOriginal_col=False):\n\n        self.colnames = colnames\n        self.targetName = targetName\n        self.n_fold = n_fold\n        self.verbosity = verbosity\n        self.discardOriginal_col = discardOriginal_col\n\n\n    def fit(self, X, y=None):\n        return self\n\n\n    def transform(self,X,test_df):\n\n        assert(type(self.targetName) == str)\n        assert(type(self.colnames) == str)\n        assert(self.colnames in X.columns)\n        assert(self.targetName in X.columns)\n\n        mean_of_target = X[self.targetName].mean()\n        kf = KFold(n_splits = self.n_fold)\n\n        col_mean_name = self.colnames + '_' + 'Kfold_Target_Enc'\n        X[col_mean_name] = np.nan\n\n        for tr_ind, val_ind in kf.split(X,X[self.targetName]):\n            X_tr, X_val = X.iloc[tr_ind], X.iloc[val_ind]\n            X[col_mean_name][val_ind] = X_val[self.colnames].fillna(-127).map(X_tr.groupby(self.colnames)[self.targetName].mean()).to_array()\n\n        X[col_mean_name].fillna(mean_of_target, inplace = True)\n        \n        if self.verbosity:\n\n            encoded_feature = X[col_mean_name].values\n            print('Correlation between the new feature, {} and, {} is {}.'.format(col_mean_name,\n                                                                                      self.targetName,\n                                                                                      np.corrcoef(X[self.targetName].values, encoded_feature)[0][1]))\n        if self.discardOriginal_col:\n            X = X.drop(self.targetName, axis=1)\n            \n\n        return X, test_df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add the training labels to add the target encoded features then drop the target\nlabels = cudf.read_csv('../input/amex-default-prediction/train_labels.csv')\nlabels['customer_ID'] = labels['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\ntrain_df['Target'] = train_df['customer_ID'].map(labels.groupby('customer_ID')['target'].last()).astype(np.int8)\ntrain_df = reduce_memory_usage(train_df)\ndel labels","metadata":{"execution":{"iopub.execute_input":"2022-08-23T10:22:22.352286Z","iopub.status.busy":"2022-08-23T10:22:22.351937Z","iopub.status.idle":"2022-08-23T10:22:24.023456Z","shell.execute_reply":"2022-08-23T10:22:24.022516Z","shell.execute_reply.started":"2022-08-23T10:22:22.352252Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_feats = [\"B_30\",\"B_38\",\"D_114\",\"D_116\",\"D_117\",\"D_120\",\"D_126\",\"D_63\",\"D_64\",\"D_66\",\"D_68\",'D_39_bins','P_2_bins']\nfor feat in tqdm(cat_feats):\n    tar_enc = KFoldTargetEncoderTrain(feat,'Target',n_fold=5)\n    tar_enc.fit(train_df)\n    train_df, _ = tar_enc.transform(train_df,'')\n    \ntrain_df.drop('Target',inplace=True,axis=1)   #Drop the target (will be added again after making all the features)\ntrain_df = reduce_memory_usage(train_df)","metadata":{"execution":{"iopub.execute_input":"2022-08-23T10:22:24.025441Z","iopub.status.busy":"2022-08-23T10:22:24.025046Z","iopub.status.idle":"2022-08-23T10:22:54.976437Z","shell.execute_reply":"2022-08-23T10:22:54.975499Z","shell.execute_reply.started":"2022-08-23T10:22:24.025405Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define The Features Lists","metadata":{}},{"cell_type":"code","source":"all_cols = [c for c in list(train_df.columns) if c not in ['customer_ID','S_2']]\ncat_features = [\"B_30\",\"B_38\",\"D_114\",\"D_116\",\"D_117\",\"D_120\",\"D_126\",\"D_63\",\"D_64\",\"D_66\",\"D_68\",\n               'D_39_bins','P_2_bins','B_1_bins']\nnum_features = [col for col in all_cols if col not in cat_features]\ncat_features = [\"B_38\",\"D_114\",\"D_117\",\"D_120\",\"D_63\",\"D_64\",\"D_66\",      # Dropped some useless features\n                'D_39_bins','P_2_bins','B_1_bins']","metadata":{"execution":{"iopub.execute_input":"2022-08-23T10:22:58.408821Z","iopub.status.busy":"2022-08-23T10:22:58.408452Z","iopub.status.idle":"2022-08-23T10:22:58.417612Z","shell.execute_reply":"2022-08-23T10:22:58.416504Z","shell.execute_reply.started":"2022-08-23T10:22:58.408788Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Adding Aggergations for Some Lagged Features","metadata":{}},{"cell_type":"code","source":"feats = ['B_2','D_39','B_1','P_2','D_106_D_39_sum','D_106_D_39_mean','D_106_D_39_ratio']\n\nfor Feature in tqdm(feats):\n    for window in [1,2,3]:\n        train_df[f'{Feature}_lag_{window}'] = train_df[Feature].shift(window)\n        \n        train_df_new[f'{Feature}_lag_{window}_Agg_customer_ID_last'] = train_df_new['customer_ID'].map(train_df.groupby('customer_ID')[f'{Feature}_lag_{window}'].last())\n        train_df_new[f'{Feature}_lag_{window}_Agg_customer_ID_current'] = train_df_new['customer_ID'].map(train_df.groupby('customer_ID')[f'{Feature}'].last())\n        train_df_new[f'{Feature}_lag_{window}_Agg_customer_ID_CurrentDiffLast'] = train_df_new[f'{Feature}_lag_{window}_Agg_customer_ID_current'] - train_df_new[f'{Feature}_lag_{window}_Agg_customer_ID_last']\n        train_df_new[f'{Feature}_lag_{window}_Agg_customer_ID_mean'] = train_df_new['customer_ID'].map(train_df.groupby('customer_ID')[f'{Feature}_lag_{window}'].mean())\n        train_df_new[f'{Feature}_lag_{window}_Agg_customer_ID_LastDiffMean'] = train_df_new[f'{Feature}_lag_{window}_Agg_customer_ID_last'] - train_df_new[f'{Feature}_lag_{window}_Agg_customer_ID_mean']\n        \n        train_df_new = reduce_memory_usage(train_df_new)\n        train_df.drop(f'{Feature}_lag_{window}',inplace=True,axis=1)\n        train_df_new.drop([f'{Feature}_lag_{window}_Agg_customer_ID_mean',f'{Feature}_lag_{window}_Agg_customer_ID_current'],inplace=True,axis=1)","metadata":{"execution":{"iopub.execute_input":"2022-08-23T10:23:03.237025Z","iopub.status.busy":"2022-08-23T10:23:03.236644Z","iopub.status.idle":"2022-08-23T10:23:17.405237Z","shell.execute_reply":"2022-08-23T10:23:17.404220Z","shell.execute_reply.started":"2022-08-23T10:23:03.236992Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Adding Aggregations for All The Other Features (We will use the aggregations for training instead of the original values to make the data fit into the memory)","metadata":{}},{"cell_type":"code","source":"#Numerical Features\nfor Feature in tqdm(num_features):\n                train_df_new[f'{Feature}_Agg_customer_ID_mean'] = train_df_new['customer_ID'].map(train_df.groupby('customer_ID')[Feature].mean())\n                train_df_new[f'{Feature}_Agg_customer_ID_std'] = train_df_new['customer_ID'].map(train_df.groupby('customer_ID')[Feature].std())  \n                train_df_new[f'{Feature}_Agg_customer_ID_min'] = train_df_new['customer_ID'].map(train_df.groupby('customer_ID')[Feature].min())  \n                train_df_new[f'{Feature}_Agg_customer_ID_max'] = train_df_new['customer_ID'].map(train_df.groupby('customer_ID')[Feature].max())  \n                train_df_new[f'{Feature}_Agg_customer_ID_first'] = train_df_new['customer_ID'].map(train_df.groupby('customer_ID')[Feature].first())\n                train_df_new[f'{Feature}_Agg_customer_ID_last'] = train_df_new['customer_ID'].map(train_df.groupby('customer_ID')[Feature].last())\n                train_df_new[f'{Feature}_Agg_customer_ID_LastDiffMean'] = train_df_new[f'{Feature}_Agg_customer_ID_last'] - train_df_new[f'{Feature}_Agg_customer_ID_mean']\n                train_df_new[f'{Feature}_Agg_customer_ID_LastDiffFirst'] = train_df_new[f'{Feature}_Agg_customer_ID_last'] - train_df_new[f'{Feature}_Agg_customer_ID_first']\n                \n                train_df.drop(Feature,inplace=True,axis=1)\n                train_df_new = reduce_memory_usage(train_df_new)\n#-------------------------------------------------------------------------------------------------------------------------------------------------\n#Categorical Features\ntrain_df_new['D_39_bins_Agg_customer_ID_nunique'] = train_df_new['customer_ID'].map(train_df.groupby('customer_ID')['D_39_bins'].nunique())  \nfor Feature in tqdm(cat_features):\n                train_df_new[f'{Feature}_Agg_customer_ID_last'] = train_df_new['customer_ID'].map(train_df.groupby('customer_ID')[Feature].last())\n                train_df_new[f'{Feature}_Agg_customer_ID_first'] = train_df_new['customer_ID'].map(train_df.groupby('customer_ID')[Feature].first())\n                \n                train_df.drop(Feature,inplace=True,axis=1)\n                train_df_new = reduce_memory_usage(train_df_new)\n\nprint('Finished!!!')\ndel train_df","metadata":{"execution":{"iopub.execute_input":"2022-08-23T10:23:17.407936Z","iopub.status.busy":"2022-08-23T10:23:17.407303Z","iopub.status.idle":"2022-08-23T10:30:05.207581Z","shell.execute_reply":"2022-08-23T10:30:05.206572Z","shell.execute_reply.started":"2022-08-23T10:23:17.407897Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Add The Target to The Dataset","metadata":{}},{"cell_type":"code","source":"labels = cudf.read_csv('../input/amex-default-prediction/train_labels.csv')\ntrain_df_new = train_df_new.sort_index().reset_index(drop=True)\nlabels['customer_ID'] = labels['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\ntrain_df_new['Target'] = train_df_new['customer_ID'].map(labels.groupby('customer_ID')['target'].last()).astype(np.int8)\ntrain_df_new = reduce_memory_usage(train_df_new)\ndel labels","metadata":{"execution":{"iopub.execute_input":"2022-08-23T10:30:05.211089Z","iopub.status.busy":"2022-08-23T10:30:05.209130Z","iopub.status.idle":"2022-08-23T10:30:09.387390Z","shell.execute_reply":"2022-08-23T10:30:09.386423Z","shell.execute_reply.started":"2022-08-23T10:30:05.211048Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Drop Some Useless Features (They increase the memory usage without any positive impact on the results)","metadata":{}},{"cell_type":"code","source":"# 1\nprint('Training Data Shape Before: ',train_df_new.shape)\nfeat_df = pd.read_csv('../input/features-importance/Features.csv')\nfeats = feat_df[feat_df['importance'] == 0.0].feature.to_array()\ntrain_df_new.drop(feats,inplace=True,axis=1)\nprint('Training Data Shape After: ',train_df_new.shape)\ndel feat_df, feats","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 2\nfeats = ['S_20_Agg_customer_ID_min','R_20_Agg_customer_ID_first','D_139_Agg_customer_ID_mean','D_138_Agg_customer_ID_min',\n         'D_137_Agg_customer_ID_LastDiffFirst','R_24_Agg_customer_ID_LastDiffFirst','D_123_Agg_customer_ID_last',\n         'D_87_Agg_customer_ID_mean','R_17_Agg_customer_ID_first','R_25_Agg_customer_ID_mean','D_86_Agg_customer_ID_LastDiffFirst',\n         'B_33_Agg_customer_ID_max','D_137_Agg_customer_ID_max','D_81_Agg_customer_ID_min','D_108_Agg_customer_ID_last',\n         'S_18_Agg_customer_ID_LastDiffFirst','D_93_Agg_customer_ID_LastDiffFirst','D_96_Agg_customer_ID_LastDiffFirst',\n         'R_28_Agg_customer_ID_LastDiffFirst','R_13_Agg_customer_ID_std','D_139_Agg_customer_ID_first','D_88_Agg_customer_ID_first',\n         'D_123_Agg_customer_ID_min','B_32_Agg_customer_ID_max','B_31_Agg_customer_ID_std','R_24_Agg_customer_ID_last',\n         'R_13_Agg_customer_ID_mean','R_15_Agg_customer_ID_LastDiffFirst','R_2_Agg_customer_ID_first','R_23_Agg_customer_ID_mean',\n         'S_18_Agg_customer_ID_first','R_17_Agg_customer_ID_last','R_17_Agg_customer_ID_max','R_18_Agg_customer_ID_max',\n         'D_89_Agg_customer_ID_min','S_18_Agg_customer_ID_max','R_17_Agg_customer_ID_std','D_89_Agg_customer_ID_max',\n         'R_20_Agg_customer_ID_min','R_8_Agg_customer_ID_max','D_88_Agg_customer_ID_max','B_31_Agg_customer_ID_LastDiffFirst',\n         'D_135_Agg_customer_ID_first','R_15_Agg_customer_ID_first','R_13_Agg_customer_ID_max','D_96_Agg_customer_ID_first',\n         'R_7_Agg_customer_ID_min','R_23_Agg_customer_ID_LastDiffMean','R_8_Agg_customer_ID_first','B_41_Agg_customer_ID_min',\n         'B_31_Agg_customer_ID_mean','D_127_Agg_customer_ID_LastDiffFirst','R_22_Agg_customer_ID_max','R_21_Agg_customer_ID_first',\n         'D_111_Agg_customer_ID_min','D_92_Agg_customer_ID_first','D_109_Agg_customer_ID_LastDiffMean','D_96_Agg_customer_ID_min',\n         'R_10_Agg_customer_ID_min','D_140_Agg_customer_ID_max','R_8_Agg_customer_ID_min','D_135_Agg_customer_ID_min',\n         'B_31_Agg_customer_ID_first','D_137_Agg_customer_ID_min','R_20_Agg_customer_ID_max','D_103_Agg_customer_ID_min',\n         'R_23_Agg_customer_ID_max','S_18_Agg_customer_ID_min','R_13_Agg_customer_ID_first','D_86_Agg_customer_ID_first',\n         'D_137_Agg_customer_ID_last','D_93_Agg_customer_ID_first','R_25_Agg_customer_ID_first','D_93_Agg_customer_ID_min',\n         'R_18_Agg_customer_ID_mean','R_22_Agg_customer_ID_last','D_137_Agg_customer_ID_first','S_18_Agg_customer_ID_last',\n         'D_93_Agg_customer_ID_max','S_6_Agg_customer_ID_first','R_23_Agg_customer_ID_std','D_92_Agg_customer_ID_max',\n         'D_109_Agg_customer_ID_LastDiffFirst','B_33_Agg_customer_ID_first','D_111_Agg_customer_ID_first','D_108_Agg_customer_ID_first',\n         'R_28_Agg_customer_ID_min','D_109_Agg_customer_ID_last','R_28_Agg_customer_ID_max','D_127_Agg_customer_ID_first',\n         'D_86_Agg_customer_ID_min','R_5_Agg_customer_ID_first','D_94_Agg_customer_ID_first','D_93_Agg_customer_ID_last',\n         'D_89_Agg_customer_ID_first','D_94_Agg_customer_ID_last','R_15_Agg_customer_ID_min','R_18_Agg_customer_ID_LastDiffFirst',\n         'D_108_Agg_customer_ID_min','R_28_Agg_customer_ID_last','R_23_Agg_customer_ID_first','D_87_Agg_customer_ID_first',\n         'R_21_Agg_customer_ID_min','R_23_Agg_customer_ID_LastDiffFirst','R_25_Agg_customer_ID_min','R_17_Agg_customer_ID_mean',\n         'B_31_Agg_customer_ID_max','R_13_Agg_customer_ID_min','D_109_Agg_customer_ID_min','D_109_Agg_customer_ID_max',\n         'D_109_Agg_customer_ID_first','R_4_Agg_customer_ID_min','D_127_Agg_customer_ID_max','D_127_Agg_customer_ID_min',\n         'R_15_Agg_customer_ID_last','R_2_Agg_customer_ID_min','R_22_Agg_customer_ID_min','R_23_Agg_customer_ID_last',\n         'D_92_Agg_customer_ID_last','R_5_Agg_customer_ID_min','D_87_Agg_customer_ID_min','D_87_Agg_customer_ID_max',\n         'D_87_Agg_customer_ID_last','D_87_Agg_customer_ID_LastDiffFirst','D_94_Agg_customer_ID_min','R_24_Agg_customer_ID_min',\n         'R_24_Agg_customer_ID_first','D_94_Agg_customer_ID_LastDiffFirst','D_139_Agg_customer_ID_min','R_4_Agg_customer_ID_first',\n         'R_17_Agg_customer_ID_min','R_18_Agg_customer_ID_min','R_18_Agg_customer_ID_first','R_18_Agg_customer_ID_last']\n\n#--------------------------------------------------------------------------------------------------------------------------------\n\nprint('Training Data Shape Before: ',train_df_new.shape)\ntrain_df_new.drop(feats,inplace=True,axis=1)\nprint('Training Data Shape After: ',train_df_new.shape)\ntrain_df = train_df_new\ndel feats","metadata":{"execution":{"iopub.execute_input":"2022-08-23T10:30:09.390325Z","iopub.status.busy":"2022-08-23T10:30:09.389963Z","iopub.status.idle":"2022-08-23T10:30:09.403197Z","shell.execute_reply":"2022-08-23T10:30:09.402268Z","shell.execute_reply.started":"2022-08-23T10:30:09.390290Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation","metadata":{}},{"cell_type":"code","source":"# Use the aggregated categorical features instead of the original ones\ncat_feats = [\"B_38\",\"D_114\",\"D_117\",\"D_120\",\"D_63\",\"D_64\",\"D_66\",'D_39_bins','P_2_bins','B_1_bins']\ncat_feats0 = [f'{Feature}_Agg_customer_ID_last' for Feature in cat_feats]\ncat_feats1 = [f'{Feature}_Agg_customer_ID_first' for Feature in cat_feats]\ncat_feats = cat_feats0 + cat_feats1\ndel cat_feats0, cat_feats1","metadata":{"execution":{"iopub.execute_input":"2022-08-23T11:49:40.780091Z","iopub.status.busy":"2022-08-23T11:49:40.779548Z","iopub.status.idle":"2022-08-23T11:49:40.786225Z","shell.execute_reply":"2022-08-23T11:49:40.785074Z","shell.execute_reply.started":"2022-08-23T11:49:40.780051Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"importances = []\noof = []\nTRAIN_SUBSAMPLE = 1.0  #Reduce if the notebook crashed due to low memory space\nFOLDS = 5\ngc.collect()\n\nskf = KFold(n_splits=FOLDS, shuffle=True, random_state=42)\nfor fold,(train_idx, valid_idx) in enumerate(skf.split( train_df, train_df.Target )):\n    \n    # Train with subsample of train fold data\n    if TRAIN_SUBSAMPLE<1.0:\n        np.random.seed(42)\n        train_idx = np.random.choice(train_idx,int(len(train_idx)*TRAIN_SUBSAMPLE), replace=False)\n        np.random.seed(None)\n    \n    print('#'*25)\n    print('### Fold',fold+1)\n    print('### Train size',len(train_idx),'Valid size',len(valid_idx))\n    print(f'### Training with {int(TRAIN_SUBSAMPLE*100)}% fold data...')\n    print('#'*25)\n    \n    # TRAIN, VALID, TEST FOR FOLD K\n    X_train = train_df.loc[train_idx, train_df.drop('Target',axis=1).columns]\n    y_train = train_df.loc[train_idx, 'Target']\n    X_valid = train_df.loc[valid_idx, train_df.drop('Target',axis=1).columns]\n    y_valid = train_df.loc[valid_idx, 'Target']\n\n    #Define the model with early stopping\n    cb_params = {'loss_function': 'Logloss','iterations': 50000,\n                 'task_type':'GPU', 'learning_rate': 0.01, 'depth': 4,\n                 'verbose': 0, 'od_type': 'Iter', 'od_wait': 500}\n    model = CatBoostClassifier(**cb_params, random_state=42)\n    model.fit(X_train.to_pandas(), y_train.to_pandas(), eval_set=[(X_valid.to_pandas(), y_valid.to_pandas())], verbose=100)\n    model.save_model(f'CB_fold{fold}')\n    del X_train, y_train\n    gc.collect()\n    \n    # Infer OOF fold K \n    oof_preds = cudf.DataFrame(model.predict_proba(X_valid.to_pandas()))\n    acc = amex_metric_mod(y_valid.to_pandas().values, oof_preds[1].to_pandas().values)\n    print('Kaggle Metric =',acc[0],'\\n')    \n    del X_valid, y_valid\n    gc.collect()\n    \n    #Store the features importance of fold K\n    df = cudf.DataFrame({'feature':model.feature_names_,f'importance_{fold}':model.feature_importances_})\n    importances.append(df)\n    del df, model\n    gc.collect()\n    \n    # Save OOF to calculate the overall score\n    df = train_df.loc[valid_idx, ['customer_ID','Target'] ].copy()\n    df['oof_pred'] = oof_preds[1].to_array()\n    oof.append(df)\n    del df\n    gc.collect()\n    \n#--------------------------------------------------------------------------------------------------------------------------------\nprint('#'*25)\noof = cudf.concat(oof,axis=0,ignore_index=True).set_index('customer_ID')\nacc = amex_metric_mod(oof.Target.values, oof.oof_pred.values)\nprint('OVERALL CV Kaggle Metric =',acc[0],'\\n')\n\n#CV Results, 100% Training Sample, about 75 mins required to finish with GPU:\n\n# Fold 1:\n# Kaggle Metric = 0.7974942621446841  \n# Fold 2:\n# Kaggle Metric = 0.796183175665393\n# Fold 3:\n# Kaggle Metric = 0.792862926329444 \n# Fold 4:\n# Kaggle Metric = 0.7902024309752875  \n# Fold 5:\n# Kaggle Metric = 0.7990614504769618 \n\n# OVERALL CV Kaggle Metric = 0.7952669259876113","metadata":{"execution":{"iopub.execute_input":"2022-08-23T11:49:43.567470Z","iopub.status.busy":"2022-08-23T11:49:43.567060Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Calculate The Features Importance","metadata":{}},{"cell_type":"code","source":"df = importances[0].copy()\nfor k in range(1,FOLDS): df = df.merge(importances[k], on='feature', how='left')\ndf['importance'] = df.iloc[:,1:].mean(axis=1)\ndf = df.sort_values('importance',ascending=False)\ndf['Index'] = range(0,len(df.index))\nNUM_FEATURES = 30\nShowImp(df.feature,df.importance,NUM_FEATURES)\ndf.to_csv('Features_Importance.csv')\ndel df","metadata":{"execution":{"iopub.execute_input":"2022-08-22T03:24:29.640856Z","iopub.status.busy":"2022-08-22T03:24:29.640159Z","iopub.status.idle":"2022-08-22T03:24:29.697123Z","shell.execute_reply":"2022-08-22T03:24:29.696113Z","shell.execute_reply.started":"2022-08-22T03:24:29.640807Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# --------------------------------------------------------------------------------------------------------------","metadata":{}},{"cell_type":"markdown","source":"# Test Data Preparation","metadata":{}},{"cell_type":"code","source":"print('Reading test data...')\nTEST_PATH = '../input/amex-data-integer-dtypes-parquet-format/test.parquet'\ntest_df = read_file(path = TEST_PATH)","metadata":{"execution":{"iopub.execute_input":"2022-08-22T04:01:28.805871Z","iopub.status.busy":"2022-08-22T04:01:28.805489Z","iopub.status.idle":"2022-08-22T04:02:13.771414Z","shell.execute_reply":"2022-08-22T04:02:13.770313Z","shell.execute_reply.started":"2022-08-22T04:01:28.805838Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Temporary Dataset","metadata":{}},{"cell_type":"code","source":"test_df_new = cudf.DataFrame(test_df['customer_ID'].drop_duplicates(),columns=['customer_ID'])","metadata":{"execution":{"iopub.execute_input":"2022-08-22T04:02:13.773972Z","iopub.status.busy":"2022-08-22T04:02:13.773423Z","iopub.status.idle":"2022-08-22T04:02:13.801166Z","shell.execute_reply":"2022-08-22T04:02:13.800293Z","shell.execute_reply.started":"2022-08-22T04:02:13.773932Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Polar Coordinates of P_2 and The Other Features","metadata":{}},{"cell_type":"code","source":"def Polar(X,y, a = 9.9, b = 9.9): # a and b represnt the center\n    r = np.sqrt((X-a)**2 + (y-b)**2)\n    phi = np.arctan2((y-a), (X-b))\n    return r, phi\n\nfeats = ['B_18','B_2','B_33','D_62','D_77','D_47','P_3','D_45','D_51','R_27','S_25','D_112',\n         'D_121','D_128','D_52','D_115','D_114','D_127','D_68','D_118','D_119','D_122','D_54',\n         'D_129','D_134','S_8','D_92','R_12','D_91','D_76','D_56','D_117','B_42','S_6','D_73',\n         'S_13','D_142','D_126','D_140','R_16','B_41','R_21','B_32','B_24','D_133','R_17','D_120',\n         'D_46','D_145','D_139','D_143','D_138','D_136','R_6','D_135','B_25','D_137','R_26','D_141',\n         'R_13','D_79','D_39','D_89','D_113','R_20','R_15','S_15','R_8','D_130','D_131','D_64','D_59',\n         'R_24','R_5','B_28','D_88','R_10','P_4','D_78','D_43','R_3','R_9','D_41','D_70','R_4','D_81',\n         'D_84','S_3','B_17','D_72','B_11','B_30','S_7','B_37','B_1','B_22','B_8','R_2','B_19','D_53',\n         'B_4','D_61','B_38','B_3','B_20','R_1','D_42','B_16','B_23','B_7','D_74','D_44','D_75','D_58',\n         'B_9','D_55','D_48']\n\nfor feat in tqdm(feats):\n    test_df[f'P_2_{feat}_R'], test_df[f'P_2_{feat}_Phi'] = Polar(test_df[\"P_2\"].fillna(127).to_array(),test_df[feat].fillna(127).to_array())\n    \n    test_df_new[f'P_2_{feat}_R_mean'] = test_df_new['customer_ID'].map(test_df.groupby('customer_ID')[f'P_2_{feat}_R'].mean())\n    test_df_new[f'P_2_{feat}_R_last'] = test_df_new['customer_ID'].map(test_df.groupby('customer_ID')[f'P_2_{feat}_R'].last())\n    test_df_new[f'P_2_{feat}_Phi_mean'] = test_df_new['customer_ID'].map(test_df.groupby('customer_ID')[f'P_2_{feat}_Phi'].mean())\n    test_df_new[f'P_2_{feat}_Phi_last'] = test_df_new['customer_ID'].map(test_df.groupby('customer_ID')[f'P_2_{feat}_Phi'].last())\n    \n    test_df.drop([f'P_2_{feat}_R',f'P_2_{feat}_Phi'],inplace=True,axis=1)\n    test_df_new = reduce_memory_usage(test_df_new)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Adding Bins for P_2, D_39, B_1","metadata":{}},{"cell_type":"code","source":"TRAIN_PATH = '../input/amex-data-integer-dtypes-parquet-format/train.parquet'   # Use the same boundaries used in training set\ntrain_df = read_file(path = TRAIN_PATH, usecols=['customer_ID','S_2','P_2','B_1','D_39']) # Needed later for Target encoding for test set\nfor feat in ['P_2','B_1']:\n    perc = {}\n    for i, per in enumerate([0.0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0]):\n        percentile[i] = train_df[feat].quantile(per)\n    test_df[f'{feat}_bins'] = pd.cut(test_df[feat],[percentile[x] for x in range(0,11)], labels=False).fillna(-127).astype(int)\n    train_df[f'{feat}_bins'] = pd.cut(train_df[feat],[percentile[x] for x in range(0,11)], labels=False).fillna(-127).astype(int)\n\ntrain_df['D_39_bins'] = pd.cut(train_df['D_39'],[0,31,61,91,121,151,181,np.inf], labels=False).fillna(-127).astype(int)   \ntest_df['D_39_bins'] = pd.cut(test_df['D_39'],[0,31,61,91,121,151,181,np.inf], labels=False).fillna(-127).astype(int)\ntest_df = reduce_memory_usage(test_df)\ndel perc","metadata":{"execution":{"iopub.execute_input":"2022-08-22T04:02:13.803016Z","iopub.status.busy":"2022-08-22T04:02:13.802482Z","iopub.status.idle":"2022-08-22T04:02:16.323997Z","shell.execute_reply":"2022-08-22T04:02:16.323035Z","shell.execute_reply.started":"2022-08-22T04:02:13.802979Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Features Groups (Have similar mean or high correlation with each other)","metadata":{}},{"cell_type":"code","source":"feats_groups = []\nfeats_groups.append(['D_111','D_108','D_87','D_136','D_138','D_135','D_137'])\nfeats_groups.append(['D_136','D_138','D_135','D_137'])\nfeats_groups.append(['R_28','R_23','R_18'])\nfeats_groups.append(['R_25','R_22'])\nfeats_groups.append(['R_17','R_4','R_19','R_21','R_15','R_13','R_24'])\nfeats_groups.append(['R_10','R_5','R_6'])\nfeats_groups.append(['B_14','B_13'])\nfeats_groups.append(['B_11','B_42'])\nfeats_groups.append(['B_11','B_25'])\nfeats_groups.append(['B_37','B_1'])\nfeats_groups.append(['B_37','B_1','B_11'])\nfeats_groups.append(['D_139','D_143'])\nfeats_groups.append(['D_43','D_69'])\nfeats_groups.append(['D_102','B_9','D_62'])\nfeats_groups.append(['D_56','B_40'])\nfeats_groups.append(['S_3','S_7'])\nfeats_groups.append(['S_12','S_6'])\nfeats_groups.append(['D_115','D_119','D_118'])\nfeats_groups.append(['D_47','D_129','D_49'])\nfeats_groups.append(['P_3','P_2'])\nfeats_groups.append(['D_63','D_75'])\nfeats_groups.append(['S_8','S_13'])\nfeats_groups.append(['B_4','B_19'])\nfeats_groups.append(['B_2','B_18'])\nfeats_groups.append(['B_2','B_33'])\nfeats_groups.append(['D_75','D_58','D_74'])\nfeats_groups.append(['D_48','P_2','D_55','D_61'])\nfeats_groups.append(['D_75','D_55','D_58'])\nfeats_groups.append(['D_48','P_2'])\nfeats_groups.append(['D_48','D_61','D_55'])\nfeats_groups.append(['D_48','D_58'])\nfeats_groups.append(['B_9','D_75'])\nfeats_groups.append(['D_69','D_65'])\nfeats_groups.append(['D_106','D_39'])\nfeats_groups.append(['B_10','B_16'])","metadata":{"execution":{"iopub.execute_input":"2022-08-22T04:02:16.326709Z","iopub.status.busy":"2022-08-22T04:02:16.326343Z","iopub.status.idle":"2022-08-22T04:02:16.337523Z","shell.execute_reply":"2022-08-22T04:02:16.336504Z","shell.execute_reply.started":"2022-08-22T04:02:16.326658Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i,k in enumerate(feats_groups):\n    test_df[f'Feats_Group{i}_Mean'] = test_df[k].mean(axis=1)\n    test_df_new[f'Feats_Group{i}_Mean_mean'] = test_df_new['customer_ID'].map(test_df.groupby('customer_ID')[f'Feats_Group{i}_Mean'].mean())\n    test_df_new[f'Feats_Group{i}_Mean_last'] = test_df_new['customer_ID'].map(test_df.groupby('customer_ID')[f'Feats_Group{i}_Mean'].last())\n    test_df[f'Feats_Group{i}_Sum'] = test_df[k].sum(axis=1)\n    test_df_new[f'Feats_Group{i}_Sum_mean'] = test_df_new['customer_ID'].map(test_df.groupby('customer_ID')[f'Feats_Group{i}_Sum'].mean())\n    test_df_new[f'Feats_Group{i}_Sum_last'] = test_df_new['customer_ID'].map(test_df.groupby('customer_ID')[f'Feats_Group{i}_Sum'].last())\n    test_df.drop([f'Feats_Group{i}_Mean',f'Feats_Group{i}_Sum'],inplace=True,axis=1)\n    test_df_new = reduce_memory_usage(test_df_new)","metadata":{"execution":{"iopub.execute_input":"2022-08-22T04:02:16.339329Z","iopub.status.busy":"2022-08-22T04:02:16.338836Z","iopub.status.idle":"2022-08-22T04:02:31.492412Z","shell.execute_reply":"2022-08-22T04:02:31.491465Z","shell.execute_reply.started":"2022-08-22T04:02:16.339290Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Add Some Interactions Between D_106 and D_39","metadata":{}},{"cell_type":"code","source":"test_df['D_106_D_39_sum'] = test_df['D_106'] + test_df['D_39']\ntest_df['D_106_D_39_mean'] = test_df['D_106'] + test_df['D_39']\ntest_df['D_106_D_39_ratio'] = test_df['D_106'] / test_df['D_39']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Target Encoding","metadata":{}},{"cell_type":"code","source":"#Target Encoding for Testing Data\nclass KFoldTargetEncoderTrain(base.BaseEstimator, base.TransformerMixin):\n\n    def __init__(self,colnames,targetName,n_fold=5,verbosity=True,discardOriginal_col=False):\n\n        self.colnames = colnames\n        self.targetName = targetName\n        self.n_fold = n_fold\n        self.verbosity = verbosity\n        self.discardOriginal_col = discardOriginal_col\n\n\n    def fit(self, X, y=None):\n        return self\n\n\n    def transform(self,X,test_df):\n\n        assert(type(self.targetName) == str)\n        assert(type(self.colnames) == str)\n        assert(self.targetName in X.columns)\n\n        mean_of_target = X[self.targetName].mean()\n        kf = KFold(n_splits = self.n_fold)\n\n\n\n        col_mean_name = self.colnames + '_' + 'Kfold_Target_Enc'\n        X[col_mean_name] = np.nan\n\n        for tr_ind, val_ind in kf.split(X,X[self.targetName]):\n            X_tr, X_val = X.iloc[tr_ind], X.iloc[val_ind]\n            X[col_mean_name][val_ind] = X_val[self.colnames].fillna(-127).map(X_tr.groupby(self.colnames)[self.targetName].mean()).to_array()\n\n        X[col_mean_name].fillna(mean_of_target, inplace = True)\n        \n        test_df[col_mean_name] = test_df[self.colnames].fillna(-127).map(X.groupby(self.colnames)[col_mean_name].mean())\n        \n        if self.verbosity:\n\n            encoded_feature = X[col_mean_name].values\n            print('Correlation between the new feature, {} and, {} is {}.'.format(col_mean_name,\n                                                                                      self.targetName,\n                                                                                      np.corrcoef(X[self.targetName].values, encoded_feature)[0][1]))\n        if self.discardOriginal_col:\n            X = X.drop(self.targetName, axis=1)\n            \n\n        return X, test_df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Need Training Data to add Target encoding for Testing data\nTRAIN_PATH = '../input/amex-data-integer-dtypes-parquet-format/train.parquet'\ncat_feats = [\"B_30\",\"B_38\",\"D_114\",\"D_116\",\"D_117\",\"D_120\",\"D_126\",\"D_63\",\"D_64\",\"D_66\",\"D_68\",'customer_ID','S_2']\ntrain_df1 = read_file(path = TRAIN_PATH, usecols = cat_feats)\n# Add Labels\nlabels = pd.read_csv('../input/amex-default-prediction/train_labels.csv')\nlabels['customer_ID'] = labels['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\ntrain_df1['Target'] = train_df1['customer_ID'].map(labels.groupby('customer_ID')['target'].last()).astype(np.int8)\n# Reduce Memory Usage\ntrain_df1 = reduce_memory_usage(train_df1)\ntrain_df1.drop(['customer_ID','S_2'],inplace=True,axis=1)\ntrain_df = pd.concat([train_df,train_df1],axis=1)\ndel labels","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add the Target encoding for testing data\nfeats = [\"B_30\",\"B_38\",\"D_114\",\"D_116\",\"D_117\",\"D_120\",\"D_126\",\"D_63\",\"D_64\",\"D_66\",\"D_68\",'D_39_bins','P_2_bins']\nfor feat in tqdm(feats):\n    tar_enc = KFoldTargetEncoderTrain(feat,'Target',n_fold=5)\n    tar_enc.fit(train_df)\n    _, test_df = tar_enc.transform(train_df,test_df)\ntest_df = reduce_memory_usage(test_df)\ndel train_df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define The Features Lists","metadata":{}},{"cell_type":"code","source":"all_cols = [c for c in list(test_df.columns) if c not in ['customer_ID','S_2']]\ncat_features = [\"B_30\",\"B_38\",\"D_114\",\"D_116\",\"D_117\",\"D_120\",\"D_126\",\"D_63\",\"D_64\",\"D_66\",\"D_68\",\n               'D_39_bins','P_2_bins','B_1_bins']\nnum_features = [col for col in all_cols if col not in cat_features]\ncat_features = [\"B_38\",\"D_114\",\"D_117\",\"D_120\",\"D_63\",\"D_64\",\"D_66\",\n                'D_39_bins','P_2_bins','B_1_bins']","metadata":{"execution":{"iopub.execute_input":"2022-08-22T04:02:37.335351Z","iopub.status.busy":"2022-08-22T04:02:37.334613Z","iopub.status.idle":"2022-08-22T04:02:37.341802Z","shell.execute_reply":"2022-08-22T04:02:37.340737Z","shell.execute_reply.started":"2022-08-22T04:02:37.335314Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Adding Aggergations for Some Lagged Features","metadata":{}},{"cell_type":"code","source":"feats = ['B_2','D_39','B_1','P_2','D_106_D_39_sum','D_106_D_39_mean','D_106_D_39_ratio']\n\nfor Feature in tqdm(feats):\n    for window in [1,2,3]:\n        test_df[f'{Feature}_lag_{window}'] = test_df[Feature].shift(window)\n        \n        test_df_new[f'{Feature}_lag_{window}_Agg_customer_ID_last'] = test_df_new['customer_ID'].map(test_df.groupby('customer_ID')[f'{Feature}_lag_{window}'].last())\n        test_df_new[f'{Feature}_lag_{window}_Agg_customer_ID_current'] = test_df_new['customer_ID'].map(test_df.groupby('customer_ID')[f'{Feature}'].last())\n        test_df_new[f'{Feature}_lag_{window}_Agg_customer_ID_CurrentDiffLast'] = test_df_new[f'{Feature}_lag_{window}_Agg_customer_ID_current'] - test_df_new[f'{Feature}_lag_{window}_Agg_customer_ID_last']\n        test_df_new[f'{Feature}_lag_{window}_Agg_customer_ID_mean'] = test_df_new['customer_ID'].map(test_df.groupby('customer_ID')[f'{Feature}_lag_{window}'].mean())\n        test_df_new[f'{Feature}_lag_{window}_Agg_customer_ID_LastDiffMean'] = test_df_new[f'{Feature}_lag_{window}_Agg_customer_ID_last'] - test_df_new[f'{Feature}_lag_{window}_Agg_customer_ID_mean']\n        \n        test_df_new = reduce_memory_usage(test_df_new)\n        test_df.drop(f'{Feature}_lag_{window}',inplace=True,axis=1)\n        test_df_new.drop([f'{Feature}_lag_{window}_Agg_customer_ID_mean',f'{Feature}_lag_{window}_Agg_customer_ID_current'],inplace=True,axis=1)","metadata":{"execution":{"iopub.execute_input":"2022-08-22T04:06:33.017994Z","iopub.status.busy":"2022-08-22T04:06:33.016846Z","iopub.status.idle":"2022-08-22T04:06:41.499713Z","shell.execute_reply":"2022-08-22T04:06:41.498610Z","shell.execute_reply.started":"2022-08-22T04:06:33.017948Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Adding Aggregations for All The Other Features","metadata":{}},{"cell_type":"code","source":"#Numerical Features\nfor Feature in tqdm(num_features):\n                test_df_new[f'{Feature}_Agg_customer_ID_mean'] = test_df_new['customer_ID'].map(test_df.groupby('customer_ID')[Feature].mean())\n                test_df_new[f'{Feature}_Agg_customer_ID_std'] = test_df_new['customer_ID'].map(test_df.groupby('customer_ID')[Feature].std())  \n                test_df_new[f'{Feature}_Agg_customer_ID_min'] = test_df_new['customer_ID'].map(test_df.groupby('customer_ID')[Feature].min())  \n                test_df_new[f'{Feature}_Agg_customer_ID_max'] = test_df_new['customer_ID'].map(test_df.groupby('customer_ID')[Feature].max())  \n                test_df_new[f'{Feature}_Agg_customer_ID_first'] = test_df_new['customer_ID'].map(test_df.groupby('customer_ID')[Feature].first())\n                test_df_new[f'{Feature}_Agg_customer_ID_last'] = test_df_new['customer_ID'].map(test_df.groupby('customer_ID')[Feature].last())\n                test_df_new[f'{Feature}_Agg_customer_ID_LastDiffMean'] = test_df_new[f'{Feature}_Agg_customer_ID_last'] - test_df_new[f'{Feature}_Agg_customer_ID_mean']\n                test_df_new[f'{Feature}_Agg_customer_ID_LastDiffFirst'] = test_df_new[f'{Feature}_Agg_customer_ID_last'] - test_df_new[f'{Feature}_Agg_customer_ID_first']\n                \n                test_df.drop(Feature,inplace=True,axis=1)\n                test_df_new = reduce_memory_usage(test_df_new)\n#-------------------------------------------------------------------------------------------------------------------------------------------------\n#Categorical Features\ntest_df_new['D_39_bins_Agg_customer_ID_nunique'] = test_df_new['customer_ID'].map(test_df.groupby('customer_ID')['D_39_bins'].nunique())  \nfor Feature in tqdm(cat_features):\n                test_df_new[f'{Feature}_Agg_customer_ID_last'] = test_df_new['customer_ID'].map(test_df.groupby('customer_ID')[Feature].last())\n                test_df_new[f'{Feature}_Agg_customer_ID_first'] = test_df_new['customer_ID'].map(test_df.groupby('customer_ID')[Feature].first())\n                \n                test_df.drop(Feature,inplace=True,axis=1)\n                test_df_new = reduce_memory_usage(test_df_new)\n\nprint('Finished!!!')\ndel test_df","metadata":{"execution":{"iopub.execute_input":"2022-08-22T04:06:58.845224Z","iopub.status.busy":"2022-08-22T04:06:58.844823Z","iopub.status.idle":"2022-08-22T04:12:29.889620Z","shell.execute_reply":"2022-08-22T04:12:29.888668Z","shell.execute_reply.started":"2022-08-22T04:06:58.845193Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Drop Some Useless Features","metadata":{}},{"cell_type":"code","source":"# 1\nprint('Testing Data Shape Before: ',test_df_new.shape)\nfeat_df = pd.read_csv('../input/features-importance/Features.csv')\nfeats = feat_df[feat_df['importance'] == 0.0].feature.to_array()\ntest_df_new.drop(feats,inplace=True,axis=1)\nprint('Testing Data Shape After: ',test_df_new.shape)\ndel feat_df, feats","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 2\nfeats = ['S_20_Agg_customer_ID_min','R_20_Agg_customer_ID_first','D_139_Agg_customer_ID_mean','D_138_Agg_customer_ID_min',\n         'D_137_Agg_customer_ID_LastDiffFirst','R_24_Agg_customer_ID_LastDiffFirst','D_123_Agg_customer_ID_last',\n         'D_87_Agg_customer_ID_mean','R_17_Agg_customer_ID_first','R_25_Agg_customer_ID_mean','D_86_Agg_customer_ID_LastDiffFirst',\n         'B_33_Agg_customer_ID_max','D_137_Agg_customer_ID_max','D_81_Agg_customer_ID_min','D_108_Agg_customer_ID_last',\n         'S_18_Agg_customer_ID_LastDiffFirst','D_93_Agg_customer_ID_LastDiffFirst','D_96_Agg_customer_ID_LastDiffFirst',\n         'R_28_Agg_customer_ID_LastDiffFirst','R_13_Agg_customer_ID_std','D_139_Agg_customer_ID_first','D_88_Agg_customer_ID_first',\n         'D_123_Agg_customer_ID_min','B_32_Agg_customer_ID_max','B_31_Agg_customer_ID_std','R_24_Agg_customer_ID_last',\n         'R_13_Agg_customer_ID_mean','R_15_Agg_customer_ID_LastDiffFirst','R_2_Agg_customer_ID_first','R_23_Agg_customer_ID_mean',\n         'S_18_Agg_customer_ID_first','R_17_Agg_customer_ID_last','R_17_Agg_customer_ID_max','R_18_Agg_customer_ID_max',\n         'D_89_Agg_customer_ID_min','S_18_Agg_customer_ID_max','R_17_Agg_customer_ID_std','D_89_Agg_customer_ID_max',\n         'R_20_Agg_customer_ID_min','R_8_Agg_customer_ID_max','D_88_Agg_customer_ID_max','B_31_Agg_customer_ID_LastDiffFirst',\n         'D_135_Agg_customer_ID_first','R_15_Agg_customer_ID_first','R_13_Agg_customer_ID_max','D_96_Agg_customer_ID_first',\n         'R_7_Agg_customer_ID_min','R_23_Agg_customer_ID_LastDiffMean','R_8_Agg_customer_ID_first','B_41_Agg_customer_ID_min',\n         'B_31_Agg_customer_ID_mean','D_127_Agg_customer_ID_LastDiffFirst','R_22_Agg_customer_ID_max','R_21_Agg_customer_ID_first',\n         'D_111_Agg_customer_ID_min','D_92_Agg_customer_ID_first','D_109_Agg_customer_ID_LastDiffMean','D_96_Agg_customer_ID_min',\n         'R_10_Agg_customer_ID_min','D_140_Agg_customer_ID_max','R_8_Agg_customer_ID_min','D_135_Agg_customer_ID_min',\n         'B_31_Agg_customer_ID_first','D_137_Agg_customer_ID_min','R_20_Agg_customer_ID_max','D_103_Agg_customer_ID_min',\n         'R_23_Agg_customer_ID_max','S_18_Agg_customer_ID_min','R_13_Agg_customer_ID_first','D_86_Agg_customer_ID_first',\n         'D_137_Agg_customer_ID_last','D_93_Agg_customer_ID_first','R_25_Agg_customer_ID_first','D_93_Agg_customer_ID_min',\n         'R_18_Agg_customer_ID_mean','R_22_Agg_customer_ID_last','D_137_Agg_customer_ID_first','S_18_Agg_customer_ID_last',\n         'D_93_Agg_customer_ID_max','S_6_Agg_customer_ID_first','R_23_Agg_customer_ID_std','D_92_Agg_customer_ID_max',\n         'D_109_Agg_customer_ID_LastDiffFirst','B_33_Agg_customer_ID_first','D_111_Agg_customer_ID_first','D_108_Agg_customer_ID_first',\n         'R_28_Agg_customer_ID_min','D_109_Agg_customer_ID_last','R_28_Agg_customer_ID_max','D_127_Agg_customer_ID_first',\n         'D_86_Agg_customer_ID_min','R_5_Agg_customer_ID_first','D_94_Agg_customer_ID_first','D_93_Agg_customer_ID_last',\n         'D_89_Agg_customer_ID_first','D_94_Agg_customer_ID_last','R_15_Agg_customer_ID_min','R_18_Agg_customer_ID_LastDiffFirst',\n         'D_108_Agg_customer_ID_min','R_28_Agg_customer_ID_last','R_23_Agg_customer_ID_first','D_87_Agg_customer_ID_first',\n         'R_21_Agg_customer_ID_min','R_23_Agg_customer_ID_LastDiffFirst','R_25_Agg_customer_ID_min','R_17_Agg_customer_ID_mean',\n         'B_31_Agg_customer_ID_max','R_13_Agg_customer_ID_min','D_109_Agg_customer_ID_min','D_109_Agg_customer_ID_max',\n         'D_109_Agg_customer_ID_first','R_4_Agg_customer_ID_min','D_127_Agg_customer_ID_max','D_127_Agg_customer_ID_min',\n         'R_15_Agg_customer_ID_last','R_2_Agg_customer_ID_min','R_22_Agg_customer_ID_min','R_23_Agg_customer_ID_last',\n         'D_92_Agg_customer_ID_last','R_5_Agg_customer_ID_min','D_87_Agg_customer_ID_min','D_87_Agg_customer_ID_max',\n         'D_87_Agg_customer_ID_last','D_87_Agg_customer_ID_LastDiffFirst','D_94_Agg_customer_ID_min','R_24_Agg_customer_ID_min',\n         'R_24_Agg_customer_ID_first','D_94_Agg_customer_ID_LastDiffFirst','D_139_Agg_customer_ID_min','R_4_Agg_customer_ID_first',\n         'R_17_Agg_customer_ID_min','R_18_Agg_customer_ID_min','R_18_Agg_customer_ID_first','R_18_Agg_customer_ID_last']\n\n#--------------------------------------------------------------------------------------------------------------------------------\n\nprint('Testing Data Shape Before: ',test_df_new.shape)\ntest_df_new.drop(feats,inplace=True,axis=1)\nprint('Testing Data Shape After: ',test_df_new.shape)\ntest_df = test_df_new\ndel feats","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"customers = test_df[['customer_ID']].drop_duplicates().sort_index().to_array().flatten()\nFOLDS = 5\ngc.collect()","metadata":{"execution":{"iopub.execute_input":"2022-08-22T04:27:03.245401Z","iopub.status.busy":"2022-08-22T04:27:03.244488Z","iopub.status.idle":"2022-08-22T04:27:03.252105Z","shell.execute_reply":"2022-08-22T04:27:03.251128Z","shell.execute_reply.started":"2022-08-22T04:27:03.245356Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predicting The Test Set in Two Parts Due to Memory Limits","metadata":{}},{"cell_type":"code","source":"print('Predicting Part 1...')\nmodel = CatBoostClassifier()\nprint(f'Fold: 0')\nmodel.load_model(f'./CB_fold0')\npreds1 = model.predict_proba(test_df[:int(len(test_df.index)/2)].to_pandas())[:,1]\nfor f in range(1,FOLDS):\n    print(f'Fold: {f}')\n    model.load_model(f'./CB_fold{f}')\n    preds1 += model.predict_proba(test_df[:int(len(test_df.index)/2)].to_pandas())[:,1]\npreds1 /= FOLDS\nprint(f'Done Part 1')\n    \n# CLEAN MEMORY\ndel model\ngc.collect()","metadata":{"execution":{"iopub.execute_input":"2022-08-22T04:27:13.785288Z","iopub.status.busy":"2022-08-22T04:27:13.784352Z","iopub.status.idle":"2022-08-22T04:28:41.826467Z","shell.execute_reply":"2022-08-22T04:28:41.825415Z","shell.execute_reply.started":"2022-08-22T04:27:13.785253Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drop the First part of The Test Set\ntest_df = test_df[int(len(test_df.index)/2):]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Predicting Part 2...')\nmodel = CatBoostClassifier()\nprint(f'Fold: 0')\nmodel.load_model(f'./CB_fold0')\npreds2 = model.predict_proba(test_df.to_pandas())[:,1]\nfor f in range(1,FOLDS):\n    print(f'Fold: {f}')\n    model.load_model(f'./CB_fold{f}')\n    preds2 += model.predict_proba(test_df.to_pandas())[:,1]\npreds2 /= FOLDS\nprint(f'Done Part 2')\n    \n# CLEAN MEMORY\ndel model, test_df\ngc.collect()","metadata":{"execution":{"iopub.execute_input":"2022-08-22T04:28:55.245613Z","iopub.status.busy":"2022-08-22T04:28:55.244730Z","iopub.status.idle":"2022-08-22T04:30:42.797798Z","shell.execute_reply":"2022-08-22T04:30:42.796737Z","shell.execute_reply.started":"2022-08-22T04:28:55.245567Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make The Submission File\n","metadata":{}},{"cell_type":"code","source":"test_preds = np.concatenate([preds1,preds2])\nPredictions = pd.DataFrame(index=customers,data={'prediction':test_preds})\nsub = pd.read_csv('../input/amex-default-prediction/sample_submission.csv')[['customer_ID']]\nsub['customer_ID_hash'] = sub['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\nsub = sub.set_index('customer_ID_hash')\nsub = sub.merge(Predictions[['prediction']], left_index=True, right_index=True, how='left')\nsub = sub.reset_index(drop=True)\ndel Predictions\n\n# DISPLAY PREDICTIONS\nsub.to_csv('AMEX_FinalSubmission.csv',index=False)\nprint('Submission file shape is', sub.shape)\nplt.hist(sub['prediction'].to_pandas(),bins=150)\nsub.head()","metadata":{"execution":{"iopub.execute_input":"2022-08-22T04:32:31.973189Z","iopub.status.busy":"2022-08-22T04:32:31.972830Z","iopub.status.idle":"2022-08-22T04:32:32.534011Z","shell.execute_reply":"2022-08-22T04:32:32.532926Z","shell.execute_reply.started":"2022-08-22T04:32:31.973159Z"}},"execution_count":null,"outputs":[]}]}