{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport gc\n\nfrom pyarrow.parquet import ParquetFile\nimport pyarrow as pa\n\n# import xgboost as xgb\n# from sklearn.model_selection import train_test_split, StratifiedKFold #, KFold\n\n#from sklearn.metrics import f1_score, plot_confusion_matrix\n\n#import matplotlib.pyplot as plt\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-14T21:49:28.605455Z","iopub.execute_input":"2022-07-14T21:49:28.606032Z","iopub.status.idle":"2022-07-14T21:49:28.614172Z","shell.execute_reply.started":"2022-07-14T21:49:28.605985Z","shell.execute_reply":"2022-07-14T21:49:28.612948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%time\ndef read_file(path = '', usecols = None):\n##read a small piece of the parquet file\n#     pf = ParquetFile(path) \n#     first_ten_rows = next(pf.iter_batches(batch_size = 10000))\n#     df = pa.Table.from_batches([first_ten_rows]).to_pandas()\n    df = pd.read_parquet(path, columns=usecols)\n    df['customer_ID'] = df['customer_ID'].apply(lambda x: int(x[-16:],16) ).astype('int64')\n    df['S_2'] = pd.to_datetime(df['S_2'])\n    return df\ntrain = read_file('/kaggle/input/amex-data-integer-dtypes-parquet-format/train.parquet')\n#train.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:49:28.622984Z","iopub.execute_input":"2022-07-14T21:49:28.624050Z","iopub.status.idle":"2022-07-14T21:49:30.138715Z","shell.execute_reply.started":"2022-07-14T21:49:28.623972Z","shell.execute_reply":"2022-07-14T21:49:30.137212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def agg_features(df, features, aggs):\n  #  print(aggs)\n    adf = df.groupby(\"customer_ID\")[features].agg(aggs)\n    adf.columns = ['_'.join(x) for x in adf.columns]\n    for col in adf.columns:\n        if adf[col].dtype=='float64': adf[col] = adf[col].astype('float32')\n        if adf[col].dtype=='int64': adf[col] = adf[col].astype('int32')\n        if '_count' in col: adf[col] = adf[col].astype('int8')\n    return adf\n\ndef agg_features_np(df, features, aggs, agg_name):\n  #  print(aggs)\n    adf = df.groupby(\"customer_ID\")[features].agg(aggs)\n    adf = adf.add_suffix(agg_name)\n    for col in adf.columns:\n        if adf[col].dtype=='float64': adf[col] = adf[col].astype('float32')\n        if adf[col].dtype=='int64': adf[col] = adf[col].astype('int32')\n        if '_count' in col: adf[col] = adf[col].astype('int8')\n    return adf\n\ndef diff_lag1(df, features):\n    adf = df.groupby(\"customer_ID\")[features].nth(-1) - df.groupby(\"customer_ID\")[features].nth(-2)\n    adf = adf.add_suffix('_diff1')\n    for col in adf.columns:\n        if adf[col].dtype=='float64': adf[col] = adf[col].astype('float32')\n        if adf[col].dtype=='int64': adf[col] = adf[col].astype('int32')\n    return adf\n\ndef diff_last_mean(df, features):\n    adf = df.groupby(\"customer_ID\")[features].last() - df.groupby(\"customer_ID\")[features].mean()\n    adf = adf.add_suffix('_dlm')\n    for col in adf.columns:\n        if adf[col].dtype=='float64': adf[col] = adf[col].astype('float32')\n        if adf[col].dtype=='int64': adf[col] = adf[col].astype('int32')\n    return adf\n\ndef root_mean_square(x):\n    return np.sqrt(np.mean(np.square(x))) if len(x) > 0 else np.NaN\n\ndef mean_diff(x):\n    return np.mean(x.values)\n    #return np.nanmean(np.diff(x.values))","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:49:30.140984Z","iopub.execute_input":"2022-07-14T21:49:30.141544Z","iopub.status.idle":"2022-07-14T21:49:30.161124Z","shell.execute_reply.started":"2022-07-14T21:49:30.141506Z","shell.execute_reply":"2022-07-14T21:49:30.160176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_and_feature_engineer(df,tt):\n    \n    features_to_drop = ['D_84_diff1', 'B_16_diff1', 'S_20_mean', 'S_19_dlm', 'S_17_max', 'R_8_var', 'R_3_min', 'R_21_mean', 'R_21_last', 'R_15_last', 'R_13_last', 'D_80_last', 'D_72_last', 'D_70_diff1', 'D_62_last', 'D_127_var', 'D_115_diff1', 'B_41_last', 'S_24_min', 'R_4_mean', 'R_25_max', 'R_24_var', 'P_4_min', 'D_96_last', 'D_96_diff1', 'D_81_mean', 'D_79_max', 'D_140_min', 'D_138_diff1', 'D_135_max', 'D_120_last', 'B_8_diff1', 'B_8_count', 'B_31_dlm', 'B_1_nunique', 'S_8_diff1', 'S_24_mean', 'D_125_mean', 'D_114_nunique', 'R_20_mean', 'D_70_dlm', 'D_55_max', 'D_136_max', 'D_125_diff1', 'D_122_dlm', 'D_108_dlm', 'B_36_diff1', 'B_16_nunique', 'D_102_mean', 'B_11_dlm', 'D_135_dlm', 'S_25_var', 'R_3_diff1', 'R_13_var', 'R_12_last', 'P_4_std', 'D_81_last', 'D_136_var', 'D_127_max', 'B_11_diff1', 'S_15_max', 'R_7_max', 'P_2_var', 'D_89_mean', 'D_86_last', 'D_65_dlm', 'D_46_min', 'D_135_var', 'D_117_nunique', 'D_111_max', 'B_22_dlm', 'B_13_nunique', 'S_15_min', 'R_21_var', 'R_19_mean', 'D_135_last', 'D_130_last', 'D_108_max', 'R_12_dlm', 'D_75_diff1', 'D_41_var', 'D_41_min', 'D_126_last', 'D_118_var', 'D_136_dlm', 'D_122_max', 'B_30_last', 'R_2_max', 'R_19_max', 'R_14_last', 'D_143_dlm', 'D_139_last', 'D_138_max', 'R_5_diff1', 'R_14_mean', 'D_125_max', 'S_7_min', 'R_7_dlm', 'R_12_mean', 'D_65_var', 'D_58_var', 'D_51_last', 'D_43_diff1', 'D_145_last', 'D_130_min', 'D_127_mean', 'D_107_max', 'S_9_last', 'S_5_mean', 'S_22_var', 'D_83_max', 'D_80_min', 'S_6_dlm', 'S_22_max', 'R_6_last', 'R_10_last', 'D_96_max', 'D_92_dlm', 'D_113_min', 'D_107_var', 'D_107_min', 'B_7_dlm', 'B_3_count', 'P_4_mean', 'D_39_mean', 'D_128_min', 'D_125_var', 'R_5_var', 'R_14_dlm', 'D_84_mean', 'D_80_var', 'D_77_min', 'D_58_last', 'D_54_var', 'D_47_min', 'D_145_max', 'D_141_mean', 'D_137_var', 'S_7_max', 'D_75_max', 'D_143_var', 'S_11_dlm', 'D_145_mean', 'D_59_max', 'D_144_min', 'D_105_min', 'B_25_last', 'B_19_dlm', 'D_62_diff1', 'S_15_diff1', 'D_136_mean', 'D_106_dlm', 'S_26_dlm', 'S_25_mean', 'S_17_min', 'D_81_dlm', 'D_144_var', 'B_32_dlm', 'S_16_var', 'S_11_var', 'D_61_mean', 'D_82_var', 'D_137_mean', 'D_107_last', 'S_8_mean', 'R_5_max', 'D_121_var', 'B_13_last', 'B_10_dlm', 'B_20_dlm', 'S_3_diff1', 'S_16_dlm', 'D_74_min', 'S_17_mean', 'D_89_dlm', 'D_74_mean', 'D_70_max', 'D_144_mean', 'S_17_var', 'R_8_max', 'R_11_diff1', 'D_74_max', 'D_54_dlm', 'D_139_var', 'B_12_dlm', 'D_79_mean', 'D_72_var', 'D_63_last', 'D_62_max', 'D_143_mean', 'D_112_max', 'S_22_min', 'D_91_last', 'D_71_diff1', 'D_65_last', 'D_139_mean', 'S_8_max', 'S_23_dlm', 'D_72_mean', 'D_70_min', 'D_104_var', 'D_59_min', 'D_49_mean', 'S_26_mean', 'S_17_dlm', 'D_141_last', 'S_20_dlm', 'D_45_min', 'D_45_diff1', 'D_44_var', 'B_8_dlm', 'D_65_mean', 'S_16_max', 'D_79_dlm', 'D_131_diff1', 'R_26_max', 'D_128_last', 'S_13_mean', 'R_7_diff1', 'D_77_mean', 'D_131_max', 'D_112_mean', 'S_26_diff1', 'D_60_dlm', 'D_55_min', 'D_128_dlm', 'D_105_var', 'D_77_var', 'B_26_dlm', 'D_124_dlm', 'D_112_diff1', 'D_105_mean', 'D_84_var', 'S_7_var', 'R_8_diff1', 'R_6_var', 'D_96_dlm', 'D_54_min', 'D_51_var', 'D_124_min', 'B_33_dlm', 'S_19_var', 'R_26_last', 'D_144_last', 'S_27_var', 'D_66_nunique', 'R_1_var', 'D_118_diff1', 'S_7_last', 'S_15_last', 'D_77_last', 'D_122_var', 'R_26_mean', 'D_59_mean', 'D_44_min', 'B_40_diff1', 'S_23_diff1', 'D_74_dlm', 'R_16_dlm', 'S_17_last', 'S_27_min', 'D_119_diff1', 'B_18_diff1', 'D_65_max', 'S_3_max', 'D_62_var', 'R_16_last', 'D_103_var', 'R_5_dlm', 'D_115_mean', 'B_27_last', 'B_15_last', 'S_15_var', 'D_107_mean', 'R_27_min', 'D_78_diff1', 'S_5_max', 'B_24_dlm', 'R_27_mean', 'S_5_min', 'S_3_dlm', 'S_12_max', 'S_25_last', 'R_16_mean', 'B_27_dlm', 'R_27_diff1', 'B_21_diff1', 'D_69_last', 'D_122_min', 'D_75_min', 'B_2_dlm', 'S_6_var', 'S_19_last', 'R_6_dlm', 'D_44_diff1', 'S_25_diff1', 'D_51_max', 'D_46_diff1', 'S_8_dlm', 'D_71_min', 'D_130_mean', 'D_115_var', 'S_27_last', 'B_21_last', 'S_16_last', 'D_69_min', 'D_133_var', 'S_9_dlm', 'R_6_diff1', 'R_3_max', 'S_25_min', 'S_13_last', 'D_52_diff1', 'D_78_dlm', 'B_13_diff1', 'D_104_diff1', 'D_71_dlm', 'D_112_dlm', 'S_23_min', 'D_69_dlm', 'B_8_last', 'B_19_nunique', 'S_16_diff1', 'R_6_max', 'S_25_dlm', 'R_3_var', 'D_71_max', 'D_54_last', 'B_16_last', 'S_27_mean', 'D_120_nunique', 'R_1_max'] #part1\n#                       'D_70_last', 'R_16_max', 'B_8_std', 'B_14_std', 'B_37_mean', 'B_41_std', 'ap_D_39-P_3_min', 'B_8_min', 'B_41_mean', 'D_48_max', 'D_131_var', 'D_135_mean', 'D_137_last', 'D_143_last', 'ap_S_16-P_2_min', 'B_31_last', 'D_128_var', 'P_4_last', 'S_6_max', 'R_9_last', 'S_18_var', 'R_20_max', 'ap_S_23-P_2_var', 'D_60_min', 'ap_S_16-P_3_min', 'B_22_std', 'D_103_mean', 'R_14_max', 'S_23_last', 'B_23_last', 'B_27_std', 'P_3_min', 'R_1_min', 'D_60_max', 'D_39_last', 'R_9_mean', 'D_80_max', 'D_83_last', 'D_86_var', 'B_12_min', 'S_9_max', 'R_15_var', 'D_123_mean', 'D_139_min', 'D_140_var', 'B_17_min', 'R_5_mean', 'D_60_mean', 'D_79_var', 'S_20_max', 'D_131_last', 'B_33_std', 'ap_S_23-P_3_var', 'D_69_mean', 'D_81_max', 'D_96_mean', 'ap_B_11-P_2_var', 'ap_B_11-P_3_max', 'B_10_min', 'B_26_last', 'D_82_min', 'R_26_var', 'D_129_max', 'ap_S_23-P_2_mean', 'D_86_max', 'R_4_last', 'D_84_last', 'ap_S_16-P_2_mean', 'ap_S_16-P_2_var', 'R_7_mean', 'D_58_max', 'D_79_last', 'D_82_max', 'D_78_var', 'R_10_max', 'D_129_last', 'D_130_max', 'D_58_mean', 'S_24_max', 'D_141_max', 'B_32_mean', 'ap_S_16-P_3_last', 'R_22_var', 'D_92_max', 'ap_B_17-P_2_min', 'D_43_mean', 'D_48_min', 'B_8_mean', 'B_9_std', 'D_104_min', 'D_108_var', 'D_113_max', 'D_131_mean', 'D_123_max', 'ap_D_39-P_3_mean', 'S_26_last', 'D_106_last', 'ap_S_16-P_3_max', 'B_16_min', 'R_7_var', 'R_9_min', 'R_11_var', 'D_136_last', 'B_20_mean', 'D_78_max', 'ap_B_14-P_2_mean', 'D_71_mean', 'D_133_last', 'ap_S_23-P_3_last', 'D_123_var', 'D_129_mean', 'ap_B_11-P_2_min', 'ap_D_131-P_2_min', 'B_7_std', 'S_22_mean', 'B_21_std', 'B_17_std', 'D_106_mean', 'B_15_mean', 'B_15_std', 'D_52_last', 'S_9_var', 'ap_S_16-P_2_max', 'B_11_min', 'ap_S_23-P_3_max', 'B_20_std', 'D_102_last', 'D_119_min', 'B_13_min', 'B_15_max', 'S_15_mean', 'B_10_std', 'B_27_min', 'ap_B_11-P_3_min', 'D_83_mean', 'D_115_min', 'ap_S_23-P_2_min', 'B_2_max', 'B_21_max', 'B_18_mean', 'D_124_var', 'B_28_min', 'D_102_var', 'ap_B_17-P_3_last', 'B_9_min', 'D_105_last', 'D_49_max', 'ap_D_131-P_3_last', 'B_5_max', 'B_16_std', 'D_56_max', 'D_106_max', 'D_118_min', 'ap_D_131-P_3_var', 'B_2_std', 'D_52_max', 'B_14_min', 'ap_D_131-P_3_min', 'S_11_last', 'S_13_var', 'ap_D_131-P_2_var', 'S_6_mean', 'D_121_mean', 'B_6_max', 'D_52_min', 'B_1_min', 'D_48_var', 'B_19_min', 'D_61_last', 'ap_D_131-P_2_mean', 'S_24_var', 'S_11_min', 'S_19_max', 'D_108_mean', 'D_128_max', 'P_2_mean', 'B_28_max', 'S_7_mean', 'B_25_min', 'B_5_min', 'D_82_last', 'D_60_last', 'D_77_max', 'D_49_var', 'S_16_mean', 'D_55_mean', 'D_122_mean', 'B_18_max', 'R_16_var', 'B_20_last', 'S_3_var', 'B_23_min', 'B_9_mean', 'R_12_min', 'B_18_min', 'ap_B_11-P_3_last', 'B_24_last', 'D_121_max', 'B_28_std', 'B_16_mean', 'B_24_std', 'D_102_min', 'D_81_var', 'B_28_mean', 'B_17_last', 'D_114_last', 'R_11_mean', 'B_18_last', 'ap_B_11-P_2_max'] #part2\n    rms_features = ['D_45', 'D_70', 'D_46', 'S_23', 'D_43', 'ap_D_39-P_2', 'D_121', 'R_1', 'D_44', 'D_124', 'ap_B_14-P_3', 'D_54', 'D_72', 'D_41', 'ap_S_23-P_3', 'ap_S_23-P_2', 'R_27', 'R_6', 'D_51', 'S_24', 'D_74', 'S_22', 'S_3', 'ap_D_131-P_3', 'D_58', 'S_9', 'D_144', 'S_25', 'R_3', 'D_84', 'ap_B_11-P_2', 'S_8', 'S_15', 'R_26', 'D_127', 'S_13', 'S_27', 'D_115', 'D_56', 'D_103', 'R_14', 'D_133', 'D_118', 'D_122', 'D_143', 'D_125', 'D_49', 'D_128', 'D_135', 'D_140', 'D_71', 'S_12', 'D_107', 'R_4', 'D_104', 'D_91', 'D_39', 'R_9', 'D_86', 'R_5', 'D_106', 'R_7', 'D_78', 'D_96', 'D_80']\n    meandiff_features = ['R_3', 'R_27', 'D_65', 'D_72', 'D_43', 'R_1', 'D_61', 'D_118', 'D_69', 'D_144', 'ap_D_131-P_2', 'S_5', 'S_8', 'D_70', 'D_107', 'S_16', 'S_27', 'D_104', 'R_16', 'S_22', 'D_115', 'D_128', 'ap_S_23-P_2', 'S_23', 'D_51', 'D_102', 'D_83', 'D_140', 'S_17', 'D_60', 'ap_D_39-P_2', 'D_127', 'R_10', 'D_84', 'S_25', 'ap_B_14-P_3', 'S_12', 'D_145', 'S_15', 'D_136', 'D_39', 'S_7', 'D_121', 'D_45', 'R_12', 'R_14', 'D_49', 'D_103', 'D_62', 'R_20', 'D_124', 'S_11', 'R_4', 'R_5', 'R_19', 'R_24', 'S_19', 'D_139', 'S_3', 'D_54', 'D_41', 'R_2', 'D_119', 'ap_S_16-P_3', 'D_122', 'D_137', 'R_7', 'R_9', 'R_8', 'D_94', 'R_6']\n    diff1_features = ['S_12', 'D_102', 'S_27', 'S_7', 'B_3', 'B_37', 'ap_D_39-P_3', 'D_41', 'B_26', 'ap_D_39-P_2', 'B_23', 'ap_S_23-P_2', 'D_59', 'D_48', 'D_128', 'ap_B_11-P_3', 'B_19', 'B_12', 'D_54', 'B_9', 'D_69', 'B_27', 'D_55', 'ap_S_16-P_3', 'S_11', 'P_3', 'D_130', 'R_10', 'P_4', 'D_113', 'B_22', 'B_2', 'B_7', 'R_2', 'D_65', 'R_13', 'D_106', 'D_39', 'S_13', 'D_80']\n    dlm_features = ['B_4', 'B_3', 'B_15', 'D_55', 'B_16', 'B_37', 'D_49', 'D_102', 'D_47', 'R_10', 'S_13', 'ap_S_23-P_2', 'R_3', 'D_52', 'D_48', 'ap_D_131-P_2', 'R_8', 'D_41', 'ap_B_11-P_2', 'D_77', 'ap_D_39-P_3', 'D_133', 'R_2', 'B_6', 'B_21', 'D_107', 'S_5', 'B_36', 'D_61', 'B_28', 'R_27', 'S_18', 'D_86', 'D_59', 'D_129', 'D_56', 'R_4', 'D_119', 'D_62', 'D_44', 'D_84', 'ap_B_14-P_2', 'D_125', 'D_144', 'ap_S_16-P_3', 'D_91', 'ap_D_131-P_3', 'B_40', 'D_140', 'R_19', 'R_13', 'R_20', 'D_127', 'D_83', 'D_43', 'D_72', 'B_23', 'R_9', 'B_18', 'ap_S_23-P_3', 'D_113', 'R_26', 'ap_B_17-P_3', 'ap_S_16-P_2', 'R_21', 'D_105', 'D_115', 'B_9', 'B_17', 'D_80']\n    \n    print('starting feature engineering')\n    \n    #dropping columns with lots of NA\n   # df = df.drop(['D_42','D_50', 'D_53', 'D_73', 'D_76', 'D_88', 'D_110', 'D_132', 'D_134', 'D_142', 'B_29', 'B_39', 'B_42'],axis=1)\n    \n    # compute \"after pay\" features\n    for bcol in [f'B_{i}' for i in [11,14,17]]+['D_39','D_131']+[f'S_{i}' for i in [16,23]]:\n        for pcol in ['P_2','P_3']:\n            if bcol in df.columns:\n                df[f'ap_{bcol}-{pcol}'] = df[bcol] - df[pcol]\n                df[f'ap_{bcol}/{pcol}'] = df[bcol] / df[pcol]\n    \n    all_cols = [c for c in list(df.columns) if c not in ['customer_ID','S_2']]\n    \n    cat_features = [\"B_30\",\"B_38\",\"D_114\",\"D_116\",\"D_117\",\"D_120\",\"D_126\",\"D_63\",\"D_64\",\"D_66\",\"D_68\"]\n    b_features = [col for col in all_cols if col.startswith('B_') and col not in cat_features]\n    p_features = [col for col in all_cols if col.startswith('P_') and col not in cat_features]\n    bp_features = b_features + p_features\n    num_features = [col for col in all_cols if col not in cat_features and col not in b_features and col not in p_features]\n    all_num_features = [col for col in all_cols if col not in cat_features] #and not col.startswith('ap_')\n    \n    _ = gc.collect()\n    \n    print('computing classic stats for num_features')\n    test_num_agg1 = agg_features(df,num_features,['last', 'mean', 'min', 'max', 'var']).drop(features_to_drop,axis=1,errors='ignore')\n#    test_num_agg2 = agg_features(df,num_features,['last','var'])#,'median','count'\n    \n    print('computing diff_last_mean for dlm_features')\n    diff_lm = diff_last_mean(df,dlm_features)\n\n    print('computing diff_lag1 for diff1_features')    \n    diff1 = diff_lag1(df,diff1_features)\n    \n    print('computing mean_diff for meandiff_features')\n    test_md_agg = agg_features_np(df, meandiff_features, mean_diff, '_meandiff')\n\n    print('computing root_mean_square for rms_features')\n    test_rms_agg = agg_features_np(df, rms_features, root_mean_square, '_rms')\n\n  #  print('computing root_mean_square for num_features')\n  #  test_rms_agg = agg_features_np(df,num_features, root_mean_square, '_rms')\n    \n    _ = gc.collect()\n    \n    test_b_agg = agg_features(df,b_features,['last', 'mean', 'min', 'max', 'std'])#added , 'mean', 'max', 'std', 'min' ,#'var',  #, 'first'#size #removed , 'nunique', 'count'\n    print('added b_features')\n    test_p_agg = agg_features(df,p_features,['last', 'mean', 'min', 'var', 'std'])# first #,'median','count','var'\n    print('added p_features')\n\n    test_cat_agg = agg_features(df,cat_features,['last', 'nunique'])#, 'count'#, 'first'#size\n    print('added cat_features')\n    \n                \n    df = pd.concat([test_num_agg1, test_cat_agg, diff_lm, diff1, test_md_agg, test_rms_agg, test_b_agg, test_p_agg], axis=1)\n    del test_num_agg1, test_cat_agg, diff_lm, diff1, test_md_agg, test_rms_agg, test_b_agg, test_p_agg\n    _ = gc.collect()\n                \n    print('shape after engineering', df.shape )\n    \n    ##df.drop(df.isna().mean()>0.5,axis=1,inplace=True)\n    df.dropna(thresh=len(df.index)/2, axis=1, inplace=True) #remove cols with more than half of the values missing\n    df = df.fillna(-127)\n    \n    df = df.drop(features_to_drop,axis=1,errors='ignore')\n    print('shape after permutation importance feature selection', df.shape)\n    \n    return df\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:49:30.162643Z","iopub.execute_input":"2022-07-14T21:49:30.163074Z","iopub.status.idle":"2022-07-14T21:49:30.228460Z","shell.execute_reply.started":"2022-07-14T21:49:30.163034Z","shell.execute_reply":"2022-07-14T21:49:30.227528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = process_and_feature_engineer(train,tt='train')\nprint('writing train features to parquet')\ntrain.to_parquet('train_features.parquet');\n#train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:49:30.230587Z","iopub.execute_input":"2022-07-14T21:49:30.231650Z","iopub.status.idle":"2022-07-14T21:49:45.825847Z","shell.execute_reply.started":"2022-07-14T21:49:30.231608Z","shell.execute_reply":"2022-07-14T21:49:45.824820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:49:45.827990Z","iopub.execute_input":"2022-07-14T21:49:45.828356Z","iopub.status.idle":"2022-07-14T21:49:45.832657Z","shell.execute_reply.started":"2022-07-14T21:49:45.828323Z","shell.execute_reply":"2022-07-14T21:49:45.831698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test data","metadata":{}},{"cell_type":"code","source":"# CALCULATE SIZE OF EACH SEPARATE TEST PART\ndef get_rows(customers, test, NUM_PARTS = 4, verbose = ''):\n    chunk = len(customers)//NUM_PARTS\n    if verbose != '':\n        print(f'We will process {verbose} data as {NUM_PARTS} separate parts.')\n        print(f'There will be {chunk} customers in each part (except the last part).')\n        print('Below are number of rows in each part:')\n    rows = []\n\n    for k in range(NUM_PARTS):\n        if k==NUM_PARTS-1: cc = customers[k*chunk:]\n        else: cc = customers[k*chunk:(k+1)*chunk]\n        s = test.loc[test.customer_ID.isin(cc)].shape[0]\n        rows.append(s)\n    if verbose != '': print( rows )\n    return rows,chunk\n\n# COMPUTE SIZE OF N PARTS FOR TEST DATA\nNUM_PARTS = 4\nTEST_PATH = '/kaggle/input/amex-data-integer-dtypes-parquet-format/test.parquet'\n\nprint(f'Reading test data - 2 cols...')\ntest = read_file(path = TEST_PATH, usecols = ['customer_ID','S_2'])\ncustomers = test[['customer_ID']].drop_duplicates().sort_index().values.flatten()\nrows,num_cust = get_rows(customers, test[['customer_ID']], NUM_PARTS = NUM_PARTS, verbose = 'test')\ndel test\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:49:45.833920Z","iopub.execute_input":"2022-07-14T21:49:45.834348Z","iopub.status.idle":"2022-07-14T21:49:48.461254Z","shell.execute_reply.started":"2022-07-14T21:49:45.834313Z","shell.execute_reply":"2022-07-14T21:49:48.460259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skip_rows = 0\nskip_cust = 0\ntest_preds = []\n\ntest_output = pd.DataFrame()\n\nfor k in range(NUM_PARTS):\n    \n     # READ PART OF TEST DATA\n    print(f'\\nReading test data...')\n    test = read_file(path = TEST_PATH)\n    test = test.iloc[skip_rows:skip_rows+rows[k]]\n    skip_rows += rows[k]\n    print(f'=> Test part {k+1} has shape', test.shape )\n    \n    # PROCESS AND FEATURE ENGINEER PART OF TEST DATA\n    test = process_and_feature_engineer(test,tt='test')\n    if k==NUM_PARTS-1: test = test.loc[customers[skip_cust:]]\n    else: test = test.loc[customers[skip_cust:skip_cust+num_cust]]\n    skip_cust += num_cust\n\n    if k==0:\n        test_output = test\n    else:\n        test_output = pd.concat([test_output,test])\n        \n    del test\n    _ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:49:48.462809Z","iopub.execute_input":"2022-07-14T21:49:48.463605Z","iopub.status.idle":"2022-07-14T21:50:16.663625Z","shell.execute_reply.started":"2022-07-14T21:49:48.463558Z","shell.execute_reply":"2022-07-14T21:50:16.662456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('writing test features to parquet')\ntest_output.to_parquet('test_features.parquet');","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:50:16.664918Z","iopub.execute_input":"2022-07-14T21:50:16.665274Z","iopub.status.idle":"2022-07-14T21:50:16.824508Z","shell.execute_reply.started":"2022-07-14T21:50:16.665245Z","shell.execute_reply":"2022-07-14T21:50:16.823218Z"},"trusted":true},"execution_count":null,"outputs":[]}]}