{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Load Libraries","metadata":{}},{"cell_type":"code","source":"!pip install -U \"setuptools<58\" \n!pip install xfeat  \n\nimport xfeat","metadata":{"execution":{"iopub.status.busy":"2022-07-16T07:24:42.432269Z","iopub.execute_input":"2022-07-16T07:24:42.432704Z","iopub.status.idle":"2022-07-16T07:25:21.547514Z","shell.execute_reply.started":"2022-07-16T07:24:42.432620Z","shell.execute_reply":"2022-07-16T07:25:21.546492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n\npd.set_option('display.max_rows', 100000)\npd.set_option('display.max_columns', 100000)\nno_need = importance[importance['importance'] < 10].index.values\n\n\nimportance_df = pd.DataFrame({\"gain\":model.feature_importances_}, index = features).sort_values(\"gain\", ascending=False)\nimportance_list += [importance_df]\nprint(\"[Importance]\")\ndisplay(importance_df)\n\nresult = permutation_importance(model, X_tr, cupy.asnumpy(y_tr), n_repeats=10, n_jobs=-1, random_state=71)\nperm_imp_df = pd.DataFrame({\"importances_mean\":result[\"importances_mean\"], \"importances_std\":result[\"importances_std\"]}, index= features)\nperm_imp_list += [perm_imp_df]\nprint(\"[Permutation feature Importance]\")\ndisplay(perm_imp_df)\nperm_imp_df.sort_values(\"importances_mean\", ascending=False).importances_mean.plot.barh()\nplt.show()\n\n\n\nno_need = ['D_39_min', 'R_2_min', 'R_2_quantile', 'D_54_max', 'R_4_min',\n       'R_4_max', 'R_4_quantile', 'R_5_min', 'R_5_quantile', 'D_65_min',\n       'D_73_std', 'D_73_last', 'R_7_min', 'R_7_quantile', 'D_78_min',\n       'D_79_min', 'R_8_min', 'R_8_quantile', 'D_80_min', 'R_10_min',\n       'R_10_quantile', 'R_11_min', 'D_81_min', 'D_81_max', 'D_81_last',\n       'D_81_quantile', 'R_12_max', 'R_12_quantile', 'R_13_std',\n       'R_13_min', 'R_13_max', 'R_13_last', 'R_13_quantile', 'D_83_min',\n       'D_83_quantile', 'R_14_min', 'R_14_quantile', 'R_15_min',\n       'R_15_last', 'R_15_quantile', 'D_84_min', 'D_84_quantile',\n       'R_16_min', 'R_16_quantile', 'S_18_std', 'S_18_min', 'S_18_last',\n       'D_86_std', 'D_86_min', 'D_86_quantile', 'D_87_mean', 'D_87_std',\n       'D_87_min', 'D_87_max', 'D_87_last', 'D_87_quantile', 'R_17_mean',\n       'R_17_min', 'R_17_max', 'R_17_last', 'R_17_quantile', 'R_18_mean',\n       'R_18_std', 'R_18_min', 'R_18_max', 'R_18_last', 'R_18_quantile',\n       'D_88_mean', 'D_88_std', 'D_88_min', 'D_88_max', 'D_88_last',\n       'D_88_quantile', 'B_31_mean', 'B_31_std', 'B_31_min', 'B_31_max',\n       'B_31_last', 'B_31_quantile', 'R_19_min', 'R_19_last',\n       'R_19_quantile', 'B_32_min', 'B_32_max', 'B_32_quantile',\n       'S_20_min', 'S_20_last', 'S_20_quantile', 'R_20_min',\n       'R_20_quantile', 'R_21_min', 'R_21_last', 'R_21_quantile',\n       'B_33_min', 'D_89_mean', 'D_89_min', 'D_89_max', 'D_89_last',\n       'D_89_quantile', 'R_22_min', 'R_22_max', 'R_22_last',\n       'R_22_quantile', 'R_23_mean', 'R_23_std', 'R_23_min', 'R_23_max',\n       'R_23_last', 'R_23_quantile', 'D_92_min', 'D_92_last',\n       'D_92_quantile', 'D_93_mean', 'D_93_std', 'D_93_min', 'D_93_max',\n       'D_93_last', 'D_93_quantile', 'D_94_mean', 'D_94_std', 'D_94_min',\n       'D_94_max', 'D_94_last', 'D_94_quantile', 'R_24_std', 'R_24_min',\n       'R_24_max', 'R_24_quantile', 'R_25_mean', 'R_25_std', 'R_25_min',\n       'R_25_max', 'R_25_last', 'R_25_quantile', 'D_96_min',\n       'D_96_quantile', 'D_103_max', 'R_26_std', 'D_108_min',\n       'D_108_last', 'D_108_quantile', 'D_109_mean', 'D_109_std',\n       'D_109_min', 'D_109_max', 'D_109_last', 'D_109_quantile',\n       'D_110_min', 'D_110_last', 'D_110_quantile', 'D_111_std',\n       'D_111_min', 'D_111_last', 'D_111_quantile', 'B_39_min',\n       'B_39_quantile', 'D_123_min', 'D_125_min', 'D_125_quantile',\n       'D_127_std', 'D_127_min', 'D_127_quantile', 'B_41_min',\n       'B_42_mean', 'B_42_std', 'B_42_min', 'B_42_max', 'B_42_last',\n       'B_42_quantile', 'D_131_min', 'R_28_mean', 'R_28_std', 'R_28_min',\n       'R_28_max', 'R_28_last', 'R_28_quantile', 'D_134_min',\n       'D_134_quantile', 'D_135_min', 'D_135_quantile', 'D_136_min',\n       'D_136_max', 'D_136_quantile', 'D_137_min', 'D_137_max',\n       'D_137_last', 'D_137_quantile', 'D_138_min', 'D_138_quantile',\n       'D_139_min', 'D_143_last', 'D_143_quantile', 'D_116_last',\n       'D_116_nunique', 'D_64_count', 'D_68_count','B_41_max', 'D_127_max', 'D_86_max', 'D_143_max', 'D_135_last',\n       'B_33_max', 'D_125_last', 'S_20_max', 'D_73_min', 'R_13_mean',\n       'R_11_quantile', 'B_38_count', 'D_79_quantile', 'D_72_min',\n       'S_18_quantile', 'D_139_quantile', 'B_41_quantile', 'D_123_last',\n       'D_136_last', 'D_83_last', 'R_19_max', 'D_107_min', 'D_103_last',\n       'D_63_count', 'S_6_max', 'B_33_quantile', 'D_89_std', 'D_103_min',\n       'D_103_quantile', 'S_6_min', 'R_24_mean', 'D_140_max', 'D_91_max',\n       'D_140_min', 'B_39_std', 'R_21_std', 'D_143_min', 'R_21_mean',\n       'R_17_std', 'R_19_mean', 'D_108_mean', 'S_20_std', 'D_114_nunique',\n       'D_132_min', 'D_136_std', 'D_135_max', 'D_83_max', 'R_20_max',\n       'D_140_std', 'B_32_std', 'D_127_last', 'B_41_last', 'B_22_min',\n       'R_12_std', 'D_139_last', 'D_111_max', 'D_120_count', 'R_2_max',\n       'B_39_max', 'D_84_max', 'S_18_max', 'D_86_mean', 'D_110_std',\n       'D_107_quantile', 'D_125_max', 'B_30_count', 'D_123_quantile',\n       'R_21_max', 'D_76_quantile', 'D_132_quantile', 'D_73_quantile',\n       'R_8_max', 'D_66_count', 'D_111_mean', 'D_65_quantile',\n       'D_126_count', 'D_145_min', 'D_49_min', 'D_138_max',\n       'D_78_quantile', 'D_138_last', 'D_139_max', 'R_22_std',\n       'S_6_quantile', 'D_127_mean', 'S_6_last', 'D_135_std', 'D_108_std',\n       'D_106_quantile', 'B_39_last', 'B_41_mean', 'D_92_max',\n       'S_20_mean', 'D_116_count', 'D_143_std', 'R_10_max', 'D_114_count',\n       'D_140_quantile', 'D_145_last', 'R_9_last', 'R_14_mean',\n       'B_32_mean', 'R_20_mean', 'D_78_max', 'D_76_min', 'R_4_std',\n       'B_22_max', 'D_117_count', 'D_51_min', 'D_64_nunique', 'D_142_min',\n       'D_135_mean', 'D_129_min', 'D_138_mean', 'D_63_nunique',\n       'D_91_quantile', 'R_22_mean', 'D_73_max', 'D_82_min', 'D_145_max',\n       'R_5_mean', 'R_8_mean', 'D_134_mean', 'B_22_quantile', 'D_108_max',\n       'B_32_last', 'D_137_mean', 'D_82_max', 'D_82_quantile', 'D_76_std',\n       'D_139_std', 'R_14_last', 'D_117_nunique', 'D_96_mean', 'D_70_min',\n       'D_92_mean', 'D_79_max', 'D_107_max', 'D_107_last', 'D_81_mean',\n       'D_68_nunique', 'D_72_last', 'B_41_std', 'B_39_mean']\n\ndef feature_correlation(data, threshold=0.1):\ncorrelations = data.corr()[\"target\"].drop(\"target\")\n# Filter the features with correlation to the target less than threshold\nfiltered_features = [c for c in train.index if correlations[abs(correlations.values) > 0.05].index]\nreturn data[filtered_features]\ntrain = feature_correlation(train)\n\ndef fill_missing_values(data, imputation_method='median'):\n    data_copy = data.copy()\n    for column in data_copy.columns:\n        if data_copy[column].dtype == np.dtype('O'):\n            data_copy[column] = data_copy[column].fillna(data_copy[column].mode().iloc[0])\n        else:\n            if imputation_method == 'median':\n                data_copy[column] = data_copy[column].fillna(data_copy[column].median())\n            elif imputation_method == 'mean':\n                data_copy[column] = data_copy[column].fillna(data_copy[column].mean())\n    return data_copy\n    '''\n\nno_need = []","metadata":{"execution":{"iopub.status.busy":"2022-07-07T14:03:17.961913Z","iopub.execute_input":"2022-07-07T14:03:17.962757Z","iopub.status.idle":"2022-07-07T14:03:17.976185Z","shell.execute_reply.started":"2022-07-07T14:03:17.962714Z","shell.execute_reply":"2022-07-07T14:03:17.975136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install lightgbm --install-option=--gpu --install-option=\"--opencl-library=/usr/lib/x86_64-linux-gnu/libOpenCL.so.1\"","metadata":{"execution":{"iopub.status.busy":"2022-07-12T15:51:49.575058Z","iopub.execute_input":"2022-07-12T15:51:49.575712Z","iopub.status.idle":"2022-07-12T15:52:02.327463Z","shell.execute_reply.started":"2022-07-12T15:51:49.575610Z","shell.execute_reply":"2022-07-12T15:52:02.326462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LOAD LIBRARIES\nimport pandas as pd, numpy as np # CPU libraries\nimport matplotlib.pyplot as plt, gc, os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator\nfrom matplotlib.colors import ListedColormap\nfrom cycler import cycler\nfrom IPython.display import display\nimport datetime\nimport scipy.stats\nimport warnings\nimport pickle\nfrom colorama import Fore, Back, Style\nimport gc\nimport sys\n\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.calibration import CalibrationDisplay\nimport lightgbm as lgb\nfrom lightgbm import LGBMClassifier \nfrom lightgbm import log_evaluation\nfrom sklearn.model_selection import KFold\nfrom sklearn.inspection import permutation_importance\nimport eli5\nfrom eli5.sklearn import PermutationImportance\n","metadata":{"execution":{"iopub.status.busy":"2022-07-16T07:30:15.719594Z","iopub.execute_input":"2022-07-16T07:30:15.721067Z","iopub.status.idle":"2022-07-16T07:30:26.889161Z","shell.execute_reply.started":"2022-07-16T07:30:15.720929Z","shell.execute_reply":"2022-07-16T07:30:26.888054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# VERSION NAME FOR SAVED MODEL FILES\nVER = 1\n\n# TRAIN RANDOM SEED\nSEED = 42\n\n# FILL NAN VALUE\nNAN_VALUE = -127 # will fit in int8\n\n# FOLDS PER MODEL\nFOLDS = 5\n\nINFERENCE = True # set to False if you only want to cross-validate","metadata":{"execution":{"iopub.status.busy":"2022-07-16T07:25:55.441510Z","iopub.execute_input":"2022-07-16T07:25:55.442243Z","iopub.status.idle":"2022-07-16T07:25:55.446922Z","shell.execute_reply.started":"2022-07-16T07:25:55.442206Z","shell.execute_reply":"2022-07-16T07:25:55.446005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Process and Feature Engineer Train Data\nWe will load @raddar Kaggle dataset from [here][1] with discussion [here][2]. Then we will engineer features suggested by @huseyincot in his notebooks [here][3] and [here][4]. We will use [RAPIDS][5] and the GPU to create new features quickly.\n\n[1]: https://www.kaggle.com/datasets/raddar/amex-data-integer-dtypes-parquet-format\n[2]: https://www.kaggle.com/competitions/amex-default-prediction/discussion/328514\n[3]: https://www.kaggle.com/code/huseyincot/amex-catboost-0-793\n[4]: https://www.kaggle.com/code/huseyincot/amex-agg-data-how-it-created\n[5]: https://rapids.ai/","metadata":{}},{"cell_type":"code","source":"def read_file(path = '', usecols = None):\n    # LOAD DATAFRAME\n    if usecols is not None: df = pd.read_parquet(path, columns=usecols)\n    else: df = pd.read_parquet(path)\n    # REDUCE DTYPE FOR CUSTOMER AND DATE\n    df['customer_ID'] = df['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\n    df.S_2 = cudf.to_datetime( df.S_2 )\n    # SORT BY CUSTOMER AND DATE (so agg('last') works correctly)\n    #df = df.sort_values(['customer_ID','S_2'])\n    #df = df.reset_index(drop=True)\n    # FILL NAN\n    df = df.fillna(NAN_VALUE) \n    print('shape of data:', df.shape)\n    \n    return df\n\nTRAIN_PATH = '../input/amex-data-integer-dtypes-parquet-format/train.parquet'\ntrain = read_file(path = TRAIN_PATH)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T07:26:02.435030Z","iopub.execute_input":"2022-07-16T07:26:02.435398Z","iopub.status.idle":"2022-07-16T07:26:24.401414Z","shell.execute_reply.started":"2022-07-16T07:26:02.435369Z","shell.execute_reply":"2022-07-16T07:26:24.400212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_parquet('../input/amex-data-integer-dtypes-parquet-format/train.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-07-16T07:30:26.891084Z","iopub.execute_input":"2022-07-16T07:30:26.892103Z","iopub.status.idle":"2022-07-16T07:30:47.298214Z","shell.execute_reply.started":"2022-07-16T07:30:26.892057Z","shell.execute_reply":"2022-07-16T07:30:47.297045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.auto import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-07-16T07:31:10.203880Z","iopub.execute_input":"2022-07-16T07:31:10.206200Z","iopub.status.idle":"2022-07-16T07:31:10.222507Z","shell.execute_reply.started":"2022-07-16T07:31:10.206123Z","shell.execute_reply":"2022-07-16T07:31:10.221083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = train.drop(['customer_ID', 'S_2'], axis = 1).columns.to_list()\ncat_features = [\n    \"B_30\",\n    \"B_38\",\n    \"D_114\",\n    \"D_116\",\n    \"D_117\",\n    \"D_120\",\n    \"D_126\",\n    \"D_63\",\n    \"D_64\",\n    \"D_66\",\n    \"D_68\",\n]\nnum_features = [col for col in features if col not in cat_features]","metadata":{"execution":{"iopub.status.busy":"2022-07-16T07:32:03.959054Z","iopub.execute_input":"2022-07-16T07:32:03.959461Z","iopub.status.idle":"2022-07-16T07:32:06.105085Z","shell.execute_reply.started":"2022-07-16T07:32:03.959431Z","shell.execute_reply":"2022-07-16T07:32:06.104058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[num_features].diff(1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[num_features].diff(1)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T07:50:45.773847Z","iopub.execute_input":"2022-07-16T07:50:45.774355Z","iopub.status.idle":"2022-07-16T07:50:56.539954Z","shell.execute_reply.started":"2022-07-16T07:50:45.774315Z","shell.execute_reply":"2022-07-16T07:50:56.538991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = []\ncustomer_ids = []\nfor customer_id, df in tqdm(train.groupby(['customer_ID'])):\n    # Get the differences\n    diff_df1 = df[num_features].diff(1).iloc[[-1]].values.astype(np.float32)\n    # Append to lists\n    df1.append(diff_df1)\n    customer_ids.append(customer_id)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T07:32:08.903474Z","iopub.execute_input":"2022-07-16T07:32:08.903999Z","iopub.status.idle":"2022-07-16T07:43:10.168376Z","shell.execute_reply.started":"2022-07-16T07:32:08.903959Z","shell.execute_reply":"2022-07-16T07:43:10.167469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer_ids","metadata":{"execution":{"iopub.status.busy":"2022-07-16T07:47:39.762060Z","iopub.execute_input":"2022-07-16T07:47:39.762515Z","iopub.status.idle":"2022-07-16T07:47:39.790659Z","shell.execute_reply.started":"2022-07-16T07:47:39.762480Z","shell.execute_reply":"2022-07-16T07:47:39.789798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1","metadata":{"execution":{"iopub.status.busy":"2022-07-16T07:47:21.775652Z","iopub.execute_input":"2022-07-16T07:47:21.776194Z","iopub.status.idle":"2022-07-16T07:47:24.253330Z","shell.execute_reply.started":"2022-07-16T07:47:21.776152Z","shell.execute_reply":"2022-07-16T07:47:24.251086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_and_feature_engineer(df):\n    # FEATURE ENGINEERING FROM \n    # https://www.kaggle.com/code/huseyincot/amex-agg-data-how-it-created\n    all_cols = [c for c in list(df.columns) if c not in ['customer_ID','S_2']]\n    cat_features = [\"B_30\",\"B_38\",\"D_114\",\"D_116\",\"D_117\",\"D_120\",\"D_126\",\"D_63\",\"D_64\",\"D_66\",\"D_68\"]\n    num_features = [col for col in all_cols if col not in cat_features]\n\n    test_num_agg = df.groupby(\"customer_ID\")[num_features].agg(['first', 'mean', 'std', 'min', 'max', 'last'])\n    test_num_agg.columns = ['_'.join(x) for x in test_num_agg.columns]\n    \n    for col in test_num_agg:\n        if 'last' in col and col.replace('last', 'first') in test_num_agg:\n            test_num_agg[col + '_lag_sub'] = test_num_agg[col] - test_num_agg[col.replace('last', 'first')]\n            test_num_agg[col + '_lag_div'] = test_num_agg[col] / test_num_agg[col.replace('last', 'first')]\n\n    test_cat_agg = df.groupby(\"customer_ID\")[cat_features].agg(['count', 'first', 'last', 'nunique'])\n    test_cat_agg.columns = ['_'.join(x) for x in test_cat_agg.columns]\n\n    df = cudf.concat([test_num_agg, test_cat_agg], axis=1)\n    del test_num_agg, test_cat_agg\n    print('shape after engineering', df.shape )\n    \n    return df\n\ntrain = process_and_feature_engineer(train)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:40:09.331446Z","iopub.execute_input":"2022-07-07T13:40:09.333238Z","iopub.status.idle":"2022-07-07T13:40:11.002607Z","shell.execute_reply.started":"2022-07-07T13:40:09.333198Z","shell.execute_reply":"2022-07-07T13:40:11.001594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_filter(data, threshold=0.1):\n    features = data.columns\n    filtered_features = []\n    for feature in features:\n        if data[feature].isnull().sum() < threshold:\n            filtered_features.append(feature)\n    return data[filtered_features]\ntrain = feature_filter(train)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:40:11.004083Z","iopub.execute_input":"2022-07-07T13:40:11.004441Z","iopub.status.idle":"2022-07-07T13:40:12.176427Z","shell.execute_reply.started":"2022-07-07T13:40:11.004405Z","shell.execute_reply":"2022-07-07T13:40:12.175665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = cudf.read_csv('../input/amex-default-prediction/train_labels.csv')\ntargets['customer_ID'] = targets['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\ntargets.index = targets['customer_ID'].sort_index()\ntargets = targets.drop('customer_ID', axis=1)\ntrain = train.join(targets,on =['customer_ID'] ).sort_index()\n\ndel targets\ngc.collect()\n\ntrain = train.fillna(NAN_VALUE)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:41:17.818822Z","iopub.execute_input":"2022-07-07T13:41:17.820419Z","iopub.status.idle":"2022-07-07T13:41:20.139468Z","shell.execute_reply.started":"2022-07-07T13:41:17.820375Z","shell.execute_reply":"2022-07-07T13:41:20.138580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def amex_metric(y_true: np.array, y_pred: np.array) -> float:\n\n    # count of positives and negatives\n    n_pos = y_true.sum()\n    n_neg = y_true.shape[0] - n_pos\n\n    # sorting by descring prediction values\n    indices = np.argsort(y_pred)[::-1]\n    preds, target = y_pred[indices], y_true[indices]\n\n    # filter the top 4% by cumulative row weights\n    weight = 20.0 - target * 19.0\n    cum_norm_weight = (weight / weight.sum()).cumsum()\n    four_pct_filter = cum_norm_weight <= 0.04\n\n    # default rate captured at 4%\n    d = target[four_pct_filter].sum() / n_pos\n\n    # weighted gini coefficient\n    lorentz = (target / n_pos).cumsum()\n    gini = ((lorentz - cum_norm_weight) * weight).sum()\n\n    # max weighted gini coefficient\n    gini_max = 10 * n_neg * (1 - 19 / (n_pos + 20 * n_neg))\n\n    # normalized weighted gini coefficient\n    g = gini / gini_max\n\n    return 0.5 * (g + d)\n\n\ndef lgb_amex_metric(y_true, y_pred):\n    \"\"\"The competition metric with lightgbm's calling convention\"\"\"\n    return ('amex',\n            amex_metric(y_true, y_pred),\n            True)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T14:03:09.418561Z","iopub.execute_input":"2022-07-07T14:03:09.419415Z","iopub.status.idle":"2022-07-07T14:03:09.429022Z","shell.execute_reply.started":"2022-07-07T14:03:09.419378Z","shell.execute_reply":"2022-07-07T14:03:09.428120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Cross-validation of the classifier\n\nONLY_FIRST_FOLD = False\n\n\nfeatures = [f for f in train.columns if f != 'customer_ID' and f != 'target' and f not in no_need]\n\ndef my_booster(random_state=1, n_estimators=1100):\n    return LGBMClassifier(n_estimators=n_estimators,\n                          learning_rate=0.03, reg_lambda=50,\n                          min_child_samples=2400,\n                          num_leaves=95,\n                          colsample_bytree=0.19,\n                          max_bins=253, \n                          device='gpu',\n                          random_state=random_state)\n      \nprint(f\"{len(features)} features\")\nscore_list = []\ny_pred_list = []\nimportance_list = []\nperm_imp_list = []\n\nkf = KFold(n_splits=5, shuffle=True, random_state=SEED)\nfor fold,(train_idx, valid_idx) in enumerate(kf.split(train, train.target)):\n    \n    X_tr, X_va, y_tr, y_va, model = None, None, None, None, None\n    start_time = datetime.datetime.now()\n    X_tr = train.iloc[train_idx][features].as_gpu_matrix()\n    X_va = train.iloc[valid_idx][features].as_gpu_matrix()\n    y_tr = cupy.asarray(train.iloc[train_idx][\"target\"])\n    y_va = cupy.asarray(train.iloc[valid_idx][\"target\"])\n\n    model = my_booster()\n    with warnings.catch_warnings():\n        warnings.filterwarnings('ignore', category=UserWarning)\n        model.fit(X_tr, cupy.asnumpy(y_tr),\n                  eval_set = [(X_va, cupy.asnumpy(y_va))], \n                  eval_metric=[lgb_amex_metric],\n                  early_stopping_rounds = 100,\n                  callbacks=[log_evaluation(500)])\n        file = f'LGBM_v{VER}_fold{fold}.pkl'\n        pickle.dump(model, open(file, 'wb'))\n        \n    \n    y_va_pred = model.predict_proba(X_va, raw_score=True)\n    score = amex_metric(y_va, y_va_pred)\n    n_trees = model.best_iteration_\n    if n_trees is None: n_trees = model.n_estimators\n    print(f\"{Fore.GREEN}{Style.BRIGHT}Fold {fold} | {str(datetime.datetime.now() - start_time)[-12:-7]} |\"\n          f\" {n_trees:5} trees |\"\n          f\"                Score = {score:.5f}{Style.RESET_ALL}\")\n    score_list.append(score)\n    \n    perm = PermutationImportance(model, random_state=1).fit(X_va, cupy.asnumpy(y_va))\n    eli5.show_weights(perm, feature_names = features)\n    \n    del X_tr, y_tr, X_va, y_va, score, model\n    gc.collect()\n\n    if ONLY_FIRST_FOLD: break # we only want the first fold\n    \nprint(f\"{Fore.GREEN}{Style.BRIGHT}OOF Score:{np.mean(cupy.asarray(score_list)):.5f}{Style.RESET_ALL}\")\ndel train_idx, valid_idx, y_va_pred, train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T14:03:27.261543Z","iopub.execute_input":"2022-07-07T14:03:27.261938Z","iopub.status.idle":"2022-07-07T14:08:12.732576Z","shell.execute_reply.started":"2022-07-07T14:03:27.261904Z","shell.execute_reply":"2022-07-07T14:08:12.731829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CALCULATE SIZE OF EACH SEPARATE TEST PART\ndef get_rows(customers, test, NUM_PARTS = 4, verbose = ''):\n    chunk = len(customers)//NUM_PARTS\n    if verbose != '':\n        print(f'We will process {verbose} data as {NUM_PARTS} separate parts.')\n        print(f'There will be {chunk} customers in each part (except the last part).')\n        print('Below are number of rows in each part:')\n    rows = []\n\n    for k in range(NUM_PARTS):\n        if k==NUM_PARTS-1: cc = customers[k*chunk:]\n        else: cc = customers[k*chunk:(k+1)*chunk]\n        s = test.loc[test.customer_ID.isin(cc)].shape[0]\n        rows.append(s)\n    if verbose != '': print( rows )\n    return rows,chunk\n\n# COMPUTE SIZE OF 4 PARTS FOR TEST DATA\nNUM_PARTS = 4\nTEST_PATH =  '../input/amex-data-integer-dtypes-parquet-format/test.parquet'\n\nprint(f'Reading test data...')\ntest = read_file(path = TEST_PATH, usecols = ['customer_ID','S_2'])\ncustomers = test[['customer_ID']].drop_duplicates().sort_index().values.flatten()\nrows,num_cust = get_rows(customers, test[['customer_ID']], NUM_PARTS = NUM_PARTS, verbose = 'test')","metadata":{"execution":{"iopub.status.busy":"2022-07-06T17:00:03.790778Z","iopub.execute_input":"2022-07-06T17:00:03.791183Z","iopub.status.idle":"2022-07-06T17:00:06.568939Z","shell.execute_reply.started":"2022-07-06T17:00:03.791149Z","shell.execute_reply":"2022-07-06T17:00:06.567693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# INFER TEST DATA IN PARTS\nskip_rows = 0\nskip_cust = 0\ntest_preds = []\n\nfor k in range(NUM_PARTS):\n    \n    # READ PART OF TEST DATA\n    print(f'\\nReading test data...')\n    test = read_file(path = TEST_PATH)\n    test = test.iloc[skip_rows:skip_rows+rows[k]]\n    skip_rows += rows[k]\n    print(f'=> Test part {k+1} has shape', test.shape )\n    \n    # PROCESS AND FEATURE ENGINEER PART OF TEST DATA\n    test = process_and_feature_engineer(test)\n    test['P_2_mean*D_39_std'] = test['P_2_mean']*test['D_39_std'] \n    test = test.fillna(NAN_VALUE)\n    if k==NUM_PARTS-1: test = test.loc[customers[skip_cust:]]\n    else: test = test.loc[customers[skip_cust:skip_cust+num_cust]]\n    skip_cust += num_cust\n    \n    dtest = test[features].as_gpu_matrix()\n    del test \n    gc.collect()\n    # reduce memory test = test[['P_2_mean']] 必要だったら追加\n     # INFER LGBM MODELS ON TEST DATA\n    with open(f'LGBM_v{VER}_fold0.pkl', 'rb') as pickle_file:\n        model = pickle.load(pickle_file)\n        preds = model.predict_proba(dtest,raw_score=True)\n    for f in range(1,FOLDS):\n        with open(f'LGBM_v{VER}_fold{f}.pkl', 'rb') as pickle_file:\n            model = pickle.load(pickle_file)\n            preds += model.predict_proba(dtest,raw_score=True)\n    preds = preds / FOLDS\n    test_preds.append(preds)\n\n# CLEAN MEMORY\ndel dtest, model\n_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T14:55:33.481548Z","iopub.execute_input":"2022-07-06T14:55:33.482216Z","iopub.status.idle":"2022-07-06T14:56:54.776857Z","shell.execute_reply.started":"2022-07-06T14:55:33.482143Z","shell.execute_reply":"2022-07-06T14:56:54.772731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds = np.concatenate(test_preds)\ntest = cudf.DataFrame(index=customers,data={'prediction':test_preds})\nsub = cudf.read_csv('../input/amex-default-prediction/sample_submission.csv')[['customer_ID']]\nsub['customer_ID_hash'] = sub['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\nsub = sub.set_index('customer_ID_hash')\nsub = sub.merge(test[['prediction']], left_index=True, right_index=True, how='left')\nsub = sub.reset_index(drop=True)\n\n# DISPLAY PREDICTIONS\nsub.to_csv(f'submission_LGBM_v{VER}.csv',index=False)\nprint('Submission file shape is', sub.shape )\nsub","metadata":{"execution":{"iopub.status.busy":"2022-07-06T14:56:54.778031Z","iopub.status.idle":"2022-07-06T14:56:54.778739Z","shell.execute_reply.started":"2022-07-06T14:56:54.77844Z","shell.execute_reply":"2022-07-06T14:56:54.778471Z"},"trusted":true},"execution_count":null,"outputs":[]}]}