{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import gc\nimport lightgbm as lgb\n\nimport pandas as pd\npd.set_option(\"display.max_rows\", 500)\npd.set_option(\"display.max_columns\", 500)\npd.set_option(\"display.width\", 1000)\n\n%pylab inline\nimport seaborn as sns\n\nfrom IPython.display import set_matplotlib_formats\nset_matplotlib_formats('svg')","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-07-09T09:53:13.981498Z","iopub.execute_input":"2022-07-09T09:53:13.981902Z","iopub.status.idle":"2022-07-09T09:53:16.671708Z","shell.execute_reply.started":"2022-07-09T09:53:13.981823Z","shell.execute_reply":"2022-07-09T09:53:16.670105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read Data","metadata":{}},{"cell_type":"code","source":"%%time\nfeatures_avg = ['B_1', 'B_2', 'B_3', 'B_4', 'B_5', 'B_6', 'B_8', 'B_9', 'B_10', 'B_11', 'B_12', 'B_13', 'B_14', 'B_15', 'B_16', 'B_17', 'B_18', 'B_19', 'B_20', 'B_21', 'B_22', 'B_23', 'B_24', 'B_25', 'B_28', 'B_29', 'B_30', 'B_32', 'B_33', 'B_37', 'B_38', 'B_39', 'B_40', 'B_41', 'B_42', 'D_39', 'D_41', 'D_42', 'D_43', 'D_44', 'D_45', 'D_46', 'D_47', 'D_48', 'D_50', 'D_51', 'D_53', 'D_54', 'D_55', 'D_58', 'D_59', 'D_60', 'D_61', 'D_62', 'D_65', 'D_66', 'D_69', 'D_70', 'D_71', 'D_72', 'D_73', 'D_74', 'D_75', 'D_76', 'D_77', 'D_78', 'D_80', 'D_82', 'D_84', 'D_86', 'D_91', 'D_92', 'D_94', 'D_96', 'D_103', 'D_104', 'D_108', 'D_112', 'D_113', 'D_114', 'D_115', 'D_117', 'D_118', 'D_119', 'D_120', 'D_121', 'D_122', 'D_123', 'D_124', 'D_125', 'D_126', 'D_128', 'D_129', 'D_131', 'D_132', 'D_133', 'D_134', 'D_135', 'D_136', 'D_140', 'D_141', 'D_142', 'D_144', 'D_145', 'P_2', 'P_3', 'P_4', 'R_1', 'R_2', 'R_3', 'R_7', 'R_8', 'R_9', 'R_10', 'R_11', 'R_14', 'R_15', 'R_16', 'R_17', 'R_20', 'R_21', 'R_22', 'R_24', 'R_26', 'R_27', 'S_3', 'S_5', 'S_6', 'S_7', 'S_9', 'S_11', 'S_12', 'S_13', 'S_15', 'S_16', 'S_18', 'S_22', 'S_23', 'S_25', 'S_26']\nfeatures_min = ['B_2', 'B_4', 'B_5', 'B_9', 'B_13', 'B_14', 'B_15', 'B_16', 'B_17', 'B_19', 'B_20', 'B_28', 'B_29', 'B_33', 'B_36', 'B_42', 'D_39', 'D_41', 'D_42', 'D_45', 'D_46', 'D_48', 'D_50', 'D_51', 'D_53', 'D_55', 'D_56', 'D_58', 'D_59', 'D_60', 'D_62', 'D_70', 'D_71', 'D_74', 'D_75', 'D_78', 'D_83', 'D_102', 'D_112', 'D_113', 'D_115', 'D_118', 'D_119', 'D_121', 'D_122', 'D_128', 'D_132', 'D_140', 'D_141', 'D_144', 'D_145', 'P_2', 'P_3', 'R_1', 'R_27', 'S_3', 'S_5', 'S_7', 'S_9', 'S_11', 'S_12', 'S_23', 'S_25']\nfeatures_max = ['B_1', 'B_2', 'B_3', 'B_4', 'B_5', 'B_6', 'B_7', 'B_8', 'B_9', 'B_10', 'B_12', 'B_13', 'B_14', 'B_15', 'B_16', 'B_17', 'B_18', 'B_19', 'B_21', 'B_23', 'B_24', 'B_25', 'B_29', 'B_30', 'B_33', 'B_37', 'B_38', 'B_39', 'B_40', 'B_42', 'D_39', 'D_41', 'D_42', 'D_43', 'D_44', 'D_45', 'D_46', 'D_47', 'D_48', 'D_49', 'D_50', 'D_52', 'D_55', 'D_56', 'D_58', 'D_59', 'D_60', 'D_61', 'D_63', 'D_64', 'D_65', 'D_70', 'D_71', 'D_72', 'D_73', 'D_74', 'D_76', 'D_77', 'D_78', 'D_80', 'D_82', 'D_84', 'D_91', 'D_102', 'D_105', 'D_107', 'D_110', 'D_111', 'D_112', 'D_115', 'D_116', 'D_117', 'D_118', 'D_119', 'D_121', 'D_122', 'D_123', 'D_124', 'D_125', 'D_126', 'D_128', 'D_131', 'D_132', 'D_133', 'D_134', 'D_135', 'D_136', 'D_138', 'D_140', 'D_141', 'D_142', 'D_144', 'D_145', 'P_2', 'P_3', 'P_4', 'R_1', 'R_3', 'R_5', 'R_6', 'R_7', 'R_8', 'R_10', 'R_11', 'R_14', 'R_17', 'R_20', 'R_26', 'R_27', 'S_3', 'S_5', 'S_7', 'S_8', 'S_11', 'S_12', 'S_13', 'S_15', 'S_16', 'S_22', 'S_23', 'S_24', 'S_25', 'S_26', 'S_27']\nfeatures_last = ['B_1', 'B_2', 'B_3', 'B_4', 'B_5', 'B_6', 'B_7', 'B_8', 'B_9', 'B_10', 'B_11', 'B_12', 'B_13', 'B_14', 'B_15', 'B_16', 'B_17', 'B_18', 'B_19', 'B_20', 'B_21', 'B_22', 'B_23', 'B_24', 'B_25', 'B_26', 'B_28', 'B_29', 'B_30', 'B_32', 'B_33', 'B_36', 'B_37', 'B_38', 'B_39', 'B_40', 'B_41', 'B_42', 'D_39', 'D_41', 'D_42', 'D_43', 'D_44', 'D_45', 'D_46', 'D_47', 'D_48', 'D_49', 'D_50', 'D_51', 'D_52', 'D_53', 'D_54', 'D_55', 'D_56', 'D_58', 'D_59', 'D_60', 'D_61', 'D_62', 'D_63', 'D_64', 'D_65', 'D_69', 'D_70', 'D_71', 'D_72', 'D_73', 'D_75', 'D_76', 'D_77', 'D_78', 'D_79', 'D_80', 'D_81', 'D_82', 'D_83', 'D_86', 'D_91', 'D_96', 'D_105', 'D_106', 'D_112', 'D_114', 'D_119', 'D_120', 'D_121', 'D_122', 'D_124', 'D_125', 'D_126', 'D_127', 'D_130', 'D_131', 'D_132', 'D_133', 'D_134', 'D_138', 'D_140', 'D_141', 'D_142', 'D_145', 'P_2', 'P_3', 'P_4', 'R_1', 'R_2', 'R_3', 'R_4', 'R_5', 'R_6', 'R_7', 'R_8', 'R_9', 'R_10', 'R_11', 'R_12', 'R_13', 'R_14', 'R_15', 'R_19', 'R_20', 'R_26', 'R_27', 'S_3', 'S_5', 'S_6', 'S_7', 'S_8', 'S_9', 'S_11', 'S_12', 'S_13', 'S_16', 'S_19', 'S_20', 'S_22', 'S_23', 'S_24', 'S_25', 'S_26', 'S_27']\n\nfor i in ['train']:\n    df = pd.read_parquet(f'../input/amex-data-integer-dtypes-parquet-format/{i}.parquet')\n    cid = pd.Categorical(df.pop('customer_ID'), ordered=True)\n    last = (cid != np.roll(cid, -1)) # mask for last statement of every customer\n    if 'target' in df.columns:\n        df.drop(columns=['target'], inplace=True)\n    gc.collect()\n    print('Read', i)\n    df_avg = (df\n              .groupby(cid)\n              .mean()[features_avg]\n              .rename(columns={f: f\"{f}_avg\" for f in features_avg})\n             )\n    gc.collect()\n    print('Computed avg', i)\n    df_min = (df\n              .groupby(cid)\n              .min()[features_min]\n              .rename(columns={f: f\"{f}_min\" for f in features_min})\n             )\n    gc.collect()\n    print('Computed min', i)\n    df_max = (df\n              .groupby(cid)\n              .max()[features_max]\n              .rename(columns={f: f\"{f}_max\" for f in features_max})\n             )\n    gc.collect()\n    print('Computed max', i)\n    df = (df.loc[last, features_last]\n          .rename(columns={f: f\"{f}_last\" for f in features_last})\n          .set_index(np.asarray(cid[last]))\n         )\n    gc.collect()\n    print('Computed last', i)\n    df = pd.concat([df, df_min, df_max, df_avg], axis=1)\n    if i == 'train': train = df\n    else: test = df\n    print(f\"{i} shape: {df.shape}\")\n    del df, df_avg, df_min, df_max, cid, last\n\ntarget = pd.read_csv('../input/amex-default-prediction/train_labels.csv').target.values\nprint(f\"target shape: {target.shape}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-09T09:53:20.54859Z","iopub.execute_input":"2022-07-09T09:53:20.549103Z","iopub.status.idle":"2022-07-09T09:55:35.304744Z","shell.execute_reply.started":"2022-07-09T09:53:20.549063Z","shell.execute_reply":"2022-07-09T09:55:35.303987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LightGBM baseline","metadata":{}},{"cell_type":"code","source":"params = {\n    \"max_depth\": 3,\n    \"learning_rate\": 0.1,\n    \"num_leaves\": 64,\n    \"n_estimators\": 200,\n    \"verbose\": -1,\n    'objective': \"binary\",\n    'device_type': 'gpu',\n    # 'metric': \"logloss\",\n}\n\nd_train = lgb.Dataset(train.iloc[:1000], label=target[:1000], free_raw_data=False)\n\nresult = lgb.cv(params, d_train, nfold=5)\npd.DataFrame(result).tail(5)","metadata":{"execution":{"iopub.status.busy":"2022-07-09T09:56:20.918329Z","iopub.execute_input":"2022-07-09T09:56:20.91884Z","iopub.status.idle":"2022-07-09T09:56:44.705577Z","shell.execute_reply.started":"2022-07-09T09:56:20.918807Z","shell.execute_reply":"2022-07-09T09:56:44.70482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# GridSearchCV","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\nmodel = lgb.LGBMClassifier(verbose=-1, verbosity=-1)\nparam_dist = {\n    \"max_depth\": [5, 7],\n    \"min_child_weight\": [1, 3, 6],\n    \"n_estimators\": [200],\n    \"learning_rate\": np.linspace(0.01, 0.3, 3),\n    'device_type': ['gpu']\n\n}\n\ngrid_search = GridSearchCV(\n    model,\n    param_dist,\n    cv=5,\n    verbose=10,\n    n_jobs=-1,\n    scoring=\"neg_log_loss\",\n)\ngrid_search.fit(train.iloc[:1000], target[:1000])","metadata":{"execution":{"iopub.status.busy":"2022-07-09T10:02:42.960551Z","iopub.execute_input":"2022-07-09T10:02:42.961404Z","iopub.status.idle":"2022-07-09T10:03:08.438257Z","shell.execute_reply.started":"2022-07-09T10:02:42.961365Z","shell.execute_reply":"2022-07-09T10:03:08.43711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_search.best_estimator_, grid_search.best_score_","metadata":{"execution":{"iopub.status.busy":"2022-07-09T10:03:15.409443Z","iopub.execute_input":"2022-07-09T10:03:15.409792Z","iopub.status.idle":"2022-07-09T10:03:15.416525Z","shell.execute_reply.started":"2022-07-09T10:03:15.409764Z","shell.execute_reply":"2022-07-09T10:03:15.415674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# RandomizedSearchCV","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import RandomizedSearchCV\n\nparam_dist = {\n    \"max_depth\": [5, 7],\n    \"min_child_weight\": [1, 3, 6],\n    \"n_estimators\": [200],\n    \"learning_rate\": np.linspace(0.01, 0.3, 3),\n    'device_type': ['gpu']\n\n}\n\nmodel = lgb.LGBMClassifier(verbose=-1, verbosity=-1)\nrandom_search = RandomizedSearchCV(\n    model,\n    param_dist,\n    cv=5,\n    n_iter=20,\n    verbose=10,\n    n_jobs=-1,\n    scoring=\"neg_log_loss\",\n)\nrandom_search.fit(train.iloc[:1000], target[:1000])","metadata":{"execution":{"iopub.status.busy":"2022-07-09T10:03:24.491536Z","iopub.execute_input":"2022-07-09T10:03:24.491886Z","iopub.status.idle":"2022-07-09T10:03:51.088073Z","shell.execute_reply.started":"2022-07-09T10:03:24.491856Z","shell.execute_reply":"2022-07-09T10:03:51.087393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_search.best_estimator_, random_search.best_score_","metadata":{"execution":{"iopub.status.busy":"2022-07-09T10:03:58.067708Z","iopub.execute_input":"2022-07-09T10:03:58.068095Z","iopub.status.idle":"2022-07-09T10:03:58.076483Z","shell.execute_reply.started":"2022-07-09T10:03:58.06806Z","shell.execute_reply":"2022-07-09T10:03:58.075541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# HalvingGridSearch","metadata":{}},{"cell_type":"code","source":"%%time\n\nfrom sklearn.experimental import enable_halving_search_cv  # noqa\nfrom sklearn.model_selection import HalvingGridSearchCV\n\nparam_dist = {\n    \"max_depth\": [5,7],\n    \"min_child_weight\": [1, 3, 6],\n    \"n_estimators\": [200],\n    \"learning_rate\": np.linspace(0.01, 0.3, 3),\n    'device_type': ['gpu']\n\n}\n\nmodel = lgb.LGBMClassifier(verbose=-1, verbosity=-1)\nhalving_grid_search = HalvingGridSearchCV(\n    model, param_dist, cv=5, scoring=\"neg_log_loss\", verbose=1, n_jobs=-1, min_resources=50\n)\nhalving_grid_search.fit(train.iloc[:1000], target[:1000])","metadata":{"execution":{"iopub.status.busy":"2022-07-09T10:04:29.450641Z","iopub.execute_input":"2022-07-09T10:04:29.450989Z","iopub.status.idle":"2022-07-09T10:05:08.268922Z","shell.execute_reply.started":"2022-07-09T10:04:29.45096Z","shell.execute_reply":"2022-07-09T10:05:08.267994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"halving_grid_search.best_estimator_, halving_grid_search.best_score_","metadata":{"execution":{"iopub.status.busy":"2022-07-09T10:05:15.841786Z","iopub.execute_input":"2022-07-09T10:05:15.842187Z","iopub.status.idle":"2022-07-09T10:05:15.852457Z","shell.execute_reply.started":"2022-07-09T10:05:15.842143Z","shell.execute_reply":"2022-07-09T10:05:15.851281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# BayesianOptimization","metadata":{}},{"cell_type":"code","source":"import time\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\nfrom bayes_opt import BayesianOptimization\n\n\ndef lgb_eval(max_depth, min_child_weight, learning_rate):\n    params = {\"metric\": \"auc\", \"verbose\": -1}\n\n    params[\"max_depth\"] = int(max(max_depth, 1))\n    params[\"min_child_weight\"] = int(max(min_child_weight, 1))\n    params[\"learning_rate\"] = float(min(learning_rate, 1))\n    params['objective'] = \"binary\"\n    cv_result = lgb.cv(\n        params,\n        lgb.Dataset(train.iloc[:1000], label=target[:1000]),\n        nfold=3,\n        num_boost_round=2000,\n        seed=0,\n        stratified=True,\n    )\n    return 1.0 * np.array(cv_result[\"auc-mean\"]).max()\n\n\nlgbBO = BayesianOptimization(\n    f=lgb_eval,\n    pbounds={\n        \"max_depth\": (3, 9),\n        \"min_child_weight\": (1, 20),\n        \"learning_rate\": (0.05, 0.5),\n    })\n\nlgbBO.maximize(init_points=10, n_iter=20, acq=\"ei\", alpha=1e-3)\nprint(lgbBO.max)","metadata":{"execution":{"iopub.status.busy":"2022-07-09T10:08:59.372232Z","iopub.execute_input":"2022-07-09T10:08:59.372806Z","iopub.status.idle":"2022-07-09T10:11:08.248223Z","shell.execute_reply.started":"2022-07-09T10:08:59.372768Z","shell.execute_reply":"2022-07-09T10:11:08.247081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}