{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%time\n%reset -f\nimport gc; gc.collect()\n\nimport cudf\nfrom sklearn.model_selection import StratifiedKFold\n\n\ndef flatten_columns(df):\n    df.columns = [\"_\".join(column) for column in df.columns]\n    return df\n\n\ndef preprocess(dataset):\n    dataset['customer_ID'] = dataset['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\n    dataset['S_2'] = cudf.to_datetime(dataset['S_2'])\n    dataset.set_index(['customer_ID', 'S_2'], inplace=True)\n    \n    return dataset\n\n\ndef engineer(dataset, feature_set):\n    if feature_set == 0:\n        dataset = dataset.groupby(level='customer_ID').last()\n        return dataset\n    \n    if feature_set == 1:\n        cat_cols = [\n            \"B_30\",\n            \"B_38\",\n            \"D_114\",\n            \"D_116\",\n            \"D_117\",\n            \"D_120\",\n            \"D_126\",\n            \"D_63\",\n            \"D_64\",\n            \"D_66\",\n            \"D_68\",\n        ]\n        cat_feat = dataset[cat_cols].groupby(level='customer_ID').agg(['last']).pipe(flatten_columns)\n\n        num_cols = [col for col in dataset.columns if col not in cat_cols + ['target']]\n        num_feat = dataset[num_cols].groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std']).pipe(flatten_columns)\n\n        dataset = cudf.concat([cat_feat, num_feat], axis=1)\n        return dataset\n    \n    if feature_set == 2:\n        cat_cols = [\n            \"B_30\",\n            \"B_38\",\n            \"D_114\",\n            \"D_116\",\n            \"D_117\",\n            \"D_120\",\n            \"D_126\",\n            \"D_63\",\n            \"D_64\",\n            \"D_66\",\n            \"D_68\",\n        ]\n        cat_feat = dataset[cat_cols].groupby(level='customer_ID').agg(['last']).pipe(flatten_columns)\n\n        num_cols = [col for col in dataset.columns if col not in cat_cols + ['target']]\n        num_feat = dataset[num_cols].groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std']).pipe(flatten_columns)\n        \n        diff_cols_a = [f\"B_{i}\" for i in [11, 14, 17]] + [\"D_39\", \"D_131\"] + [f\"S_{i}\" for i in [16, 23]]\n        diff_cols_b = [\"P_2\", \"P_3\"]\n        diff_feat = dataset[diff_cols_a + diff_cols_b]\n        for a in diff_cols_a:\n            for b in diff_cols_b:\n                    diff_feat[f\"{a}-{b}\"] = diff_feat[a] - diff_feat[b]\n        diff_feat.drop(diff_cols_a + diff_cols_b, axis=1, inplace=True)\n        diff_feat = diff_feat.groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std']).pipe(flatten_columns)\n\n        dataset = cudf.concat([cat_feat, num_feat, diff_feat], axis=1)\n        return dataset\n    \n    if feature_set == 3:\n        cat_cols = [\n            \"B_30\",\n            \"B_38\",\n            \"D_114\",\n            \"D_116\",\n            \"D_117\",\n            \"D_120\",\n            \"D_126\",\n            \"D_63\",\n            \"D_64\",\n            \"D_66\",\n            \"D_68\",\n        ]\n        cat_feat = dataset[cat_cols].groupby(level='customer_ID').agg(['count', 'last', 'nunique']).pipe(flatten_columns)\n\n        num_cols = [col for col in dataset.columns if col not in cat_cols + ['target']]\n        num_feat = dataset[num_cols].groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std']).pipe(flatten_columns)\n        \n        diff_cols_a = [f\"B_{i}\" for i in [11, 14, 17]] + [\"D_39\", \"D_131\"] + [f\"S_{i}\" for i in [16, 23]]\n        diff_cols_b = [\"P_2\", \"P_3\"]\n        diff_feat = dataset[diff_cols_a + diff_cols_b]\n        for a in diff_cols_a:\n            for b in diff_cols_b:\n                    diff_feat[f\"{a}-{b}\"] = diff_feat[a] - diff_feat[b]\n        diff_feat.drop(diff_cols_a + diff_cols_b, axis=1, inplace=True)\n        diff_feat = diff_feat.groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std']).pipe(flatten_columns)\n\n        dataset = cudf.concat([cat_feat, num_feat, diff_feat], axis=1)\n        return dataset\n    \n    if feature_set == 4:\n        cat_cols = [\n            \"B_30\",\n            \"B_38\",\n            \"D_114\",\n            \"D_116\",\n            \"D_117\",\n            \"D_120\",\n            \"D_126\",\n            \"D_63\",\n            \"D_64\",\n            \"D_66\",\n            \"D_68\",\n        ]\n        cat_feat = dataset[cat_cols].groupby(level='customer_ID').agg(['count', 'last', 'nunique']).pipe(flatten_columns)\n\n        num_cols = [col for col in dataset.columns if col not in cat_cols + ['target']]\n        num_feat = dataset[num_cols].groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std', 'min', 'max']).pipe(flatten_columns)\n        \n        diff_cols_a = [f\"B_{i}\" for i in [11, 14, 17]] + [\"D_39\", \"D_131\"] + [f\"S_{i}\" for i in [16, 23]]\n        diff_cols_b = [\"P_2\", \"P_3\"]\n        diff_feat = dataset[diff_cols_a + diff_cols_b]\n        for a in diff_cols_a:\n            for b in diff_cols_b:\n                    diff_feat[f\"{a}-{b}\"] = diff_feat[a] - diff_feat[b]\n        diff_feat.drop(diff_cols_a + diff_cols_b, axis=1, inplace=True)\n        diff_feat = diff_feat.groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std', 'min', 'max']).pipe(flatten_columns)\n\n        dataset = cudf.concat([cat_feat, num_feat, diff_feat], axis=1)\n        return dataset\n\n    if feature_set == 5:\n        cat_cols = [\n            \"B_30\",\n            \"B_38\",\n            \"D_114\",\n            \"D_116\",\n            \"D_117\",\n            \"D_120\",\n            \"D_126\",\n            \"D_63\",\n            \"D_64\",\n            \"D_66\",\n            \"D_68\",\n        ]\n        cat_feat = dataset[cat_cols].groupby(level='customer_ID').agg(['count', 'last', 'nunique']).pipe(flatten_columns)\n\n        num_cols = [col for col in dataset.columns if col not in cat_cols + ['target']]\n        num_feat = dataset[num_cols].groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std']).pipe(flatten_columns)\n        \n        for col in num_feat.columns:\n            if 'last' in col and col.replace('last', 'first') in num_feat.columns:\n                num_feat[col + '_lag_sub'] = num_feat[col] - num_feat[col.replace('last', 'first')]\n                num_feat[col + '_lag_div'] = num_feat[col] / num_feat[col.replace('last', 'first')]\n        \n        diff_cols_a = [f\"B_{i}\" for i in [11, 14, 17]] + [\"D_39\", \"D_131\"] + [f\"S_{i}\" for i in [16, 23]]\n        diff_cols_b = [\"P_2\", \"P_3\"]\n        diff_feat = dataset[diff_cols_a + diff_cols_b]\n        for a in diff_cols_a:\n            for b in diff_cols_b:\n                    diff_feat[f\"{a}-{b}\"] = diff_feat[a] - diff_feat[b]\n        diff_feat.drop(diff_cols_a + diff_cols_b, axis=1, inplace=True)\n        diff_feat = diff_feat.groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std']).pipe(flatten_columns)\n\n        dataset = cudf.concat([cat_feat, num_feat, diff_feat], axis=1)\n        return dataset\n\n    if feature_set == 6:\n        cat_cols = [\n            \"B_30\",\n            \"B_38\",\n            \"D_114\",\n            \"D_116\",\n            \"D_117\",\n            \"D_120\",\n            \"D_126\",\n            \"D_63\",\n            \"D_64\",\n            \"D_66\",\n            \"D_68\",\n        ]\n        cat_feat = dataset[cat_cols].groupby(level='customer_ID').agg(['count', 'last', 'nunique']).pipe(flatten_columns)\n\n        num_cols = [col for col in dataset.columns if col not in cat_cols + ['target']]\n        num_feat = dataset[num_cols].groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std']).pipe(flatten_columns)\n        \n        diff_cols_a = [f\"B_{i}\" for i in [11, 14, 17]] + [\"D_39\", \"D_131\"] + [f\"S_{i}\" for i in [16, 23]]\n        diff_cols_b = [\"P_2\", \"P_3\"]\n        diff_feat = dataset[diff_cols_a + diff_cols_b]\n        for a in diff_cols_a:\n            for b in diff_cols_b:\n                    diff_feat[f\"{a}-{b}\"] = diff_feat[a] - diff_feat[b]\n        diff_feat.drop(diff_cols_a + diff_cols_b, axis=1, inplace=True)\n        diff_feat = diff_feat.groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std']).pipe(flatten_columns)\n\n        for col in num_feat.columns:\n            if 'last' in col:\n                num_feat[col + '_round2'] = num_feat[col].round(2)\n        \n        dataset = cudf.concat([cat_feat, num_feat, diff_feat], axis=1)\n        return dataset\n\n    if feature_set == 7:\n        cat_cols = [\n            \"B_30\",\n            \"B_38\",\n            \"D_114\",\n            \"D_116\",\n            \"D_117\",\n            \"D_120\",\n            \"D_126\",\n            \"D_63\",\n            \"D_64\",\n            \"D_66\",\n            \"D_68\",\n        ]\n        cat_feat = dataset[cat_cols].groupby(level='customer_ID').agg(['count', 'last', 'nunique']).pipe(flatten_columns)\n\n        num_cols = [col for col in dataset.columns if col not in cat_cols + ['target']]\n        num_feat = dataset[num_cols].groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std']).pipe(flatten_columns)\n        \n        for col in num_cols:\n            num_feat[col + \"_sub_mean\"] = num_feat[col + \"_last\"] - num_feat[col + \"_mean\"]\n        \n        diff_cols_a = [f\"B_{i}\" for i in [11, 14, 17]] + [\"D_39\", \"D_131\"] + [f\"S_{i}\" for i in [16, 23]]\n        diff_cols_b = [\"P_2\", \"P_3\"]\n        diff_feat = dataset[diff_cols_a + diff_cols_b]\n        for a in diff_cols_a:\n            for b in diff_cols_b:\n                    diff_feat[f\"{a}-{b}\"] = diff_feat[a] - diff_feat[b]\n        diff_feat.drop(diff_cols_a + diff_cols_b, axis=1, inplace=True)\n        diff_feat = diff_feat.groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std']).pipe(flatten_columns)\n\n        dataset = cudf.concat([cat_feat, num_feat, diff_feat], axis=1)\n        return dataset    \n    \n    \ndef add_train_labels(train):\n    train_labels = cudf.read_csv('/kaggle/input/amex-default-prediction/train_labels.csv')\n\n    train_labels['customer_ID'] = train_labels['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\n    train_labels.set_index('customer_ID', inplace=True)\n    \n    return cudf.merge(train, train_labels, how='inner', left_index=True, right_index=True).sort_index()\n\n\ndef main(*, feature_set, num_rows):\n    train = cudf.read_parquet(\"../input/amex-data-integer-dtypes-parquet-format/train.parquet\", num_rows=num_rows)\n    train = preprocess(train)\n    train = engineer(train, feature_set)\n    train = add_train_labels(train)\n    \n    kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n    train['fold'] = -1\n    for fold_ix, (train_ixs, valid_ixs) in enumerate(kfold.split(train, train['target'].to_array())):\n        train['fold'].iloc[valid_ixs] = fold_ix\n    \n    print(feature_set, train.shape)\n    train.to_parquet('train.pq')\n\n\nmain(\n    feature_set = 7,\n    num_rows = None\n)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-26T18:24:51.674403Z","iopub.execute_input":"2022-07-26T18:24:51.674797Z","iopub.status.idle":"2022-07-26T18:25:05.089197Z","shell.execute_reply.started":"2022-07-26T18:24:51.674759Z","shell.execute_reply":"2022-07-26T18:25:05.087265Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%reset -f\nimport gc; gc.collect()\nimport cudf\nfrom colorama import Style, Fore\nimport pandas as pd\nimport xgboost as xgb\nimport cupy as cp\n\n\ndef check_input(arr):\n    if type(arr) is pd.DataFrame:\n        arr = arr[arr.columns[0]]\n        \n    if type(arr) is pd.Series:\n        arr = arr.values\n        \n    if len(arr.shape) > 1:\n        arr = arr[:, 0]\n        \n    return arr\n\n\ndef gini(cs_0, cs_1, sum_0, sum_1):\n    auc_ = (cs_0 - sum_0 / 2) * sum_1\n    tot = cs_0[-1] * cs_1[-1]\n\n    return 2 * float(auc_.sum() / tot) - 1\n\n\ndef recall_at4(cs_0, cs_1, sum_1):\n    cs_tot = cs_0 + cs_1\n    th = cs_tot[-1] * 0.96\n    \n    return float(sum_1[cs_tot >= th].sum() / cs_1[-1])\n\n\ndef amex_metric_cupy(y_true, y_pred):\n    y_true = cp.asarray(check_input(y_true))\n    y_pred = cp.asarray(check_input(y_pred))\n    \n    unique = cp.unique(y_pred)\n    rank = cp.searchsorted(unique, y_pred)\n    \n    sum_1 = cp.zeros_like(unique, dtype=cp.float64)\n    sum_1.scatter_add(rank, y_true)\n    \n    sum_0 = cp.zeros_like(unique, dtype=cp.float64)\n    sum_0.scatter_add(rank, 1 - y_true)\n    sum_0 *= 20\n    \n    cs_0, cs_1 = sum_0.cumsum(), sum_1.cumsum()\n    \n    g = gini(cs_0, cs_1, sum_0, sum_1)\n    d = recall_at4(cs_0, cs_1, sum_1)\n    \n    return (g + d) / 2\n\n\ndef xgb_amex(y_pred, dmatrix):\n    return \"amex\", amex_metric_cupy(dmatrix.get_label(), y_pred)\n\n\ndef amex_metric(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n    def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = pd.concat([y_true, y_pred], axis=\"columns\").sort_values(\n            \"prediction\", ascending=False\n        )\n        df[\"weight\"] = df[\"target\"].apply(lambda x: 20 if x == 0 else 1)\n        four_pct_cutoff = int(0.04 * df[\"weight\"].sum())\n        df[\"weight_cumsum\"] = df[\"weight\"].cumsum()\n        df_cutoff = df.loc[df[\"weight_cumsum\"] <= four_pct_cutoff]\n        return (df_cutoff[\"target\"] == 1).sum() / (df[\"target\"] == 1).sum()\n\n    def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = pd.concat([y_true, y_pred], axis=\"columns\").sort_values(\n            \"prediction\", ascending=False\n        )\n        df[\"weight\"] = df[\"target\"].apply(lambda x: 20 if x == 0 else 1)\n        df[\"random\"] = (df[\"weight\"] / df[\"weight\"].sum()).cumsum()\n        total_pos = (df[\"target\"] * df[\"weight\"]).sum()\n        df[\"cum_pos_found\"] = (df[\"target\"] * df[\"weight\"]).cumsum()\n        df[\"lorentz\"] = df[\"cum_pos_found\"] / total_pos\n        df[\"gini\"] = (df[\"lorentz\"] - df[\"random\"]) * df[\"weight\"]\n        return df[\"gini\"].sum()\n\n    def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        y_true_pred = y_true.rename(columns={\"target\": \"prediction\"})\n        return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n    g = normalized_weighted_gini(y_true, y_pred)\n    d = top_four_percent_captured(y_true, y_pred)\n\n    return 0.5 * (g + d)\n\n\ndef main(*, xgb_parameters=None, num_rows=None):\n    folds = cudf.read_parquet(\"train.pq\", num_rows=num_rows)\n    \n    features = [col for col in folds.columns if col not in ['target', 'fold']]\n    print(len(features))\n    \n    predictions = []\n    \n    for fold_ix in range(5):\n        print(Fore.BLUE + \"#\" * 10, f\"Fold {fold_ix}\", \"#\" * 10 + Style.RESET_ALL)\n        \n        train = folds[folds.fold != fold_ix]\n        valid = folds[folds.fold == fold_ix]\n        \n        dtrain = xgb.DMatrix(data=train[features], label=train['target'])\n        dvalid = xgb.DMatrix(data=valid[features], label=valid['target'])\n\n        model = xgb.train(\n            xgb_parameters,\n            dtrain=dtrain,\n            num_boost_round=9999,\n\n            evals=[(dtrain, \"train\"), (dvalid, \"valid\")],\n            early_stopping_rounds=500,\n            \n            custom_metric=xgb_amex,\n            maximize=True,\n\n            verbose_eval=100\n        )\n        \n        model.save_model(f\"xgb_fold{fold_ix}_seed{xgb_parameters['random_state']}.xgb\")\n        \n        prediction = pd.DataFrame({\n            \"prediction\": model.predict(dvalid, iteration_range=(0, model.best_iteration + 1)),\n            \"target\": valid['target'].to_array()\n        })\n        \n        print(f\"Fold: {amex_metric(prediction[['target']], prediction[['prediction']]):.4f} CV\")\n        predictions.append(prediction)\n\n        del dtrain, dvalid, model\n        gc.collect()\n    \n        print(Fore.BLUE + \"#\" * 28, \"\\n\" + Style.RESET_ALL)\n    \n    prediction = pd.concat(predictions)\n    print(Style.BRIGHT + f\"Results: {amex_metric(prediction[['target']], prediction[['prediction']]):.4f} CV\" + Style.RESET_ALL)\n\nmain(\n    xgb_parameters={\n        'max_depth': 7,\n        'eta': 0.03,\n\n        'subsample': 0.88,\n        'colsample_bytree': 0.5,\n        \n        'objective': 'binary:logistic',\n        \n        'tree_method': 'gpu_hist',\n        'predictor': 'gpu_predictor',\n        \n        'random_state': 42,\n        \n        'gamma': 1.5,\n        'min_child_weight': 8,\n        'lambda': 70,\n    },\n    num_rows = None\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T18:25:53.38078Z","iopub.execute_input":"2022-07-26T18:25:53.381183Z","iopub.status.idle":"2022-07-26T18:49:07.348678Z","shell.execute_reply.started":"2022-07-26T18:25:53.381153Z","shell.execute_reply":"2022-07-26T18:49:07.346877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm train.pq","metadata":{"execution":{"iopub.status.busy":"2022-07-26T18:56:03.605767Z","iopub.execute_input":"2022-07-26T18:56:03.606568Z","iopub.status.idle":"2022-07-26T18:56:04.444765Z","shell.execute_reply.started":"2022-07-26T18:56:03.606535Z","shell.execute_reply":"2022-07-26T18:56:04.443525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%reset -f\nimport gc; gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-26T18:56:05.418947Z","iopub.execute_input":"2022-07-26T18:56:05.420091Z","iopub.status.idle":"2022-07-26T18:56:07.133287Z","shell.execute_reply.started":"2022-07-26T18:56:05.420034Z","shell.execute_reply":"2022-07-26T18:56:07.132232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%reset -f\nimport gc; gc.collect()\nimport cudf\nimport xgboost as xgb\nimport numpy as np\n\n\ndef flatten_columns(df):\n    df.columns = [\"_\".join(column) for column in df.columns]\n    return df\n\n\ndef preprocess(dataset):\n    dataset['customer_ID'] = dataset['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\n    dataset['S_2'] = cudf.to_datetime(dataset['S_2'])\n    dataset.set_index(['customer_ID', 'S_2'], inplace=True)\n    \n    return dataset\n\n\ndef engineer(dataset, feature_set):\n    if feature_set == 0:\n        dataset = dataset.groupby(level='customer_ID').last()\n        return dataset\n    \n    if feature_set == 1:\n        cat_cols = [\n            \"B_30\",\n            \"B_38\",\n            \"D_114\",\n            \"D_116\",\n            \"D_117\",\n            \"D_120\",\n            \"D_126\",\n            \"D_63\",\n            \"D_64\",\n            \"D_66\",\n            \"D_68\",\n        ]\n        cat_feat = dataset[cat_cols].groupby(level='customer_ID').agg(['last']).pipe(flatten_columns)\n\n        num_cols = [col for col in dataset.columns if col not in cat_cols + ['target']]\n        num_feat = dataset[num_cols].groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std']).pipe(flatten_columns)\n\n        dataset = cudf.concat([cat_feat, num_feat], axis=1)\n        return dataset\n    \n    if feature_set == 2:\n        cat_cols = [\n            \"B_30\",\n            \"B_38\",\n            \"D_114\",\n            \"D_116\",\n            \"D_117\",\n            \"D_120\",\n            \"D_126\",\n            \"D_63\",\n            \"D_64\",\n            \"D_66\",\n            \"D_68\",\n        ]\n        cat_feat = dataset[cat_cols].groupby(level='customer_ID').agg(['last']).pipe(flatten_columns)\n\n        num_cols = [col for col in dataset.columns if col not in cat_cols + ['target']]\n        num_feat = dataset[num_cols].groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std']).pipe(flatten_columns)\n        \n        diff_cols_a = [f\"B_{i}\" for i in [11, 14, 17]] + [\"D_39\", \"D_131\"] + [f\"S_{i}\" for i in [16, 23]]\n        diff_cols_b = [\"P_2\", \"P_3\"]\n        diff_feat = dataset[diff_cols_a + diff_cols_b]\n        for a in diff_cols_a:\n            for b in diff_cols_b:\n                    diff_feat[f\"{a}-{b}\"] = diff_feat[a] - diff_feat[b]\n        diff_feat.drop(diff_cols_a + diff_cols_b, axis=1, inplace=True)\n        diff_feat = diff_feat.groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std']).pipe(flatten_columns)\n\n        dataset = cudf.concat([cat_feat, num_feat, diff_feat], axis=1)\n        return dataset\n    \n    if feature_set == 3:\n        cat_cols = [\n            \"B_30\",\n            \"B_38\",\n            \"D_114\",\n            \"D_116\",\n            \"D_117\",\n            \"D_120\",\n            \"D_126\",\n            \"D_63\",\n            \"D_64\",\n            \"D_66\",\n            \"D_68\",\n        ]\n        cat_feat = dataset[cat_cols].groupby(level='customer_ID').agg(['count', 'last', 'nunique']).pipe(flatten_columns)\n\n        num_cols = [col for col in dataset.columns if col not in cat_cols + ['target']]\n        num_feat = dataset[num_cols].groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std']).pipe(flatten_columns)\n        \n        diff_cols_a = [f\"B_{i}\" for i in [11, 14, 17]] + [\"D_39\", \"D_131\"] + [f\"S_{i}\" for i in [16, 23]]\n        diff_cols_b = [\"P_2\", \"P_3\"]\n        diff_feat = dataset[diff_cols_a + diff_cols_b]\n        for a in diff_cols_a:\n            for b in diff_cols_b:\n                    diff_feat[f\"{a}-{b}\"] = diff_feat[a] - diff_feat[b]\n        diff_feat.drop(diff_cols_a + diff_cols_b, axis=1, inplace=True)\n        diff_feat = diff_feat.groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std']).pipe(flatten_columns)\n\n        dataset = cudf.concat([cat_feat, num_feat, diff_feat], axis=1)\n        return dataset\n    \n    if feature_set == 4:\n        cat_cols = [\n            \"B_30\",\n            \"B_38\",\n            \"D_114\",\n            \"D_116\",\n            \"D_117\",\n            \"D_120\",\n            \"D_126\",\n            \"D_63\",\n            \"D_64\",\n            \"D_66\",\n            \"D_68\",\n        ]\n        cat_feat = dataset[cat_cols].groupby(level='customer_ID').agg(['count', 'last', 'nunique']).pipe(flatten_columns)\n\n        num_cols = [col for col in dataset.columns if col not in cat_cols + ['target']]\n        num_feat = dataset[num_cols].groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std', 'min', 'max']).pipe(flatten_columns)\n        \n        diff_cols_a = [f\"B_{i}\" for i in [11, 14, 17]] + [\"D_39\", \"D_131\"] + [f\"S_{i}\" for i in [16, 23]]\n        diff_cols_b = [\"P_2\", \"P_3\"]\n        diff_feat = dataset[diff_cols_a + diff_cols_b]\n        for a in diff_cols_a:\n            for b in diff_cols_b:\n                    diff_feat[f\"{a}-{b}\"] = diff_feat[a] - diff_feat[b]\n        diff_feat.drop(diff_cols_a + diff_cols_b, axis=1, inplace=True)\n        diff_feat = diff_feat.groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std', 'min', 'max']).pipe(flatten_columns)\n\n        dataset = cudf.concat([cat_feat, num_feat, diff_feat], axis=1)\n        return dataset\n\n    if feature_set == 5:\n        cat_cols = [\n            \"B_30\",\n            \"B_38\",\n            \"D_114\",\n            \"D_116\",\n            \"D_117\",\n            \"D_120\",\n            \"D_126\",\n            \"D_63\",\n            \"D_64\",\n            \"D_66\",\n            \"D_68\",\n        ]\n        cat_feat = dataset[cat_cols].groupby(level='customer_ID').agg(['count', 'last', 'nunique']).pipe(flatten_columns)\n\n        num_cols = [col for col in dataset.columns if col not in cat_cols + ['target']]\n        num_feat = dataset[num_cols].groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std']).pipe(flatten_columns)\n        \n        for col in num_feat.columns:\n            if 'last' in col and col.replace('last', 'first') in num_feat.columns:\n                num_feat[col + '_lag_sub'] = num_feat[col] - num_feat[col.replace('last', 'first')]\n                num_feat[col + '_lag_div'] = num_feat[col] / num_feat[col.replace('last', 'first')]\n        \n        diff_cols_a = [f\"B_{i}\" for i in [11, 14, 17]] + [\"D_39\", \"D_131\"] + [f\"S_{i}\" for i in [16, 23]]\n        diff_cols_b = [\"P_2\", \"P_3\"]\n        diff_feat = dataset[diff_cols_a + diff_cols_b]\n        for a in diff_cols_a:\n            for b in diff_cols_b:\n                    diff_feat[f\"{a}-{b}\"] = diff_feat[a] - diff_feat[b]\n        diff_feat.drop(diff_cols_a + diff_cols_b, axis=1, inplace=True)\n        diff_feat = diff_feat.groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std']).pipe(flatten_columns)\n\n        dataset = cudf.concat([cat_feat, num_feat, diff_feat], axis=1)\n        return dataset\n\n    if feature_set == 6:\n        cat_cols = [\n            \"B_30\",\n            \"B_38\",\n            \"D_114\",\n            \"D_116\",\n            \"D_117\",\n            \"D_120\",\n            \"D_126\",\n            \"D_63\",\n            \"D_64\",\n            \"D_66\",\n            \"D_68\",\n        ]\n        cat_feat = dataset[cat_cols].groupby(level='customer_ID').agg(['count', 'last', 'nunique']).pipe(flatten_columns)\n\n        num_cols = [col for col in dataset.columns if col not in cat_cols + ['target']]\n        num_feat = dataset[num_cols].groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std']).pipe(flatten_columns)\n        \n        diff_cols_a = [f\"B_{i}\" for i in [11, 14, 17]] + [\"D_39\", \"D_131\"] + [f\"S_{i}\" for i in [16, 23]]\n        diff_cols_b = [\"P_2\", \"P_3\"]\n        diff_feat = dataset[diff_cols_a + diff_cols_b]\n        for a in diff_cols_a:\n            for b in diff_cols_b:\n                    diff_feat[f\"{a}-{b}\"] = diff_feat[a] - diff_feat[b]\n        diff_feat.drop(diff_cols_a + diff_cols_b, axis=1, inplace=True)\n        diff_feat = diff_feat.groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std']).pipe(flatten_columns)\n\n        for col in num_feat.columns:\n            if 'last' in col:\n                num_feat[col + '_round2'] = num_feat[col].round(2)\n        \n        dataset = cudf.concat([cat_feat, num_feat, diff_feat], axis=1)\n        return dataset\n\n    if feature_set == 7:\n        cat_cols = [\n            \"B_30\",\n            \"B_38\",\n            \"D_114\",\n            \"D_116\",\n            \"D_117\",\n            \"D_120\",\n            \"D_126\",\n            \"D_63\",\n            \"D_64\",\n            \"D_66\",\n            \"D_68\",\n        ]\n        cat_feat = dataset[cat_cols].groupby(level='customer_ID').agg(['count', 'last', 'nunique']).pipe(flatten_columns)\n\n        num_cols = [col for col in dataset.columns if col not in cat_cols + ['target']]\n        num_feat = dataset[num_cols].groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std']).pipe(flatten_columns)\n        \n        for col in num_cols:\n            num_feat[col + \"_sub_mean\"] = num_feat[col + \"_last\"] - num_feat[col + \"_mean\"]\n        \n        diff_cols_a = [f\"B_{i}\" for i in [11, 14, 17]] + [\"D_39\", \"D_131\"] + [f\"S_{i}\" for i in [16, 23]]\n        diff_cols_b = [\"P_2\", \"P_3\"]\n        diff_feat = dataset[diff_cols_a + diff_cols_b]\n        for a in diff_cols_a:\n            for b in diff_cols_b:\n                    diff_feat[f\"{a}-{b}\"] = diff_feat[a] - diff_feat[b]\n        diff_feat.drop(diff_cols_a + diff_cols_b, axis=1, inplace=True)\n        diff_feat = diff_feat.groupby(level='customer_ID').agg(['first', 'last', 'mean', 'std']).pipe(flatten_columns)\n\n        dataset = cudf.concat([cat_feat, num_feat, diff_feat], axis=1)\n        return dataset\n\n\ndef predict(customers, rows, num_cust):\n    skip_rows = 0\n    skip_cust = 0\n    test_preds = []\n\n    for k in range(len(rows)):\n        print(\"#\" * 25)\n        print(f\"### {k}\")\n        print(\"#\" * 25)\n        \n        test = cudf.read_parquet(\n            \"../input/amex-data-integer-dtypes-parquet-format/test.parquet\",\n            skiprows=skip_rows, num_rows=rows[k]\n        )\n\n        test['customer_ID'] = test['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\n        test['S_2'] = cudf.to_datetime(test['S_2'])\n        test.set_index(['customer_ID', 'S_2'], inplace=True)\n\n        gc.collect()\n        skip_rows += rows[k]\n\n        test = engineer(test, 7)\n        \n        if k == len(rows) - 1:\n            test = test.loc[customers[skip_cust:]]\n        else:\n            test = test.loc[customers[skip_cust : skip_cust + num_cust]]\n        \n        skip_cust += num_cust\n\n        # Prepare data for inference\n        dtest = xgb.DMatrix(data=test)\n        gc.collect()\n\n        # Compute predictions and average blend all fold models\n        model = xgb.Booster()\n        model.load_model(f\"xgb_fold0_seed42.xgb\")\n        preds = model.predict(dtest)\n        for f in range(1, 5):\n            model.load_model(f\"xgb_fold{f}_seed42.xgb\")\n            preds += model.predict(dtest, iteration_range=(0, model.best_iteration + 1))\n        preds /= 5\n        test_preds.append(preds)\n\n        # Cleanup\n        del dtest, model\n        _ = gc.collect()\n\n    return test_preds\n\n\ndef main():\n    test_customers = cudf.read_parquet(\n        \"../input/amex-data-integer-dtypes-parquet-format/test.parquet\",\n        columns=['customer_ID']\n    )\n    test_customers[\"customer_ID\"] = test_customers[\"customer_ID\"].str[-16:].str.hex_to_int().astype(\"int64\")\n    \n    def get_rows(customers, test, num_parts):\n        \"\"\"Divides the test dataset in `num_parts` parts.\n        Each part contains approximately `chunk` customers.\n        Returns the number of rows and then number of customers in\n        each part, except the last which has fewer.\n        \"\"\"\n        chunk = len(customers) // num_parts\n        rows = []\n\n        for k in range(num_parts):\n            if k == num_parts - 1:\n                cc = customers[k * chunk :]\n            else:\n                cc = customers[k * chunk : (k + 1) * chunk]\n\n            s = test.loc[test.customer_ID.isin(cc)].shape[0]\n            rows.append(s)\n\n        return rows, chunk\n    \n    customers = test_customers[[\"customer_ID\"]].drop_duplicates().sort_index().values.flatten()\n    rows, num_cust = get_rows(customers, test_customers[[\"customer_ID\"]], num_parts=10)\n    \n    test_preds = predict(customers, rows, num_cust)\n    \n    test_preds = np.concatenate(test_preds)\n    test = cudf.DataFrame(index=customers, data={\"prediction\": test_preds})\n    sub = cudf.read_csv(\"../input/amex-default-prediction/sample_submission.csv\")[\n        [\"customer_ID\"]\n    ]\n    sub[\"customer_ID_hash\"] = sub[\"customer_ID\"].str[-16:].str.hex_to_int().astype(\"int64\")\n    sub = sub.set_index(\"customer_ID_hash\")\n    sub = sub.merge(test[[\"prediction\"]], left_index=True, right_index=True, how=\"left\")\n    sub = sub.reset_index(drop=True)\n\n    # Display predictions\n    sub.to_csv(f\"submission_xgb.csv\", index=False)\n    print(\"Submission file shape is\", sub.shape)\n\n\nmain()","metadata":{"execution":{"iopub.status.busy":"2022-07-26T18:56:44.447711Z","iopub.execute_input":"2022-07-26T18:56:44.448075Z","iopub.status.idle":"2022-07-26T19:02:31.288479Z","shell.execute_reply.started":"2022-07-26T18:56:44.448047Z","shell.execute_reply":"2022-07-26T19:02:31.287455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -al","metadata":{"execution":{"iopub.status.busy":"2022-07-26T19:02:37.778202Z","iopub.execute_input":"2022-07-26T19:02:37.778542Z","iopub.status.idle":"2022-07-26T19:02:38.478928Z","shell.execute_reply.started":"2022-07-26T19:02:37.778514Z","shell.execute_reply":"2022-07-26T19:02:38.477807Z"},"trusted":true},"execution_count":null,"outputs":[]}]}