{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport gc\nimport os\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\nimport itertools\nimport random\n\nimport joblib\nimport numpy as np\nimport pandas as pd\nimport scipy as sp\n\npd.set_option(\"display.max_rows\", 500)\npd.set_option(\"display.max_columns\", 500)\npd.set_option(\"display.width\", 1000)\nimport gc\nimport warnings\nfrom itertools import combinations\n\nimport lightgbm as lgb\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom tqdm.auto import tqdm\n\nwarnings.filterwarnings(\"ignore\")\nimport numpy as np\nimport pandas as pd\nimport scipy as sp\n\npd.set_option(\"display.max_rows\", 500)\npd.set_option(\"display.max_columns\", 500)\npd.set_option(\"display.width\", 1000)\nimport itertools\n\nfrom tqdm.auto import tqdm\n\n\n# ====================================================\n# Read & preprocess data and save it to disk\n# ====================================================\ndef read_preprocess_data(path):\n    train = pd.read_parquet(path + \"/train.parquet\")\n    features = train.drop([\"customer_ID\", \"S_2\"], axis=1).columns.to_list()\n    cat_features = [\n        \"B_30\",\n        \"B_38\",\n        \"D_114\",\n        \"D_116\",\n        \"D_117\",\n        \"D_120\",\n        \"D_126\",\n        \"D_63\",\n        \"D_64\",\n        \"D_66\",\n        \"D_68\",\n    ]\n    num_features = [col for col in features if col not in cat_features]\n    print(\"Starting training feature engineer...\")\n    train_num_agg = train.groupby(\"customer_ID\")[num_features].agg(\n        [\"mean\", \"std\", \"min\", \"max\", \"last\"]\n    )\n    train_num_agg.columns = [\"_\".join(x) for x in train_num_agg.columns]\n    train_num_agg.reset_index(inplace=True)\n    train_cat_agg = train.groupby(\"customer_ID\")[cat_features].agg(\n        [\"count\", \"last\", \"nunique\"]\n    )\n    train_cat_agg.columns = [\"_\".join(x) for x in train_cat_agg.columns]\n    train_cat_agg.reset_index(inplace=True)\n\n    train_labels = pd.read_csv('../input/amex-default-prediction/train_labels.csv')\n    train = train_num_agg.merge(train_cat_agg, how=\"inner\", on=\"customer_ID\").merge(\n        train_labels, how=\"inner\", on=\"customer_ID\"\n    )\n    del train_num_agg, train_cat_agg\n    gc.collect()\n    test = pd.read_parquet(path + \"/test.parquet\")\n    print(\"Starting test feature engineer...\")\n    test_num_agg = test.groupby(\"customer_ID\")[num_features].agg(\n        [\"mean\", \"std\", \"min\", \"max\", \"last\"]\n    )\n    test_num_agg.columns = [\"_\".join(x) for x in test_num_agg.columns]\n    test_num_agg.reset_index(inplace=True)\n    test_cat_agg = test.groupby(\"customer_ID\")[cat_features].agg(\n        [\"count\", \"last\", \"nunique\"]\n    )\n    test_cat_agg.columns = [\"_\".join(x) for x in test_cat_agg.columns]\n    test_cat_agg.reset_index(inplace=True)\n    test = test_num_agg.merge(test_cat_agg, how=\"inner\", on=\"customer_ID\")\n    del test_num_agg, test_cat_agg\n    gc.collect()\n    # Save files to disk\n    train.to_parquet(\"train_fe.parquet\")\n    test.to_parquet(\"test_fe.parquet\")\n\n\n# Read & Preprocess Data\nread_preprocess_data(\"../input/amex-data-integer-dtypes-parquet-format/\")","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.16385,"end_time":"2022-06-19T23:15:37.941388","exception":false,"start_time":"2022-06-19T23:15:37.777538","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-27T08:00:41.068980Z","iopub.execute_input":"2022-06-27T08:00:41.071659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Configurations\n# ====================================================\nclass CFG:\n    input_dir = \"./\"\n    seed = 42\n    n_folds = 6\n    target = \"target\"\n\n\n# ====================================================\n# Seed everything\n# ====================================================\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n\n\n# ====================================================\n# Read data\n# ====================================================\ndef read_data():\n    train = pd.read_parquet(CFG.input_dir + \"train_fe.parquet\")\n    test = pd.read_parquet(CFG.input_dir + \"test_fe.parquet\")\n    return train, test\n\n\n# ====================================================\n# Amex metric\n# ====================================================\ndef amex_metric(y_true, y_pred):\n    labels = np.transpose(np.array([y_true, y_pred]))\n    labels = labels[labels[:, 1].argsort()[::-1]]\n    weights = np.where(labels[:, 0] == 0, 20, 1)\n    cut_vals = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four = np.sum(cut_vals[:, 0]) / np.sum(labels[:, 0])\n    gini = [0, 0]\n    for i in [1, 0]:\n        labels = np.transpose(np.array([y_true, y_pred]))\n        labels = labels[labels[:, i].argsort()[::-1]]\n        weight = np.where(labels[:, 0] == 0, 20, 1)\n        weight_random = np.cumsum(weight / np.sum(weight))\n        total_pos = np.sum(labels[:, 0] * weight)\n        cum_pos_found = np.cumsum(labels[:, 0] * weight)\n        lorentz = cum_pos_found / total_pos\n        gini[i] = np.sum((lorentz - weight_random) * weight)\n    return 0.5 * (gini[1] / gini[0] + top_four)\n\n\n# ====================================================\n# LGBM amex metric\n# ====================================================\ndef lgb_amex_metric(y_pred, y_true):\n    y_true = y_true.get_label()\n    return \"amex_metric\", amex_metric(y_true, y_pred), True\n\n\nimport pickle\n\nclass DartEarlyStopping(object):\n    def __init__(self, data_name, monitor_metric, stopping_round):\n        self.data_name = data_name\n        self.monitor_metric = monitor_metric\n        self.stopping_round = stopping_round\n        self.best_score = None\n        self.best_model = None\n        self.best_score_list = []\n        self.best_iter = 0\n\n    def _is_higher_score(self, metric_score, is_higher_better):\n        if self.best_score is None:\n            return True\n        return (self.best_score < metric_score) if is_higher_better else (self.best_score > metric_score)\n\n    def _deepcopy(self, x):\n        return pickle.loads(pickle.dumps(x))\n\n    def __call__(self, env):\n        evals = env.evaluation_result_list\n        for data, metric, score, is_higher_better in evals:\n            if data != self.data_name or metric != self.monitor_metric:\n                continue\n            if not self._is_higher_score(score, is_higher_better):\n                if env.iteration - self.best_iter > self.stopping_round:\n                    eval_result_str = '\\t'.join([lgb.callback._format_eval_result(x) for x in self.best_score_list])\n                    print(f\"Early stopping, best iteration is:\\n[{self.best_iter+1}]\\t{eval_result_str}\") \n                    print(f\"You can get best model by \\\"DartEarlyStopping.best_model\\\"\")\n                    raise lgb.callback.EarlyStopException(self.best_iter, self.best_score_list)\n                return\n\n            self.best_model = self._deepcopy(env.model)\n            self.best_iter = env.iteration\n            self.best_score_list = evals\n            self.best_score = score\n            return\n        raise ValueError(\"monitoring metric not found\")\n\n\n\n# ====================================================\n# Train & Evaluate\n# ====================================================\ndef train_and_evaluate(train, test):\n    # Label encode categorical features\n    cat_features = [\n        \"B_30\",\n        \"B_38\",\n        \"D_114\",\n        \"D_116\",\n        \"D_117\",\n        \"D_120\",\n        \"D_126\",\n        \"D_63\",\n        \"D_64\",\n        \"D_66\",\n        \"D_68\",\n    ]\n    cat_features = [f\"{cf}_last\" for cf in cat_features]\n    for cat_col in cat_features:\n        encoder = LabelEncoder()\n        train[cat_col] = encoder.fit_transform(train[cat_col])\n        test[cat_col] = encoder.transform(test[cat_col])\n    # Round last float features to 2 decimal place\n    num_cols = list(\n        train.dtypes[(train.dtypes == \"float32\") | (train.dtypes == \"float64\")].index\n    )\n    num_cols = [col for col in num_cols if \"last\" in col]\n    for col in num_cols:\n        train[col + \"_round2\"] = train[col].round(2)\n        test[col + \"_round2\"] = test[col].round(2)\n    # Get feature list\n    features = [col for col in train.columns if col not in [\"customer_ID\", CFG.target]]\n    params = {\n        'objective': 'binary',\n        'metric': \"binary_logloss\",\n        'boosting': 'dart',\n        'seed': CFG.seed,\n        'num_leaves': 100,\n        'learning_rate': 0.01,\n        'feature_fraction': 0.20,\n        'bagging_freq': 10,\n        'bagging_fraction': 0.50,\n        'n_jobs': -1,\n        'lambda_l2': 2,\n        'min_data_in_leaf': 40\n        }\n    \n    # Create a numpy array to store test predictions\n    test_predictions = np.zeros(len(test))\n    # Create a numpy array to store out of folds predictions\n    oof_predictions = np.zeros(len(train))\n    kfold = StratifiedKFold(n_splits=CFG.n_folds, shuffle=True, random_state=CFG.seed)\n    for fold, (trn_ind, val_ind) in enumerate(kfold.split(train, train[CFG.target])):\n        print(\" \")\n        print(\"-\" * 50)\n        print(f\"Training fold {fold} with {len(features)} features...\")\n        x_train, x_val = train[features].iloc[trn_ind], train[features].iloc[val_ind]\n        y_train, y_val = (\n            train[CFG.target].iloc[trn_ind],\n            train[CFG.target].iloc[val_ind],\n        )\n        lgb_train = lgb.Dataset(x_train, y_train, categorical_feature=cat_features)\n        lgb_valid = lgb.Dataset(x_val, y_val, categorical_feature=cat_features)\n\n        es = DartEarlyStopping(\"valid_0\", \"amex_metric\", stopping_round=500)\n        model = lgb.train(\n            params=params,\n            train_set=lgb_train,\n            num_boost_round=5000,\n            valid_sets=[lgb_valid],\n            early_stopping_rounds=100,\n            callbacks=[es],\n            verbose_eval=10,\n            feval=lgb_amex_metric,\n        )\n\n        # Save best model\n        joblib.dump(\n            model,\n            f\"lgbm_fold{fold}_seed{CFG.seed}.pkl\",\n        )\n        # Predict validation\n        val_pred = model.predict(x_val)\n        # Add to out of folds array\n        oof_predictions[val_ind] = val_pred\n        # Predict the test set\n        test_pred = model.predict(test[features])\n        test_predictions += test_pred / CFG.n_folds\n        # Compute fold metric\n        score = amex_metric(y_val, val_pred)\n        print(f\"Our fold {fold} CV score is {score}\")\n        del x_train, x_val, y_train, y_val, lgb_train, lgb_valid\n        gc.collect()\n    \n    # Compute out of folds metric\n    score = amex_metric(train[CFG.target], oof_predictions)\n    print(f\"Our out of folds CV score is {score}\")\n    # Create a dataframe to store out of folds predictions\n    oof_df = pd.DataFrame(\n        {\n            \"customer_ID\": train[\"customer_ID\"],\n            \"target\": train[CFG.target],\n            \"prediction\": oof_predictions,\n        }\n    )\n    oof_df.to_csv(\n        f\"oof_lgbm_baseline_{CFG.n_folds}fold_seed{CFG.seed}.csv\",\n        index=False,\n    )\n    # Create a dataframe to store test prediction\n    test_df = pd.DataFrame(\n        {\"customer_ID\": test[\"customer_ID\"], \"prediction\": test_predictions}\n    )\n    test_df.to_csv(\n        f\"test_lgbm_baseline_{CFG.n_folds}fold_seed{CFG.seed}.csv\",\n        index=False,\n    )\n\nseed_everything(CFG.seed)\ntrain, test = read_data()\ntrain_and_evaluate(train, test)","metadata":{"execution":{"iopub.execute_input":"2022-06-27T07:46:05.258857Z","iopub.status.busy":"2022-06-27T07:46:05.258620Z","iopub.status.idle":"2022-06-27T07:55:08.228344Z","shell.execute_reply":"2022-06-27T07:55:08.227655Z","shell.execute_reply.started":"2022-06-27T07:46:05.258840Z"},"papermill":{"duration":1.91354,"end_time":"2022-06-19T23:15:39.862701","exception":false,"start_time":"2022-06-19T23:15:37.949161","status":"completed"},"scrolled":true,"tags":[]},"execution_count":null,"outputs":[]}]}