{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":35332,"databundleVersionId":3723648,"sourceType":"competition"},{"sourceId":3910635,"sourceType":"datasetVersion","datasetId":2314156},{"sourceId":7343067,"sourceType":"datasetVersion","datasetId":4256784}],"dockerImageVersionId":30558,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport datetime\nimport pickle\nimport matplotlib.pyplot as plt\nfrom colorama import Fore, Back, Style\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score, precision_score, recall_score, auc\nfrom sklearn.ensemble import RandomForestClassifier\nimport optuna, gc\n\nimport lightgbm as lgb\nfrom boruta import BorutaPy\n#pip install Boruta","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-10T22:11:22.788066Z","iopub.execute_input":"2024-01-10T22:11:22.788729Z","iopub.status.idle":"2024-01-10T22:11:27.231874Z","shell.execute_reply.started":"2024-01-10T22:11:22.788693Z","shell.execute_reply":"2024-01-10T22:11:27.230921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read the single statement data including the mentioned lag features: \ndf_train_lag = pd.read_parquet(\"/kaggle/input/amex-fe/train_fe_plus_plus.parquet\").rename(\n    {'D_63_count':'ntot_statement'}, axis = 1\n)\nprint(df_train_lag.info())\ndf_train_lag.head(3)\n\n# Create validation set:\ndf_valid_lag = df_train_lag.sample(frac=0.2, random_state = 42)\ndf_train_lag = df_train_lag[~df_train_lag.customer_ID.isin(df_valid_lag.customer_ID.tolist())]","metadata":{"execution":{"iopub.status.busy":"2024-01-02T15:34:47.300386Z","iopub.execute_input":"2024-01-02T15:34:47.300698Z","iopub.status.idle":"2024-01-02T15:35:12.678869Z","shell.execute_reply.started":"2024-01-02T15:34:47.300674Z","shell.execute_reply":"2024-01-02T15:35:12.677865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prgauc_metric(y_true, y_prob, sample_weight = None, plot = False):\n    '''Function to obtain the Precision Recall Gain Area under the Curve (PRG AUC). The baseline is computed from\n    the mean of y_true. Points with non-infinite recall or precision are plotted (TP != 0) (our decision-making).\n    Only points with the recall gain in the 1st quadrant (recall gain >= 0) are plotted (according to the reference).\n    Additionally, we do the manual inclusion of the [recall gain = 0, precision gain = max(precision gain)], to make\n    the curve artificially touch the y-axis to obtain a better PRG AUC approximation (our decision-making).\n    References:\n    https://research-information.bris.ac.uk/ws/portalfiles/portal/72164009/5867_precision_recall_gain_curves_pr_analysis_done_right.pdf\n    \n    The function computes the precision gain (_precG_) and the recall gain (_recG_), and computes the area under the \n    precision_gain(recall_gain). The baseline of this metric (random model) is 0.50 (similar interpretation to the \n    ROC AUC). According to the reference, the PRGAUC is prefered over the typical PR AUC since it has better\n    interpretability (PR AUC has reference the y_true avg., making the metric harder to evaluate different datasets\n    with different baselines). Like the PR AUC, the PRG AUC is better than the regular ROC AUC for imbalanced\n    datasets (ROC AUC gives equal weight to the positive and negative class, given the impression the model is\n    performing well even if the identification of the positive class is not so good). \n    \n    _precG_ = (precision - y_true avg.)/[(1 - y_true avg.)*precision] = 1 - (y_true avg.*FP)/[(1 - y_true avg.)*TP]\n    \n    _recG_ = (recall - y_true avg.)/[(1 - y_true avg.)*recall] = 1 - (y_true avg.*FN)/[(1 - y_true avg.)*TP]\n    \n    Arguments of the function:\n        y_true: (list, pd.Series) vector with the true labels;\n        y_prob: (list, pd.Series) vector with the predicted probabilidades of the positive class;\n        sample_weight: (pd.DataFrame) df with sample weights, with index in concordance with the y_true (case when this func\n        is called as the scoring of sklearn.model_selection.cross_val_score;\n        plot: (boolean) if True, plot the Precision-Recall Gain curve, along with the PRG AUC;\n    Package dependencies:\n        np (numpy)\n        auc (sklearn.metrics.auc)\n        plt (matplotlib.pyplot)\n        precision_score (sklearn-metrics.precision_score)\n        recall_score (sklearn-metrics.recall_score)\n    '''\n    \n    # Build threshold vectors (more granular between 0-5% and 95-100%, 5-95% the step is +1%):\n    \n    main_thr = 0.002\n    # main_thr was obtained by weighting in the runtime and the PRG AUC approximation (assuming main_thr -> 0)\n    # gives the true value of PRG AUC. Additionally, below we do the manual inclusion of the \n    # [recall gain = 0, precision gain = max(precision gain)], to make the curve artificially touch the y-axis\n    # and obtain a better PRG AUC approximation.\n    thr_vec = [0] + list(np.arange(0.0005, 0.01, 0.0015)) + list(np.arange(0.01, 0.05, 0.005)) + list(np.arange(0.05, 0.95, main_thr)) + list(np.arange(0.95, 0.99, 0.005)) + list(np.arange(0.99, 1, 0.0015)) + [1]\n    thr_vec = [round(i, 6) for i in thr_vec]\n    # (copy this code and print thr_vec to visualize the threshold values)\n    \n    # Baseline:\n    baseline_mean = np.mean(y_true) #(between 0 - 1)\n    \n    # Lists do save the precision gain and recall gain values:\n    precG_vec, recG_vec, thr_used_vec = [], [], []\n    \n    # Prepare sample_weight:\n    if sample_weight is not None:\n        if isinstance(sample_weight, pd.DataFrame) and (isinstance(y_true, pd.DataFrame) or isinstance(y_true, pd.Series)):\n            sample_weight = sample_weight=sample_weight.loc[y_true.index.values].values.reshape(-1)\n        else: #list case (just in case)\n            sample_weight = sample_weight\n        \n    for thr in thr_vec:\n        \n        # Build predictions vector for the thr:\n        pred_vec = [(0 if prob < thr else 1) for prob in y_prob]\n        \n        FP, TP, FN = 0, 0, 0\n        for pred, y in zip(pred_vec, y_true):\n            FP += 1 if pred == 1 and y == 0 else 0 # Build FP\n            TP += 1 if pred == 1 and y == 1 else 0 # Build TP\n            FN += 1 if pred == 0 and y == 1 else 0 # Build FN\n        \n        # Exclude infinite points:\n        if (TP) > 0:\n            # Use precision_score and recall_score to take advantage of the sample_weight argument:\n            precision = precision_score(y_true, pred_vec, sample_weight=sample_weight)\n            recall = recall_score(y_true, pred_vec, sample_weight=sample_weight)\n\n            #_precG_ = 1 - (baseline_mean*FP)/((1 - baseline_mean)*TP)\n            #_recG_ = 1 - (baseline_mean*FN)/((1 - baseline_mean)*TP)\n            _precG_ = (precision - baseline_mean)/((1 - baseline_mean)*precision)\n            _recG_ = (recall - baseline_mean)/((1 - baseline_mean)*recall)\n            \n            # Append to lists:\n            precG_vec.append(_precG_)\n            recG_vec.append(_recG_)\n            thr_used_vec.append(thr)\n    \n    # Create dataframe with accepted thr, precision gain and recall gain:\n    df_metrics = pd.DataFrame({'thr':thr_used_vec, 'precG':precG_vec, 'recG':recG_vec})\n    \n    # According to the reference (see the function's help), we should only consider the points with \n    df_metrics = df_metrics[df_metrics.recG >= 0]\n    \n    # Add the point for recall = 0 (according to the used threshold step, there might be a pice of the \n    # graph missing):\n    df_metrics = pd.concat([df_metrics, pd.DataFrame({'thr':[df_metrics.thr.max()], 'precG':[df_metrics.precG.max()], 'recG':[0]})])\n    df_metrics = df_metrics.sort_values(['recG'])\n    #display(df_metrics)\n       \n    # Compute area under the curve:\n    prg_auc = auc(x = df_metrics.recG, y = df_metrics.precG)\n    \n    # Graph plot:\n    if plot == True:\n        \n        plt.plot(df_metrics.recG, df_metrics.precG, label='PRG ({}%)'.format(round(prg_auc*100, 2)))\n        plt.plot([0,1], [1,0], linestyle = '--', color='gray', label='Rand. model (50%)')\n        plt.plot([0,1,1], [1,1,0], linestyle = '--', color='green', label='Perf. model (100%)')\n        plt.plot(df_metrics.recG, df_metrics.thr, label='Thresh. used')\n        \n        plt.title(\"PRG curve + PRG AUC\")\n        plt.xlabel(\"Recall Gain\")\n        plt.ylabel(\"Precision Gain\")\n        plt.legend()\n        plt.grid()\n        plt.show()\n    \n    return prg_auc","metadata":{"execution":{"iopub.status.busy":"2024-01-02T15:39:53.348850Z","iopub.execute_input":"2024-01-02T15:39:53.349173Z","iopub.status.idle":"2024-01-02T15:39:53.369829Z","shell.execute_reply.started":"2024-01-02T15:39:53.349149Z","shell.execute_reply":"2024-01-02T15:39:53.368739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndef amex_metric(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n\n    def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        y_true_pred = y_true.rename(columns={'target': 'prediction'})\n        return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n    g = normalized_weighted_gini(y_true, y_pred)\n    d = top_four_percent_captured(y_true, y_pred)\n\n    return 0.5 * (g + d)\n'''\n# @yunchonggan's fast metric implementation\n# From https://www.kaggle.com/competitions/amex-default-prediction/discussion/328020\ndef amex_metric(y_true: np.array, y_pred: np.array) -> float:\n\n    # count of positives and negatives\n    n_pos = y_true.sum()\n    n_neg = y_true.shape[0] - n_pos\n\n    # sorting by descring prediction values\n    indices = np.argsort(y_pred)[::-1]\n    preds, target = y_pred[indices], y_true[indices]\n\n    # filter the top 4% by cumulative row weights\n    weight = 20.0 - target * 19.0\n    cum_norm_weight = (weight / weight.sum()).cumsum()\n    four_pct_filter = cum_norm_weight <= 0.04\n\n    # default rate captured at 4%\n    d = target[four_pct_filter].sum() / n_pos\n\n    # weighted gini coefficient\n    lorentz = (target / n_pos).cumsum()\n    gini = ((lorentz - cum_norm_weight) * weight).sum()\n\n    # max weighted gini coefficient\n    gini_max = 10 * n_neg * (1 - 19 / (n_pos + 20 * n_neg))\n\n    # normalized weighted gini coefficient\n    g = gini / gini_max\n\n    return 0.5 * (g + d)","metadata":{"execution":{"iopub.status.busy":"2024-01-02T16:38:52.300995Z","iopub.execute_input":"2024-01-02T16:38:52.301854Z","iopub.status.idle":"2024-01-02T16:38:52.312062Z","shell.execute_reply.started":"2024-01-02T16:38:52.301819Z","shell.execute_reply":"2024-01-02T16:38:52.310980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lgb_amex_metric(y_pred, data):\n    \"\"\"The competition metric with lightgbm's calling convention\"\"\"\n    return ('amex', amex_metric(data.label, y_pred), True)","metadata":{"execution":{"iopub.status.busy":"2024-01-02T16:38:52.452865Z","iopub.execute_input":"2024-01-02T16:38:52.453139Z","iopub.status.idle":"2024-01-02T16:38:52.457637Z","shell.execute_reply.started":"2024-01-02T16:38:52.453117Z","shell.execute_reply":"2024-01-02T16:38:52.456813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lgbm_train(df, params):\n    \n    start = datetime.datetime.now()\n    \n    X = df.drop(columns=['target'])  # Features\n    y = df['target']  # Target variable\n    n_splits = 5\n    amex_scores = []\n    \n    # Initialize StratifiedKFold with the desired number of splits\n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n\n    # Perform stratified k-fold cross-validation\n    fold = 0\n    n_trees = params['n_estimators']\n    params.pop(\"n_estimators\")\n    for train_idx, valid_idx in skf.split(X, y):\n        fold += 1\n        cv_start_time = datetime.datetime.now()\n        X_train, X_valid = X.iloc[train_idx], X.iloc[valid_idx]\n        y_train, y_valid = y.iloc[train_idx], y.iloc[valid_idx]\n\n        # Create LightGBM datasets\n        train_data = lgb.Dataset(X_train, label=y_train)\n        valid_data = lgb.Dataset(X_valid, label=y_valid, reference=train_data)\n\n        # Train the LightGBM model\n        model = lgb.train(params, train_data, valid_sets=[valid_data], num_boost_round = n_trees,\n                          feval=[lgb_amex_metric],\n                          callbacks=[lgb.log_evaluation(period=100)])\n\n        # Metric:\n        prob_valid = model.predict(X_valid)\n        amex = amex_metric(y_valid.values, y_pred = prob_valid)\n        amex_scores.append(amex)\n        \n        print(f\"{Fore.GREEN}{Style.BRIGHT}Fold {fold} | {datetime.datetime.now() - cv_start_time} |\"\n          f\" {n_trees:5} trees |\"\n          f\"                Score = {amex:.5f}{Style.RESET_ALL}\")\n        gc.collect()\n\n    mean_amex = np.mean(amex_scores)\n    print(f\"Mean AMEX across {n_splits} folds: {mean_amex:.4f}\")\n    print(f\"Runtime {n_splits} folds:\", datetime.datetime.now() - start)\n    return mean_amex","metadata":{"execution":{"iopub.status.busy":"2024-01-02T16:38:52.642072Z","iopub.execute_input":"2024-01-02T16:38:52.642408Z","iopub.status.idle":"2024-01-02T16:38:52.653245Z","shell.execute_reply.started":"2024-01-02T16:38:52.642381Z","shell.execute_reply":"2024-01-02T16:38:52.652353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def objective(trial, df, gpu_settings = True):\n    params = {\n        'objective': 'binary',\n        'metric': 'binary_logloss',\n        'boosting_type': 'gbdt',\n        'n_jobs': -1,\n        'random_state': 42,\n        'verbose':-1,\n        \n        'num_leaves': 95,\n        'bagging_fraction': 0.8,\n        'max_depth': -1, #2**trial.suggest_int('num_leaves', 1, 8, 1),\n        #'reg_alpha': 1e-3,\n        \n            #'learning_rate': trial.suggest_float('learning_rate', 0.0025, 0.1), #V1\n            #'learning_rate': trial.suggest_float('learning_rate', 0.008, 0.025), #V2\n        'learning_rate': trial.suggest_float('learning_rate', 0.009, 0.016), #V3\n            #'feature_fraction': trial.suggest_float('feature_fraction', 0.2, 0.4), #V1\n            #'feature_fraction': trial.suggest_float('feature_fraction', 0.2, 0.25), #V2\n        'feature_fraction': trial.suggest_float('feature_fraction', 0.195, 0.215), #V3\n            #'n_estimators':trial.suggest_int('n_estimators', 1000, 2400, 100), #V1\n            #'n_estimators':trial.suggest_int('n_estimators', 900, 3100, 100), #V2\n        'n_estimators':trial.suggest_int('n_estimators', 1950, 2650, 50), #V3\n            #'reg_lambda': trial.suggest_categorical('reg_lambda', [1e-3, 1, 50]), #V1\n            #'reg_lambda': trial.suggest_categorical('reg_lambda', [1e-3, 1e-2, 1]), #V2\n        'reg_lambda': trial.suggest_categorical('reg_lambda', [0.005, 0.01, 0.02, 0.05]), #V3\n            #'min_child_samples': trial.suggest_categorical('min_child_samples', [2400, 2400*2]) #V1\n            #'min_child_samples': trial.suggest_categorical('min_child_samples', [1200, 2400, 2400*2]) #V2\n        'min_child_samples': trial.suggest_int('min_child_samples', 650, 1350, 50), #V3\n        'reg_alpha': trial.suggest_categorical('reg_alpha', [0, 0.001]) # V3 (added this hyperp.)\n        #'max_bin': trial.suggest_int('max_bin', 32, 256)\n    }\n    if gpu_settings:\n        params['device'] = 'gpu'\n        params['gpu_platform_id'] = 0\n        params['gpu_device_id'] = 0\n    \n    # Train lgbm with params and cross-validation:\n    amex_cv = lgbm_train(df, params)\n    return amex_cv","metadata":{"execution":{"iopub.status.busy":"2024-01-02T16:39:43.155132Z","iopub.execute_input":"2024-01-02T16:39:43.156056Z","iopub.status.idle":"2024-01-02T16:39:43.163935Z","shell.execute_reply.started":"2024-01-02T16:39:43.156023Z","shell.execute_reply":"2024-01-02T16:39:43.162895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def optuna_opt(df, df_valid, weights_01 = [20,1], n_trials = 100, gpu_settings = True):\n    # Create a study object and optimize the objective function\n    clock_start = datetime.datetime.now()\n    gc.collect() # clear files / free memory\n    print(f\">> Start time: {clock_start}\")\n    sampler = optuna.samplers.TPESampler(seed=42) \n    study = optuna.create_study(direction='maximize', sampler = sampler)\n    study.optimize(\n        lambda trial: objective(\n            trial,\n            df,\n            gpu_settings = gpu_settings\n        ),\n        n_trials=n_trials\n    )\n    clock_end = datetime.datetime.now()\n    \n    # Get the best parameters\n    best_params = study.best_params\n    best_value = study.best_value\n    best_value_iteration = study.best_trial.number\n    print(f\"[Runtime:{clock_end - clock_start}] Best Value={best_params} at iteration={best_value_iteration} with\\n parameters={best_params}.\")\n    \n    results = {\n        \"study\":study, \"best_params\":best_params, \"best_value\":best_value,\n        \"best_value_iteration\":best_value_iteration\n    }\n    \n    # Retrain model with best_params and all train data:\n    # > Logic: if best_params are the parameters that best generalize to out of sample data\n    #          then to boost performance one needs only to increase training data quantity.\n    retrain_params = results['best_params']\n\n    retrain_params['objective'] ='binary'\n    retrain_params['metric'] ='binary_logloss'\n    retrain_params['boosting_type'] ='gbdt'\n    retrain_params['n_jobs'] = -1\n    retrain_params['random_state'] = 42\n    retrain_params['verbose'] = -1\n    \n    retrain_params['num_leaves'] = 95\n    retrain_params['max_depth'] = 0.8\n    retrain_params['max_depth'] = -1\n    \n    if gpu_settings:\n        retrain_params['device'] = 'gpu'\n        retrain_params['gpu_platform_id'] = 0\n        retrain_params['gpu_device_id'] = 0\n    \n    train_data = lgb.Dataset(df.drop([\"target\"], axis = 1), label=df['target'])\n    best_model = lgb.train(\n        retrain_params,\n        train_data,\n        callbacks=[\n            lgb.log_evaluation(period=100)\n        ]\n    )\n    # Eval metrics on hood out validation:\n    prob_valid = best_model.predict(df_valid.drop(['target'], axis = 1))\n    y_valid = df_valid['target']\n    prgauc = prgauc_metric(y_valid, prob_valid, sample_weight = y_valid.apply(lambda x: weights_01[0] if x == 0 else weights_01[1]))\n    amex = amex_metric(y_valid.values, prob_valid)\n    gc.collect()\n    \n    results['best_model'] = best_model\n    results['valid_metrics'] = {'prgauc':prgauc, 'amex':amex}\n    print(f\"Final metrics -> prgauc:{prgauc}, amex:{amex}\")\n    return results","metadata":{"execution":{"iopub.status.busy":"2024-01-02T16:39:45.960498Z","iopub.execute_input":"2024-01-02T16:39:45.960874Z","iopub.status.idle":"2024-01-02T16:39:45.975758Z","shell.execute_reply.started":"2024-01-02T16:39:45.960846Z","shell.execute_reply":"2024-01-02T16:39:45.974640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## A. Preprocessing: imputation and feature selection (correlation check + _boruta_)\n\nTo increase performance (and reduce runtime) we will perform feature selection:\n* remove features with absolute correlation above |0.95|;\n* use `boruta` to conduct feature selection ([source](https://github.com/scikit-learn-contrib/boruta_py)).\n\nBoruta doesn't work with missing values - I will have to impute them. Since modelling wise (and we will use random forests with boruta) uses decision trees, we will impute NaN with an out of distribution values (-999999). `inf`, `-inf`  (infinite values; div features have them) will be replaced with 99999999.","metadata":{}},{"cell_type":"code","source":"'''\n# To reduce runtime, sample 10000 observations to calculate feature correlations:\ndf_train_extraction = df_train_lag.drop(['customer_ID', 'target'], axis = 1).sample(\n    10000, random_state = 42)\ndf_corr = df_train_extraction.corr()\n# Set to NaN upper triangle values:\ncorrelation_matrix = df_corr.where(np.tril(np.ones(df_corr.shape), k=-1).astype(bool))\nmodified_correlation_df = pd.DataFrame(\n    correlation_matrix, columns=df_corr.columns, index=df_corr.columns)\n\n# Create dataframe with a single column (corr_value) and \"feat1_X_feat2\" as index:\nmodified_correlation_df2 = modified_correlation_df.stack() \nfinal_corr = pd.DataFrame(modified_correlation_df2, columns = ['corr_value']).reset_index()\nfinal_corr.index = final_corr.apply(lambda row: f'{row[\"level_0\"]}_X_{row[\"level_1\"]}', axis = 1)\nfinal_corr = final_corr.drop(['level_0', 'level_1'], axis = 1).sort_values(['corr_value'],\n                                                                           ascending = [False], axis = 0)\n#display(final_corr)\n\n# Consider only features above <thr> to be removed from data:\nthr = 0.95\nfinal_corr['corr_value_abs'] = final_corr.corr_value.apply(lambda x: abs(x))\nfinal_corr = final_corr.sort_values(['corr_value_abs'], axis = 0)\nfinal_corr_thr = final_corr[final_corr.corr_value_abs >= thr]\nprint(f\"Number of pairs above absolute threshold {thr*100}%: {len(final_corr_thr)/len(final_corr)*100}%\")\n\n# For each pair feat1 X feat2, remove feat2 if not previously removed:\nto_high_corr_feat = []\nfor pair_ix in range(len(final_corr_thr)):\n    line = final_corr_thr.iloc[pair_ix,:]\n    feat1, feat2 = line.name.split(\"_X_\")\n    if feat2 not in to_high_corr_feat:\n        to_high_corr_feat.append(feat2)\n#print(to_high_corr_feat)\nprint(f\"Number of features to be removed: {len(to_high_corr_feat)}\")\n'''","metadata":{"execution":{"iopub.status.busy":"2023-12-31T15:41:07.078521Z","iopub.status.idle":"2023-12-31T15:41:07.078951Z","shell.execute_reply.started":"2023-12-31T15:41:07.078767Z","shell.execute_reply":"2023-12-31T15:41:07.078786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_nan = df_train_lag.drop(['customer_ID', 'target'], axis = 1).isna().sum().to_frame().rename(\n    {0:'nan_count'}, axis = 1).sort_values(['nan_count'], axis = 0)\ndf_nan.reset_index(drop=True).plot()\nplt.xlabel(\"#Feature\")\nplt.ylabel(\"#NaN\")\nplt.title(\"NaN Dist in Features:\")\nplt.show()\n\n# Simply impute missing values with -999999:\ndf_train_lag_imp = df_train_lag.fillna(-999999)\ndf_valid_lag_imp = df_valid_lag.fillna(-999999)\n\n# Impute inf values with 99999999\ncol_w_inf = []\nfor col in df_train_lag_imp.drop(['customer_ID'], axis = 1):\n    col_w_inf.append(col)\n    if np.isinf(df_train_lag_imp[col]).any():\n        df_train_lag_imp[col] = df_train_lag_imp[col].replace([np.inf, -np.inf], 99999999)\n        df_valid_lag_imp[col] = df_valid_lag_imp[col].replace([np.inf, -np.inf], 99999999)","metadata":{"execution":{"iopub.status.busy":"2024-01-02T15:35:12.767702Z","iopub.execute_input":"2024-01-02T15:35:12.767951Z","iopub.status.idle":"2024-01-02T15:35:18.339122Z","shell.execute_reply.started":"2024-01-02T15:35:12.767929Z","shell.execute_reply":"2024-01-02T15:35:18.338246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_train_lag, df_valid_lag","metadata":{"execution":{"iopub.status.busy":"2024-01-02T15:35:18.340181Z","iopub.execute_input":"2024-01-02T15:35:18.340486Z","iopub.status.idle":"2024-01-02T15:35:18.357602Z","shell.execute_reply.started":"2024-01-02T15:35:18.340460Z","shell.execute_reply":"2024-01-02T15:35:18.356580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n# Boruta py feature classification:\nrf = RandomForestClassifier(n_jobs=-1, class_weight=None, max_depth=7, random_state=42)\nfeat_selector = BorutaPy(rf, n_estimators='auto', verbose=2, random_state=42)\ndf_boruta = df_train_lag_imp.sample(20000, random_state = 42).drop(\n        ['customer_ID'] + to_high_corr_feat, axis = 1)\nfeat_selector.fit(\n    df_boruta.drop(['target'], axis = 1).values,\n    df_boruta.target.values\n)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-12-31T15:41:07.083942Z","iopub.status.idle":"2023-12-31T15:41:07.084287Z","shell.execute_reply.started":"2023-12-31T15:41:07.084119Z","shell.execute_reply":"2023-12-31T15:41:07.084135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nboruta_results = pd.DataFrame({\n    'feature':df_boruta.drop(['target'], axis = 1).columns.tolist(),\n    'ranking':feat_selector.ranking_, \n    'chosen':feat_selector.support_})\nboruta_results.sort_values(['ranking'], axis = 0)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-12-31T15:41:07.085324Z","iopub.status.idle":"2023-12-31T15:41:07.085691Z","shell.execute_reply.started":"2023-12-31T15:41:07.085486Z","shell.execute_reply":"2023-12-31T15:41:07.085501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Explicit enumeration of boruta results to avoid running the method:\nboruta_selection = ['P_2_std', 'P_2_max', 'P_2_last', 'D_39_mean', 'D_39_max', 'D_39_last', 'B_2_first',\n 'B_2_mean', 'B_2_std', 'B_2_min', 'B_2_max', 'B_2_last', 'R_1_mean', 'R_1_min', 'R_1_max', 'R_1_last',\n 'S_3_first', 'S_3_mean', 'S_3_std', 'S_3_min', 'S_3_max', 'S_3_last', 'D_41_mean', 'D_41_max',\n 'D_41_last', 'B_3_mean', 'B_3_std', 'B_3_min', 'B_3_max', 'B_3_last', 'D_42_std', 'D_42_last',\n 'D_43_first', 'D_43_mean', 'D_43_min', 'D_43_max', 'D_43_last', 'D_44_first', 'D_44_mean', 'D_44_std',\n 'D_44_min', 'D_44_max', 'D_44_last', 'B_4_std', 'B_4_max', 'B_4_last', 'D_45_last', 'B_5_first',\n 'B_5_mean', 'B_5_min', 'B_5_max', 'B_5_last', 'R_2_mean', 'R_2_max', 'D_46_mean', 'D_46_min', 'D_46_max',\n 'D_46_last', 'D_47_last', 'D_48_first', 'D_48_mean', 'D_48_min', 'D_48_max', 'D_48_last', 'D_49_mean',\n 'D_49_std', 'D_49_last', 'B_6_mean', 'B_6_min', 'B_6_max', 'B_6_last', 'B_8_std', 'B_8_max',\n 'D_50_first', 'D_50_min', 'D_50_last', 'D_51_mean', 'D_51_last', 'B_9_first', 'B_9_mean', 'B_9_std',\n 'B_9_min', 'B_9_max', 'B_9_last', 'R_3_first', 'R_3_mean', 'R_3_std', 'R_3_min', 'R_3_max', 'R_3_last',\n 'D_52_first', 'D_52_mean', 'D_52_min', 'D_52_max', 'D_52_last', 'P_3_first', 'P_3_mean', 'P_3_std',\n 'P_3_min', 'P_3_max', 'P_3_last', 'B_10_first', 'B_10_min', 'B_10_max', 'B_10_last', 'D_53_first',\n 'D_53_mean', 'D_53_std', 'D_53_min', 'D_53_last', 'D_54_mean', 'D_54_min', 'D_54_last', 'R_4_mean',\n 'R_4_max', 'S_7_first', 'S_7_mean', 'S_7_std', 'S_7_min', 'S_7_max', 'S_7_last', 'B_12_std', 'B_12_last',\n 'S_8_first', 'S_8_mean', 'S_8_min', 'S_8_max', 'S_8_last', 'D_55_std', 'D_55_min', 'D_55_last',\n 'D_56_std', 'D_56_min', 'D_56_last', 'B_13_min', 'B_13_max', 'B_13_last', 'R_5_mean', 'R_5_max',\n 'R_5_last', 'D_58_std', 'D_58_min', 'D_58_max', 'D_58_last', 'B_14_std', 'B_14_max', 'B_14_last',\n 'D_59_std', 'D_59_min', 'D_59_last', 'D_60_last', 'D_61_first', 'D_61_mean', 'D_61_min', 'D_61_max',\n 'D_61_last', 'S_11_mean', 'S_11_min', 'D_62_first', 'D_62_min', 'D_62_last', 'D_65_mean', 'D_65_max',\n 'B_16_max', 'B_17_mean', 'B_17_min', 'B_17_last', 'B_18_mean', 'B_18_min', 'B_18_max', 'B_18_last',\n 'B_19_mean', 'B_19_std', 'B_19_max', 'B_19_last', 'B_20_mean', 'B_20_max', 'B_20_last', 'R_6_mean',\n 'R_6_max', 'R_6_last', 'S_13_mean', 'S_13_max', 'S_13_last', 'B_21_mean', 'B_21_std', 'B_21_max',\n 'B_21_last', 'D_69_max', 'B_22_mean', 'B_22_std', 'B_22_max', 'B_22_last', 'D_70_mean', 'D_70_std',\n 'D_71_last', 'S_15_first', 'S_15_mean', 'S_15_std', 'S_15_max', 'S_15_last', 'B_23_mean', 'B_23_std',\n 'B_23_min', 'B_23_max', 'B_23_last', 'P_4_mean', 'P_4_max', 'D_75_std', 'D_75_min', 'D_75_last',\n 'B_24_mean', 'B_24_last', 'R_7_mean', 'R_7_max', 'R_7_last', 'D_77_first', 'D_77_mean', 'D_77_min',\n 'D_77_max', 'D_77_last', 'B_25_mean', 'B_25_max', 'B_25_last', 'B_26_mean', 'B_26_max', 'B_26_last',\n 'D_78_mean', 'D_78_std', 'D_78_max', 'D_78_last', 'D_79_mean', 'D_79_std', 'R_8_mean', 'R_8_max',\n 'R_8_last', 'S_16_mean', 'S_16_max', 'S_16_last', 'R_10_mean', 'R_10_max', 'R_11_mean', 'B_28_first',\n 'B_28_std', 'B_28_last', 'R_13_max', 'R_15_mean', 'R_15_max', 'D_84_max', 'D_84_last', 'R_16_mean',\n 'R_16_max', 'B_33_mean', 'B_33_min', 'D_91_first', 'D_91_mean', 'D_91_min', 'S_23_mean', 'S_23_std',\n 'S_23_min', 'S_23_max', 'S_23_last', 'S_24_mean', 'S_24_max', 'S_24_last', 'S_25_mean', 'S_25_min',\n 'S_25_max', 'S_25_last', 'S_26_std', 'S_26_max', 'S_26_last', 'D_106_mean', 'D_106_max', 'D_106_last',\n 'B_36_std', 'B_37_mean', 'B_37_std', 'B_37_min', 'B_37_max', 'B_37_last', 'R_26_last', 'R_27_last',\n 'D_112_mean', 'D_112_min', 'D_112_last', 'B_40_mean', 'B_40_min', 'B_40_max', 'B_40_last', 'S_27_std',\n 'S_27_max', 'S_27_last', 'D_113_first', 'D_113_min', 'D_115_first', 'D_115_std', 'D_115_min', 'D_115_max',\n 'D_115_last', 'D_119_first', 'D_119_std', 'D_119_min', 'D_119_max', 'D_119_last', 'D_121_std',\n 'D_121_last', 'D_122_first', 'D_122_mean', 'D_122_min', 'D_123_mean', 'D_124_min', 'D_125_mean',\n 'D_125_min', 'D_128_std', 'D_129_first', 'D_129_mean', 'D_131_mean', 'D_133_mean', 'D_133_std',\n 'P_2_last_lag_sub', 'P_2_last_lag_div', 'D_39_last_lag_sub', 'D_39_last_lag_div', 'B_1_last_lag_div',\n 'B_2_last_lag_sub', 'B_2_last_lag_div', 'R_1_last_lag_sub', 'R_1_last_lag_div', 'S_3_last_lag_sub',\n 'S_3_last_lag_div', 'D_41_last_lag_sub', 'D_41_last_lag_div', 'B_3_last_lag_sub', 'B_3_last_lag_div',\n 'D_42_last_lag_sub', 'D_42_last_lag_div', 'D_43_last_lag_sub', 'D_44_last_lag_sub', 'D_44_last_lag_div',\n 'B_4_last_lag_sub', 'D_45_last_lag_div', 'B_5_last_lag_sub', 'D_48_last_lag_sub', 'D_48_last_lag_div',\n 'D_49_last_lag_sub', 'D_49_last_lag_div', 'B_6_last_lag_sub', 'B_6_last_lag_div', 'B_7_last_lag_div',\n 'B_8_last_lag_div', 'D_50_last_lag_sub', 'D_51_last_lag_div', 'B_9_last_lag_sub', 'B_9_last_lag_div',\n 'P_3_last_lag_sub', 'P_3_last_lag_div', 'B_10_last_lag_sub', 'B_10_last_lag_div', 'D_53_last_lag_sub',\n 'D_53_last_lag_div', 'B_11_last_lag_div', 'D_54_last_lag_sub', 'D_54_last_lag_div', 'S_7_last_lag_sub',\n 'S_7_last_lag_div', 'B_12_last_lag_sub', 'D_55_last_lag_sub', 'D_55_last_lag_div', 'R_5_last_lag_sub',\n 'D_58_last_lag_sub', 'D_58_last_lag_div', 'B_14_last_lag_sub', 'B_14_last_lag_div', 'D_59_last_lag_sub',\n 'D_59_last_lag_div', 'D_61_last_lag_sub', 'D_61_last_lag_div', 'D_65_last_lag_sub', 'B_16_last_lag_sub',\n 'B_16_last_lag_div', 'B_17_last_lag_sub', 'B_17_last_lag_div', 'B_18_last_lag_sub', 'B_18_last_lag_div',\n 'B_19_last_lag_sub', 'B_19_last_lag_div', 'B_20_last_lag_sub', 'S_13_last_lag_div', 'B_23_last_lag_sub',\n 'B_23_last_lag_div', 'P_4_last_lag_div', 'D_74_last_lag_div', 'D_75_last_lag_sub', 'D_75_last_lag_div',\n 'R_7_last_lag_sub', 'B_25_last_lag_sub', 'B_25_last_lag_div', 'B_26_last_lag_sub', 'B_26_last_lag_div',\n 'D_78_last_lag_sub', 'D_78_last_lag_div', 'D_79_last_lag_div', 'R_8_last_lag_sub', 'R_10_last_lag_div',\n 'B_28_last_lag_sub', 'B_28_last_lag_div', 'D_84_last_lag_div', 'B_33_last_lag_div', 'S_22_last_lag_div',\n 'S_23_last_lag_sub', 'S_23_last_lag_div', 'S_24_last_lag_sub', 'S_24_last_lag_div', 'S_25_last_lag_sub',\n 'S_25_last_lag_div', 'S_26_last_lag_sub', 'B_37_last_lag_sub', 'B_37_last_lag_div', 'R_27_last_lag_div',\n 'D_112_last_lag_sub', 'D_112_last_lag_div', 'B_40_last_lag_sub', 'B_40_last_lag_div', 'D_121_last_lag_sub',\n 'D_128_last_lag_div', 'D_131_last_lag_div', 'D_132_last_lag_div', 'B_30_last', 'B_30_nunique',\n 'B_38_last', 'D_114_first', 'D_117_first', 'D_68_count', 'D_68_first']","metadata":{"execution":{"iopub.status.busy":"2024-01-10T22:16:58.822122Z","iopub.execute_input":"2024-01-10T22:16:58.822506Z","iopub.status.idle":"2024-01-10T22:16:58.842583Z","shell.execute_reply.started":"2024-01-10T22:16:58.822474Z","shell.execute_reply":"2024-01-10T22:16:58.841708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## B. LGBM v1: improve on the baseline\n\nNow that we have a baseline we can try to improve the performance. In this iteration I will try implementing: \n* hyperparameter optimization (`optuna`);\n* use more hyperparameters (in the baseline model, we only used default values). Final 3rd search space version:\n```python\nparams = {\n    'objective': 'binary',\n    'metric': 'binary_logloss',\n    'boosting_type': 'gbdt',\n    'n_jobs': -1,\n    'random_state': 42,\n    'verbose':-1,\n\n    'num_leaves': 95,\n    'bagging_fraction': 0.8,\n    'max_depth': -1,\n\n    'learning_rate': trial.suggest_float('learning_rate', 0.009, 0.016), \n    'feature_fraction': trial.suggest_float('feature_fraction', 0.195, 0.215), \n    'n_estimators':trial.suggest_int('n_estimators', 1950, 2650, 50),\n    'reg_lambda': trial.suggest_categorical('reg_lambda', [0.005, 0.01, 0.02, 0.05]),\n    'min_child_samples': trial.suggest_int('min_child_samples', 650, 1350, 50),\n    'reg_alpha': trial.suggest_categorical('reg_alpha', [0, 0.001])\n    }\n```\n\nSince `AUC` proved to be an inadequate metric, I'll stop tracking it.","metadata":{}},{"cell_type":"code","source":"def objective(trial, df, gpu_settings = True):\n    params = {\n        'objective': 'binary',\n        'metric': 'binary_logloss',\n        'boosting_type': 'gbdt',\n        'n_jobs': -1,\n        'random_state': 42,\n        'verbose':-1,\n        \n        'num_leaves': 95,\n        'bagging_fraction': 0.8,\n        'max_depth': -1, #2**trial.suggest_int('num_leaves', 1, 8, 1),\n        #'reg_alpha': 1e-3,\n        \n            #'learning_rate': trial.suggest_float('learning_rate', 0.0025, 0.1), #V1\n            #'learning_rate': trial.suggest_float('learning_rate', 0.008, 0.025), #V2\n        'learning_rate': trial.suggest_float('learning_rate', 0.009, 0.016), #V3\n            #'feature_fraction': trial.suggest_float('feature_fraction', 0.2, 0.4), #V1\n            #'feature_fraction': trial.suggest_float('feature_fraction', 0.2, 0.25), #V2\n        'feature_fraction': trial.suggest_float('feature_fraction', 0.195, 0.215), #V3\n            #'n_estimators':trial.suggest_int('n_estimators', 1000, 2400, 100), #V1\n            #'n_estimators':trial.suggest_int('n_estimators', 900, 3100, 100), #V2\n        'n_estimators':trial.suggest_int('n_estimators', 1950, 2650, 50), #V3\n            #'reg_lambda': trial.suggest_categorical('reg_lambda', [1e-3, 1, 50]), #V1\n            #'reg_lambda': trial.suggest_categorical('reg_lambda', [1e-3, 1e-2, 1]), #V2\n        'reg_lambda': trial.suggest_categorical('reg_lambda', [0.005, 0.01, 0.02, 0.05]), #V3\n            #'min_child_samples': trial.suggest_categorical('min_child_samples', [2400, 2400*2]) #V1\n            #'min_child_samples': trial.suggest_categorical('min_child_samples', [1200, 2400, 2400*2]) #V2\n        'min_child_samples': trial.suggest_int('min_child_samples', 650, 1350, 50), #V3\n        'reg_alpha': trial.suggest_categorical('reg_alpha', [0, 0.001]) # V3 (added this hyperp.)\n        #'max_bin': trial.suggest_int('max_bin', 32, 256)\n    }\n    if gpu_settings:\n        params['device'] = 'gpu'\n        params['gpu_platform_id'] = 0\n        params['gpu_device_id'] = 0\n    \n    # Train lgbm with params and cross-validation:\n    amex_cv = lgbm_train(df, params)\n    return amex_cv","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_v1a = optuna_opt(\n    df = df_train_lag_imp[boruta_selection + ['target']].sample(160000, random_state = 42),\n    df_valid = df_valid_lag_imp[boruta_selection + ['target']],\n    weights_01 = [20,1], n_trials = 65, gpu_settings = True\n)","metadata":{"execution":{"iopub.status.busy":"2024-01-02T16:39:49.318695Z","iopub.execute_input":"2024-01-02T16:39:49.319628Z","iopub.status.idle":"2024-01-02T16:44:29.758144Z","shell.execute_reply.started":"2024-01-02T16:39:49.319593Z","shell.execute_reply":"2024-01-02T16:44:29.756578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_v1a","metadata":{"execution":{"iopub.status.busy":"2024-01-02T16:44:35.456147Z","iopub.execute_input":"2024-01-02T16:44:35.457033Z","iopub.status.idle":"2024-01-02T16:44:35.463610Z","shell.execute_reply.started":"2024-01-02T16:44:35.456999Z","shell.execute_reply":"2024-01-02T16:44:35.462576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save run stats:\nout = pd.DataFrame({'value':[]})\nout.loc['best_params','value'] = str(results_v1a['best_params'])\nout.loc['best_value','value'] = results_v1a['best_value']\nout.loc['best_value_iteration','value'] = results_v1a['best_value_iteration']\nout.loc['valid_metrics','value'] = str(results_v1a['valid_metrics'])\ndisplay(out)\nout.to_csv('/kaggle/working/run_stats_lgbm_v1.csv')\n\n# Save study object:\nwith open('/kaggle/working/study_lgbm_v1.pkl', \"wb\") as file:\n    pickle.dump(results_v1a['study'], file)\n\n# Save best model:\nwith open('/kaggle/working/model_lgbm_v1.pkl', \"wb\") as file:\n    pickle.dump(results_v1a['best_model'], file)","metadata":{"execution":{"iopub.status.busy":"2024-01-02T16:44:37.207504Z","iopub.execute_input":"2024-01-02T16:44:37.208653Z","iopub.status.idle":"2024-01-02T16:44:37.234279Z","shell.execute_reply.started":"2024-01-02T16:44:37.208618Z","shell.execute_reply":"2024-01-02T16:44:37.233477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nwith open('/kaggle/input/amex-lgbm-v1-output/model_lgbm_v1.pkl', \"rb\") as file:\n    model = pickle.load(file)\nwith open('/kaggle/input/amex-lgbm-v1-output/study_lgbm_v1.pkl', \"rb\") as file:\n    study = pickle.load(file)\nresults = pd.read_csv('/kaggle/input/amex-lgbm-v1-output/run_stats_lgbm_v1.csv')\n\n#study = results_v1a['study']\n#model = results_v1a['best_model']","metadata":{"execution":{"iopub.status.busy":"2024-01-10T22:16:47.223414Z","iopub.execute_input":"2024-01-10T22:16:47.223735Z","iopub.status.idle":"2024-01-10T22:16:47.521885Z","shell.execute_reply.started":"2024-01-10T22:16:47.223710Z","shell.execute_reply":"2024-01-10T22:16:47.520913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(optuna.visualization.plot_optimization_history(study))\ndisplay(optuna.visualization.plot_param_importances(study))\ndisplay(optuna.visualization.plot_slice(study))","metadata":{"execution":{"iopub.status.busy":"2024-01-06T20:09:21.552432Z","iopub.execute_input":"2024-01-06T20:09:21.553244Z","iopub.status.idle":"2024-01-06T20:09:24.068607Z","shell.execute_reply.started":"2024-01-06T20:09:21.553209Z","shell.execute_reply":"2024-01-06T20:09:24.067610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## C. Test submission:","metadata":{}},{"cell_type":"code","source":"# Read the single statement data including the mentioned lag features:\ndf_test_lag = pd.read_parquet(\"/kaggle/input/amex-fe/test_fe_plus_plus.parquet\").rename(\n    {'D_63_count':'ntot_statement'}, axis = 1\n)\nprint(df_test_lag.info())\ndf_test_lag.head(3)","metadata":{"execution":{"iopub.status.busy":"2024-01-10T22:11:30.034769Z","iopub.execute_input":"2024-01-10T22:11:30.035574Z","iopub.status.idle":"2024-01-10T22:12:03.305947Z","shell.execute_reply.started":"2024-01-10T22:11:30.035541Z","shell.execute_reply":"2024-01-10T22:12:03.304956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Simply impute missing values with -999999:\ndf_test_lag_imp = df_test_lag.fillna(-999999)\n\n# Impute inf values with 99999999\ncol_w_inf = []\nfor col in df_test_lag_imp.drop(['customer_ID'], axis = 1):\n    col_w_inf.append(col)\n    if np.isinf(df_test_lag_imp[col]).any():\n        df_test_lag_imp[col] = df_test_lag_imp[col].replace([np.inf, -np.inf], 99999999)","metadata":{"execution":{"iopub.status.busy":"2024-01-10T22:17:10.060369Z","iopub.execute_input":"2024-01-10T22:17:10.060991Z","iopub.status.idle":"2024-01-10T22:17:18.042170Z","shell.execute_reply.started":"2024-01-10T22:17:10.060960Z","shell.execute_reply":"2024-01-10T22:17:18.041386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_lag = None","metadata":{"execution":{"iopub.status.busy":"2024-01-10T22:17:18.043650Z","iopub.execute_input":"2024-01-10T22:17:18.043932Z","iopub.status.idle":"2024-01-10T22:17:18.049256Z","shell.execute_reply.started":"2024-01-10T22:17:18.043907Z","shell.execute_reply":"2024-01-10T22:17:18.048411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_test = model.predict(df_test_lag_imp[boruta_selection])","metadata":{"execution":{"iopub.status.busy":"2024-01-10T22:17:18.050248Z","iopub.execute_input":"2024-01-10T22:17:18.050486Z","iopub.status.idle":"2024-01-10T22:19:16.585927Z","shell.execute_reply.started":"2024-01-10T22:17:18.050464Z","shell.execute_reply":"2024-01-10T22:19:16.584946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({'customer_ID': df_test_lag_imp['customer_ID'],\n                           'prediction': preds_test}).set_index('customer_ID')\nsubmission.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-01-10T22:19:41.014182Z","iopub.execute_input":"2024-01-10T22:19:41.014566Z","iopub.status.idle":"2024-01-10T22:19:45.614024Z","shell.execute_reply.started":"2024-01-10T22:19:41.014537Z","shell.execute_reply":"2024-01-10T22:19:45.613263Z"},"trusted":true},"execution_count":null,"outputs":[]}]}