{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# LightGBM TPS August 2022","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport xgboost as xgb\nfrom lightgbm import LGBMRegressor, log_evaluation, early_stopping\nimport optuna\noptuna.logging.set_verbosity(optuna.logging.ERROR)\nimport matplotlib.pyplot as plt\n\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.preprocessing import PowerTransformer\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import mean_squared_error\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-02T16:00:37.602176Z","iopub.execute_input":"2022-08-02T16:00:37.603737Z","iopub.status.idle":"2022-08-02T16:00:40.559668Z","shell.execute_reply.started":"2022-08-02T16:00:37.603032Z","shell.execute_reply":"2022-08-02T16:00:40.558707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reference\n\n[LightGBM + Optuna notebook](https://www.kaggle.com/code/jackyron/tps-aug-22-lightgbm-optuna)<br>\n[EDA + Logistic Regression](https://www.kaggle.com/code/cabaxiom/tps-aug-22-eda-logistic-regression-baseline)<br>\n[XGBoost + Optuna notebook](https://www.kaggle.com/code/juanhubert/tps-aug-22-xgboost-optuna)","metadata":{}},{"cell_type":"markdown","source":"## Data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv')\ntest = pd.read_csv('../input/tabular-playground-series-aug-2022/test.csv')\nsub = pd.read_csv('../input/tabular-playground-series-aug-2022/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-02T16:00:40.561429Z","iopub.execute_input":"2022-08-02T16:00:40.561759Z","iopub.status.idle":"2022-08-02T16:00:40.790879Z","shell.execute_reply.started":"2022-08-02T16:00:40.561732Z","shell.execute_reply":"2022-08-02T16:00:40.789940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe().T","metadata":{"execution":{"iopub.status.busy":"2022-08-02T16:00:40.792424Z","iopub.execute_input":"2022-08-02T16:00:40.792776Z","iopub.status.idle":"2022-08-02T16:00:40.895876Z","shell.execute_reply.started":"2022-08-02T16:00:40.792739Z","shell.execute_reply":"2022-08-02T16:00:40.894843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"train.drop(columns=['id'], inplace=True)\ntest.drop(columns=['id'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-02T16:00:40.898693Z","iopub.execute_input":"2022-08-02T16:00:40.899353Z","iopub.status.idle":"2022-08-02T16:00:40.911664Z","shell.execute_reply.started":"2022-08-02T16:00:40.899316Z","shell.execute_reply":"2022-08-02T16:00:40.910773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"float_columns = train.select_dtypes(include=[float]).columns\none_hot_columns = set(list(train.columns)) - set(list(float_columns)) - set(list(['failure']))","metadata":{"execution":{"iopub.status.busy":"2022-08-02T16:00:40.913826Z","iopub.execute_input":"2022-08-02T16:00:40.914489Z","iopub.status.idle":"2022-08-02T16:00:40.926480Z","shell.execute_reply.started":"2022-08-02T16:00:40.914453Z","shell.execute_reply":"2022-08-02T16:00:40.925513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoders = {}\n\nfor col in one_hot_columns:\n    enc = OneHotEncoder(handle_unknown='ignore')\n    enc.fit(train[col].values.reshape(-1, 1))\n    encoders[col] = enc","metadata":{"execution":{"iopub.status.busy":"2022-08-02T16:00:40.927970Z","iopub.execute_input":"2022-08-02T16:00:40.928331Z","iopub.status.idle":"2022-08-02T16:00:40.946122Z","shell.execute_reply.started":"2022-08-02T16:00:40.928298Z","shell.execute_reply":"2022-08-02T16:00:40.945271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_df_encoding(data, encoders):\n    \n    df = pd.DataFrame()\n\n    for col in one_hot_columns:\n        header_cols = encoders[col].get_feature_names_out([col])\n        arr = encoders[col].transform(data[col].values.reshape(-1, 1)).toarray()\n        df = pd.concat([df, pd.DataFrame(arr, columns=header_cols)], axis=1)\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-08-02T16:00:40.948716Z","iopub.execute_input":"2022-08-02T16:00:40.949373Z","iopub.status.idle":"2022-08-02T16:00:40.955483Z","shell.execute_reply.started":"2022-08-02T16:00:40.949338Z","shell.execute_reply":"2022-08-02T16:00:40.954511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = pd.concat([train[float_columns], get_df_encoding(train, encoders)], axis=1)\nlabels = train['failure']\n\ntest_prep = pd.concat([test[float_columns], get_df_encoding(test, encoders)], axis=1)\n\nnew_one_hot_columns = set(features.columns) - set(float_columns)","metadata":{"execution":{"iopub.status.busy":"2022-08-02T16:00:40.958762Z","iopub.execute_input":"2022-08-02T16:00:40.959081Z","iopub.status.idle":"2022-08-02T16:00:41.110322Z","shell.execute_reply.started":"2022-08-02T16:00:40.959049Z","shell.execute_reply":"2022-08-02T16:00:41.109265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Verify imbalance","metadata":{}},{"cell_type":"code","source":"x = [len(labels[labels==0]), len(labels[labels==1])]\nplt_labels = ['Negative', 'Positive']\n\nfig, ax = plt.subplots()\nax.pie(x, labels = plt_labels)\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-02T16:00:41.111791Z","iopub.execute_input":"2022-08-02T16:00:41.112334Z","iopub.status.idle":"2022-08-02T16:00:41.235884Z","shell.execute_reply.started":"2022-08-02T16:00:41.112296Z","shell.execute_reply":"2022-08-02T16:00:41.234664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## KFold Training","metadata":{}},{"cell_type":"markdown","source":"### Optuna functions","metadata":{}},{"cell_type":"code","source":"def objective_lgb(trial, data,target):\n    \n    X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.10, random_state=42)\n    \n    params = {\n                'metric': 'rmse', \n                'random_state': 22,\n                'n_estimators': 20000,\n                'boosting_type': trial.suggest_categorical(\"boosting_type\", [\"gbdt\", \"goss\"]),\n                'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-3, 10.0),\n                'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-3, 10.0),\n                'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.5, 0.6, 0.7, 0.8, 0.9, 1.0]),\n                'subsample': trial.suggest_categorical('subsample', [0.6, 0.7, 0.85, 1.0]),\n                'learning_rate': trial.suggest_categorical('learning_rate', [0.005, 0.01, 0.02, 0.03, 0.05, 0.1]),\n                'max_depth': trial.suggest_int('max_depth', 2, 12, step=1),\n                'num_leaves' : trial.suggest_int('num_leaves', 13, 148, step=5),\n                'min_child_samples': trial.suggest_int('min_child_samples', 1, 96, step=5),\n            }\n    \n    reg = LGBMRegressor(**params)  \n    reg.fit(X_train ,y_train,\n            eval_set=[(X_valid, y_valid)],\n            verbose=False,\n            callbacks=[log_evaluation(period=1000), \n                       early_stopping(stopping_rounds=50)\n                      ],\n           )\n    \n    y_pred = reg.predict(X_valid)\n    rmse = mean_squared_error(y_valid, y_pred, squared=False)\n    \n    return rmse","metadata":{"execution":{"iopub.status.busy":"2022-08-02T16:00:41.245053Z","iopub.execute_input":"2022-08-02T16:00:41.248171Z","iopub.status.idle":"2022-08-02T16:00:41.270057Z","shell.execute_reply.started":"2022-08-02T16:00:41.248119Z","shell.execute_reply":"2022-08-02T16:00:41.268771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def objective_xgb(trial,data,target):\n    \n    train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.15,random_state=42)\n    param = {\n        'tree_method':'gpu_hist',\n        'lambda': trial.suggest_loguniform('lambda', 1e-3, 10.0),\n        'alpha': trial.suggest_loguniform('alpha', 1e-3, 10.0),\n        'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.3,0.4,0.5,0.6,0.7,0.8,0.9, 1.0]),\n        'subsample': trial.suggest_categorical('subsample', [0.4,0.5,0.6,0.7,0.8,1.0]),\n        'learning_rate': trial.suggest_categorical('learning_rate', [0.008,0.01,0.012,0.014,0.016,0.018, 0.02]),\n        'n_estimators': 10000,\n        'max_depth': trial.suggest_categorical('max_depth', [5,7,9,11,13,15,17]),\n        'random_state': trial.suggest_categorical('random_state', [2020]),\n        'min_child_weight': trial.suggest_int('min_child_weight', 1, 300),\n        'early_stopping_rounds':100\n    }\n    model = xgb.XGBRegressor(**param)  \n    \n    model.fit(train_x,train_y,eval_set=[(test_x,test_y)],verbose=False)\n    \n    preds = model.predict(test_x)\n    \n    rmse = mean_squared_error(test_y, preds,squared=False)\n    \n    return rmse","metadata":{"execution":{"iopub.status.busy":"2022-08-02T16:00:41.276221Z","iopub.execute_input":"2022-08-02T16:00:41.279649Z","iopub.status.idle":"2022-08-02T16:00:41.298513Z","shell.execute_reply.started":"2022-08-02T16:00:41.279599Z","shell.execute_reply":"2022-08-02T16:00:41.297300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training","metadata":{}},{"cell_type":"code","source":"weights = {}\nweights['xgb'] = 0.7\nweights['lgbm'] = 0.1\nweights['logistic'] = 0.2","metadata":{"execution":{"iopub.status.busy":"2022-08-02T16:00:41.300705Z","iopub.execute_input":"2022-08-02T16:00:41.301061Z","iopub.status.idle":"2022-08-02T16:00:41.314523Z","shell.execute_reply.started":"2022-08-02T16:00:41.301028Z","shell.execute_reply":"2022-08-02T16:00:41.312744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n\nn_splits = 5\nskf = StratifiedKFold(n_splits=n_splits)\noof_preds = np.zeros((n_splits, len(test)))\n\nlgb_best_params = []\nxgb_best_params = []\n\nfor i, (train_index, test_index) in enumerate(skf.split(features, labels)):\n    X_train, X_test = features.iloc[train_index], features.iloc[test_index]\n    y_train, y_test = labels.iloc[train_index], labels.iloc[test_index]\n    \n    imp = SimpleImputer(missing_values=np.nan, strategy='mean')\n    X_train = imp.fit_transform(X_train)\n    test_df = imp.transform(test_prep)\n    \n    X_train = pd.DataFrame(X_train, columns=features.columns)\n    test_df = pd.DataFrame(test_df, columns=features.columns)\n    \n    scaler = PowerTransformer().fit(X_train[float_columns])\n    temp_train_float = pd.DataFrame(scaler.transform(X_train[float_columns]), columns=float_columns)\n    temp_test_float = pd.DataFrame(scaler.transform(test_df[float_columns]), columns=float_columns)\n    \n    train_df = np.hstack((X_train[new_one_hot_columns].values, temp_train_float.values))\n    test_df = np.hstack((test_df[new_one_hot_columns].values, temp_test_float.values))\n    \n    X, y = train_df, y_train.values\n    \n    func = lambda trial: objective_lgb(trial, X, y)\n    study_lgb = optuna.create_study(direction='minimize')\n    history = study_lgb.optimize(func, n_trials=50)\n    lgb_best_params.append(study_lgb.best_params)\n    \n    clf_lgbm = LGBMRegressor(**study_lgb.best_params)\n    clf_lgbm.fit(train_df, y_train.values)\n    \n    func = lambda trial: objective_xgb(trial, X, y)\n    study_xgb = optuna.create_study(direction='minimize')\n    history = study_xgb.optimize(func, n_trials=50)\n    xgb_best_params.append(study_xgb.best_params)\n    \n    clf_xgb = xgb.XGBRegressor(**study_xgb.best_params)\n    clf_xgb.fit(train_df, y_train.values)\n    \n    clf_logistic = LogisticRegression(penalty='elasticnet', l1_ratio=0.8, C=0.007, tol = 1e-2, solver='saga', max_iter=1000, random_state=i)\n    clf_logistic.fit(train_df, y_train.values)\n    \n    log_preds = np.argmax(clf_logistic.predict_proba(test_df), axis=1)\n    \n    oof_preds[i] = (weights['xgb'] * clf_xgb.predict(test_df)) + \\\n                    (weights['lgbm'] * clf_lgbm.predict(test_df)) + \\\n                    (weights['logistic'] * log_preds)","metadata":{"_kg_hide-output":false,"execution":{"iopub.status.busy":"2022-08-02T16:02:30.562533Z","iopub.execute_input":"2022-08-02T16:02:30.562928Z","iopub.status.idle":"2022-08-02T16:03:00.207587Z","shell.execute_reply.started":"2022-08-02T16:02:30.562886Z","shell.execute_reply":"2022-08-02T16:03:00.206574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Best Parameters","metadata":{}},{"cell_type":"markdown","source":"### XGBoost","metadata":{}},{"cell_type":"code","source":"for i in range(n_splits):\n    print('Fold: %2d' % (i+1))\n    print(xgb_best_params[i])\n    print('')","metadata":{"execution":{"iopub.status.busy":"2022-08-02T16:00:41.338613Z","iopub.execute_input":"2022-08-02T16:00:41.338871Z","iopub.status.idle":"2022-08-02T16:00:41.500156Z","shell.execute_reply.started":"2022-08-02T16:00:41.338848Z","shell.execute_reply":"2022-08-02T16:00:41.498823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### LightGBM","metadata":{}},{"cell_type":"code","source":"for i in range(n_splits):\n    print('Fold: %2d' % (i+1))\n    print(lgb_best_params[i])\n    print('')","metadata":{"execution":{"iopub.status.busy":"2022-08-02T16:00:41.501414Z","iopub.status.idle":"2022-08-02T16:00:41.502143Z","shell.execute_reply.started":"2022-08-02T16:00:41.501866Z","shell.execute_reply":"2022-08-02T16:00:41.501905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"sub['failure'] = np.mean(oof_preds, axis=0)\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-02T16:00:41.503683Z","iopub.status.idle":"2022-08-02T16:00:41.504442Z","shell.execute_reply.started":"2022-08-02T16:00:41.504191Z","shell.execute_reply":"2022-08-02T16:00:41.504215Z"},"trusted":true},"execution_count":null,"outputs":[]}]}