{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# LightGBM TPS August 2022","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom lightgbm import LGBMRegressor, log_evaluation, early_stopping\nimport optuna\nimport matplotlib.pyplot as plt\n\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.preprocessing import PowerTransformer\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import mean_squared_error","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-02T13:44:33.385446Z","iopub.execute_input":"2022-08-02T13:44:33.385959Z","iopub.status.idle":"2022-08-02T13:44:33.393803Z","shell.execute_reply.started":"2022-08-02T13:44:33.385899Z","shell.execute_reply":"2022-08-02T13:44:33.392582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reference\n\n[LightGBM + Optuna notebook](https://www.kaggle.com/code/jackyron/tps-aug-22-lightgbm-optuna)<br>\n[EDA + Logistic Regression](https://www.kaggle.com/code/cabaxiom/tps-aug-22-eda-logistic-regression-baseline)","metadata":{}},{"cell_type":"markdown","source":"## Data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv')\ntest = pd.read_csv('../input/tabular-playground-series-aug-2022/test.csv')\nsub = pd.read_csv('../input/tabular-playground-series-aug-2022/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-02T13:42:03.869636Z","iopub.execute_input":"2022-08-02T13:42:03.870220Z","iopub.status.idle":"2022-08-02T13:42:04.236787Z","shell.execute_reply.started":"2022-08-02T13:42:03.870164Z","shell.execute_reply":"2022-08-02T13:42:04.235530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe().T","metadata":{"execution":{"iopub.status.busy":"2022-08-02T13:42:04.238618Z","iopub.execute_input":"2022-08-02T13:42:04.239455Z","iopub.status.idle":"2022-08-02T13:42:04.381876Z","shell.execute_reply.started":"2022-08-02T13:42:04.239400Z","shell.execute_reply":"2022-08-02T13:42:04.380362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"train.drop(columns=['id'], inplace=True)\ntest.drop(columns=['id'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-02T13:42:04.384715Z","iopub.execute_input":"2022-08-02T13:42:04.385141Z","iopub.status.idle":"2022-08-02T13:42:04.399771Z","shell.execute_reply.started":"2022-08-02T13:42:04.385104Z","shell.execute_reply":"2022-08-02T13:42:04.397809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"float_columns = train.select_dtypes(include=[float]).columns\none_hot_columns = set(list(train.columns)) - set(list(float_columns)) - set(list(['failure']))","metadata":{"execution":{"iopub.status.busy":"2022-08-02T13:42:04.402765Z","iopub.execute_input":"2022-08-02T13:42:04.403535Z","iopub.status.idle":"2022-08-02T13:42:04.417366Z","shell.execute_reply.started":"2022-08-02T13:42:04.403480Z","shell.execute_reply":"2022-08-02T13:42:04.415575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoders = {}\n\nfor col in one_hot_columns:\n    enc = OneHotEncoder(handle_unknown='ignore')\n    enc.fit(train[col].values.reshape(-1, 1))\n    encoders[col] = enc","metadata":{"execution":{"iopub.status.busy":"2022-08-02T13:42:04.420669Z","iopub.execute_input":"2022-08-02T13:42:04.421735Z","iopub.status.idle":"2022-08-02T13:42:04.440309Z","shell.execute_reply.started":"2022-08-02T13:42:04.421681Z","shell.execute_reply":"2022-08-02T13:42:04.438936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_df_encoding(data, encoders):\n    \n    df = pd.DataFrame()\n\n    for col in one_hot_columns:\n        header_cols = encoders[col].get_feature_names_out([col])\n        arr = encoders[col].transform(data[col].values.reshape(-1, 1)).toarray()\n        df = pd.concat([df, pd.DataFrame(arr, columns=header_cols)], axis=1)\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-08-02T13:42:04.442369Z","iopub.execute_input":"2022-08-02T13:42:04.443653Z","iopub.status.idle":"2022-08-02T13:42:04.452387Z","shell.execute_reply.started":"2022-08-02T13:42:04.443596Z","shell.execute_reply":"2022-08-02T13:42:04.451176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = pd.concat([train[float_columns], get_df_encoding(train, encoders)], axis=1)\nlabels = train['failure']\n\ntest_prep = pd.concat([test[float_columns], get_df_encoding(test, encoders)], axis=1)\n\nnew_one_hot_columns = set(features.columns) - set(float_columns)","metadata":{"execution":{"iopub.status.busy":"2022-08-02T13:42:04.455215Z","iopub.execute_input":"2022-08-02T13:42:04.456785Z","iopub.status.idle":"2022-08-02T13:42:04.724456Z","shell.execute_reply.started":"2022-08-02T13:42:04.456730Z","shell.execute_reply":"2022-08-02T13:42:04.723132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Verify imbalance","metadata":{}},{"cell_type":"code","source":"x = [len(labels[labels==0]), len(labels[labels==1])]\nplt_labels = ['Negative', 'Positive']\n\nfig, ax = plt.subplots()\nax.pie(x, labels = plt_labels)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-02T13:42:04.725621Z","iopub.execute_input":"2022-08-02T13:42:04.725978Z","iopub.status.idle":"2022-08-02T13:42:05.069510Z","shell.execute_reply.started":"2022-08-02T13:42:04.725943Z","shell.execute_reply":"2022-08-02T13:42:05.067911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## KFold Training","metadata":{}},{"cell_type":"code","source":"def objective(trial, data,target):\n    \n    X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.10, random_state=42)\n    \n    params = {\n                'metric': 'rmse', \n                'random_state': 22,\n                'n_estimators': 20000,\n                'boosting_type': trial.suggest_categorical(\"boosting_type\", [\"gbdt\", \"goss\"]),\n                'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-3, 10.0),\n                'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-3, 10.0),\n                'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.5, 0.6, 0.7, 0.8, 0.9, 1.0]),\n                'subsample': trial.suggest_categorical('subsample', [0.6, 0.7, 0.85, 1.0]),\n                'learning_rate': trial.suggest_categorical('learning_rate', [0.005, 0.01, 0.02, 0.03, 0.05, 0.1]),\n                'max_depth': trial.suggest_int('max_depth', 2, 12, step=1),\n                'num_leaves' : trial.suggest_int('num_leaves', 13, 148, step=5),\n                'min_child_samples': trial.suggest_int('min_child_samples', 1, 96, step=5),\n            }\n    \n    reg = LGBMRegressor(**params)  \n    reg.fit(X_train ,y_train,\n            eval_set=[(X_valid, y_valid)],\n            #categorical_feature=cat_indices,\n            callbacks=[log_evaluation(period=1000), \n                       early_stopping(stopping_rounds=50)\n                      ],\n           )\n    \n    y_pred = reg.predict(X_valid)\n    rmse = mean_squared_error(y_valid, y_pred, squared=False)\n    \n    return rmse","metadata":{"execution":{"iopub.status.busy":"2022-08-02T13:51:25.724942Z","iopub.execute_input":"2022-08-02T13:51:25.725366Z","iopub.status.idle":"2022-08-02T13:51:25.737816Z","shell.execute_reply.started":"2022-08-02T13:51:25.725332Z","shell.execute_reply":"2022-08-02T13:51:25.736609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_splits = 5\nskf = StratifiedKFold(n_splits=n_splits)\noof_preds = np.zeros((n_splits, len(test)))\n\nfor i, (train_index, test_index) in enumerate(skf.split(features, labels)):\n    X_train, X_test = features.iloc[train_index], features.iloc[test_index]\n    y_train, y_test = labels.iloc[train_index], labels.iloc[test_index]\n    \n    imp = SimpleImputer(missing_values=np.nan, strategy='mean')\n    X_train = imp.fit_transform(X_train)\n    test_df = imp.transform(test_prep)\n    \n    X_train = pd.DataFrame(X_train, columns=features.columns)\n    test_df = pd.DataFrame(test_df, columns=features.columns)\n    \n    scaler = PowerTransformer().fit(X_train[float_columns])\n    temp_train_float = pd.DataFrame(scaler.transform(X_train[float_columns]), columns=float_columns)\n    temp_test_float = pd.DataFrame(scaler.transform(test_df[float_columns]), columns=float_columns)\n    \n    train_df = np.hstack((X_train[new_one_hot_columns].values, temp_train_float.values))\n    test_df = np.hstack((test_df[new_one_hot_columns].values, temp_test_float.values))\n    \n    X, y = train_df, y_train.values\n    \n    func = lambda trial: objective(trial, X, y)\n    study = optuna.create_study(direction='minimize')\n    history = study.optimize(func, n_trials=20)\n    \n    clf_lgbm = LGBMRegressor(**study.best_params)\n    clf_lgbm.fit(train_df, y_train.values)\n    \n    clf_logistic = LogisticRegression(penalty='elasticnet', l1_ratio=0.8, C=0.007, tol = 1e-2, solver='saga', max_iter=1000, random_state=i)\n    clf_logistic.fit(train_df, y_train.values)\n    \n    log_preds = np.argmax(clf_logistic.predict_proba(test_df), axis=1)\n    \n    oof_preds[i] = (clf_lgbm.predict(test_df) + log_preds) / 2","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-02T13:51:26.941499Z","iopub.execute_input":"2022-08-02T13:51:26.942163Z","iopub.status.idle":"2022-08-02T13:52:43.143050Z","shell.execute_reply.started":"2022-08-02T13:51:26.942123Z","shell.execute_reply":"2022-08-02T13:52:43.142074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"sub['failure'] = np.mean(oof_preds, axis=0)\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T15:41:08.379418Z","iopub.execute_input":"2022-08-01T15:41:08.380008Z","iopub.status.idle":"2022-08-01T15:41:08.416872Z","shell.execute_reply.started":"2022-08-01T15:41:08.379968Z","shell.execute_reply":"2022-08-01T15:41:08.415958Z"},"trusted":true},"execution_count":null,"outputs":[]}]}