{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# LightGBM TPS August 2022","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom lightgbm import LGBMRegressor, log_evaluation, early_stopping\nimport optuna\noptuna.logging.set_verbosity(optuna.logging.ERROR)\nimport matplotlib.pyplot as plt\n\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-01T17:54:25.863841Z","iopub.execute_input":"2022-08-01T17:54:25.864289Z","iopub.status.idle":"2022-08-01T17:54:25.872399Z","shell.execute_reply.started":"2022-08-01T17:54:25.864252Z","shell.execute_reply":"2022-08-01T17:54:25.871196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv')\ntest = pd.read_csv('../input/tabular-playground-series-aug-2022/test.csv')\nsub = pd.read_csv('../input/tabular-playground-series-aug-2022/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-01T17:50:11.908248Z","iopub.execute_input":"2022-08-01T17:50:11.908665Z","iopub.status.idle":"2022-08-01T17:50:12.246083Z","shell.execute_reply.started":"2022-08-01T17:50:11.908624Z","shell.execute_reply":"2022-08-01T17:50:12.244802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe().T","metadata":{"execution":{"iopub.status.busy":"2022-08-01T17:50:12.247503Z","iopub.execute_input":"2022-08-01T17:50:12.248493Z","iopub.status.idle":"2022-08-01T17:50:12.397743Z","shell.execute_reply.started":"2022-08-01T17:50:12.248454Z","shell.execute_reply":"2022-08-01T17:50:12.396521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"train.drop(columns=['id'], inplace=True)\ntest.drop(columns=['id'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T17:50:12.401031Z","iopub.execute_input":"2022-08-01T17:50:12.402032Z","iopub.status.idle":"2022-08-01T17:50:12.416087Z","shell.execute_reply.started":"2022-08-01T17:50:12.401983Z","shell.execute_reply":"2022-08-01T17:50:12.414643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"float_columns = train.select_dtypes(include=[float]).columns\none_hot_columns = set(list(train.columns)) - set(list(float_columns)) - set(list(['failure']))","metadata":{"execution":{"iopub.status.busy":"2022-08-01T17:50:12.417822Z","iopub.execute_input":"2022-08-01T17:50:12.418867Z","iopub.status.idle":"2022-08-01T17:50:12.433827Z","shell.execute_reply.started":"2022-08-01T17:50:12.418817Z","shell.execute_reply":"2022-08-01T17:50:12.432739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoders = {}\n\nfor col in one_hot_columns:\n    enc = OneHotEncoder(handle_unknown='ignore')\n    enc.fit(train[col].values.reshape(-1, 1))\n    encoders[col] = enc","metadata":{"execution":{"iopub.status.busy":"2022-08-01T17:50:12.435802Z","iopub.execute_input":"2022-08-01T17:50:12.437934Z","iopub.status.idle":"2022-08-01T17:50:12.457004Z","shell.execute_reply.started":"2022-08-01T17:50:12.437884Z","shell.execute_reply":"2022-08-01T17:50:12.455928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_df_encoding(data, encoders):\n    \n    df = pd.DataFrame()\n\n    for col in one_hot_columns:\n        header_cols = encoders[col].get_feature_names_out([col])\n        arr = encoders[col].transform(data[col].values.reshape(-1, 1)).toarray()\n        df = pd.concat([df, pd.DataFrame(arr, columns=header_cols)], axis=1)\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-08-01T17:50:12.458864Z","iopub.execute_input":"2022-08-01T17:50:12.459677Z","iopub.status.idle":"2022-08-01T17:50:12.468702Z","shell.execute_reply.started":"2022-08-01T17:50:12.459630Z","shell.execute_reply":"2022-08-01T17:50:12.467380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = pd.concat([train[float_columns], get_df_encoding(train, encoders)], axis=1)\nlabels = train['failure']\n\ntest_prep = pd.concat([test[float_columns], get_df_encoding(test, encoders)], axis=1)\n\nnew_one_hot_columns = set(features.columns) - set(float_columns)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T17:50:12.472791Z","iopub.execute_input":"2022-08-01T17:50:12.473165Z","iopub.status.idle":"2022-08-01T17:50:12.739597Z","shell.execute_reply.started":"2022-08-01T17:50:12.473134Z","shell.execute_reply":"2022-08-01T17:50:12.738383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Verify imbalance","metadata":{}},{"cell_type":"code","source":"x = [len(labels[labels==0]), len(labels[labels==1])]\nplt_labels = ['Negative', 'Positive']\n\nfig, ax = plt.subplots()\nax.pie(x, labels = plt_labels)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T17:50:12.741271Z","iopub.execute_input":"2022-08-01T17:50:12.741753Z","iopub.status.idle":"2022-08-01T17:50:13.062823Z","shell.execute_reply.started":"2022-08-01T17:50:12.741709Z","shell.execute_reply":"2022-08-01T17:50:13.061662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## KFold Training","metadata":{}},{"cell_type":"code","source":"def objective(trial, data,target):\n    \n    X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.10, random_state=42)\n    \n    params = {\n                'metric': 'rmse', \n                'random_state': 22,\n                'n_estimators': 20000,\n                'boosting_type': trial.suggest_categorical(\"boosting_type\", [\"gbdt\", \"goss\"]),\n                'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-3, 10.0),\n                'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-3, 10.0),\n                'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.5, 0.6, 0.7, 0.8, 0.9, 1.0]),\n                'subsample': trial.suggest_categorical('subsample', [0.6, 0.7, 0.85, 1.0]),\n                'learning_rate': trial.suggest_categorical('learning_rate', [0.005, 0.01, 0.02, 0.03, 0.05, 0.1]),\n                'max_depth': trial.suggest_int('max_depth', 2, 12, step=1),\n                'num_leaves' : trial.suggest_int('num_leaves', 13, 148, step=5),\n                'min_child_samples': trial.suggest_int('min_child_samples', 1, 96, step=5),\n            }\n    \n    reg = LGBMRegressor(**params)  \n    reg.fit(X_train ,y_train,\n            eval_set=[(X_valid, y_valid)],\n            #categorical_feature=cat_indices,\n            callbacks=[log_evaluation(period=1000), \n                       early_stopping(stopping_rounds=50)\n                      ],\n           )\n    \n    y_pred = reg.predict(X_valid)\n    rmse = mean_squared_error(y_valid, y_pred, squared=False)\n    \n    return rmse","metadata":{"execution":{"iopub.status.busy":"2022-08-01T17:50:13.067311Z","iopub.execute_input":"2022-08-01T17:50:13.068197Z","iopub.status.idle":"2022-08-01T17:50:13.085622Z","shell.execute_reply.started":"2022-08-01T17:50:13.068144Z","shell.execute_reply":"2022-08-01T17:50:13.084493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_splits = 5\nskf = StratifiedKFold(n_splits=n_splits)\noof_preds = np.zeros((n_splits, len(test)))\n\nfor i, (train_index, test_index) in enumerate(skf.split(features, labels)):\n    X_train, X_test = features.iloc[train_index], features.iloc[test_index]\n    y_train, y_test = labels.iloc[train_index], labels.iloc[test_index]\n    \n    scaler = StandardScaler().fit(X_train[float_columns])\n    temp_train_float = pd.DataFrame(scaler.transform(X_train[float_columns]), columns=float_columns)\n    temp_test_float = pd.DataFrame(scaler.transform(test_prep[float_columns]), columns=float_columns)\n    \n    train_df = np.hstack((X_train[new_one_hot_columns].values, temp_train_float.values))\n    test_df = np.hstack((test_prep[new_one_hot_columns].values, temp_test_float.values))\n    \n    X, y = train_df, y_train.values\n    \n    func = lambda trial: objective(trial, X, y)\n    study = optuna.create_study(direction='minimize')\n    history = study.optimize(func, n_trials=50)\n    \n    clf = LGBMRegressor(**study.best_params)\n    clf.fit(X, y)\n    \n    loss = mean_squared_error(y, clf.predict(train_df), squared=False)\n    print('Fold:%2d, Loss:%1.3f' % (i, loss))\n    \n    oof_preds[i] = clf.predict(test_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T17:55:51.036614Z","iopub.execute_input":"2022-08-01T17:55:51.037049Z","iopub.status.idle":"2022-08-01T17:56:36.797727Z","shell.execute_reply.started":"2022-08-01T17:55:51.037014Z","shell.execute_reply":"2022-08-01T17:56:36.795871Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"sub['failure'] = np.mean(oof_preds, axis=0)\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T15:41:08.379418Z","iopub.execute_input":"2022-08-01T15:41:08.380008Z","iopub.status.idle":"2022-08-01T15:41:08.416872Z","shell.execute_reply.started":"2022-08-01T15:41:08.379968Z","shell.execute_reply":"2022-08-01T15:41:08.415958Z"},"trusted":true},"execution_count":null,"outputs":[]}]}