{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport optuna\nimport lightgbm as lgb\nfrom path import Path\nfrom sklearn.model_selection import StratifiedKFold","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":1.960473,"end_time":"2022-07-12T13:42:44.382649","exception":false,"start_time":"2022-07-12T13:42:42.422176","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-30T08:36:42.547970Z","iopub.execute_input":"2022-07-30T08:36:42.548361Z","iopub.status.idle":"2022-07-30T08:36:42.555839Z","shell.execute_reply.started":"2022-07-30T08:36:42.548330Z","shell.execute_reply":"2022-07-30T08:36:42.554096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    input_path = Path('../input/porto-seguro-safe-driver-prediction')\n    optuna_lgb = False\n    n_estimators = 1500\n    early_stopping_round = 150\n    cv_folds = 5\n    random_state = 0\n    params = {'objective': 'binary',\n              'boosting_type': 'gbdt',\n              'learning_rate': 0.01,\n              'max_bin': 25,\n              'num_leaves': 31,\n              'min_child_samples': 1500,\n              'colsample_bytree': 0.7,\n              'subsample_freq': 1,\n              'subsample': 0.7,\n              'reg_alpha': 1.0,\n              'reg_lambda': 1.0,\n              'verbosity': 0,\n              'random_state': 0}\n    \nconfig = Config()","metadata":{"papermill":{"duration":0.012252,"end_time":"2022-07-12T13:42:44.399285","exception":false,"start_time":"2022-07-12T13:42:44.387033","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-30T08:36:42.583429Z","iopub.execute_input":"2022-07-30T08:36:42.583808Z","iopub.status.idle":"2022-07-30T08:36:42.591718Z","shell.execute_reply.started":"2022-07-30T08:36:42.583764Z","shell.execute_reply":"2022-07-30T08:36:42.590547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(config.input_path / 'train.csv', index_col='id')\ntest = pd.read_csv(config.input_path / 'test.csv', index_col='id')\nsubmission = pd.read_csv(config.input_path / 'sample_submission.csv', index_col='id')","metadata":{"papermill":{"duration":10.018154,"end_time":"2022-07-12T13:42:54.421457","exception":false,"start_time":"2022-07-12T13:42:44.403303","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-30T08:36:42.609667Z","iopub.execute_input":"2022-07-30T08:36:42.610351Z","iopub.status.idle":"2022-07-30T08:36:51.529346Z","shell.execute_reply.started":"2022-07-30T08:36:42.610315Z","shell.execute_reply":"2022-07-30T08:36:51.528218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calc_features = [feat for feat in train.columns if \"_calc\" in feat]\ncat_features = [feat for feat in train.columns if \"_cat\" in feat]","metadata":{"papermill":{"duration":0.014063,"end_time":"2022-07-12T13:42:54.439579","exception":false,"start_time":"2022-07-12T13:42:54.425516","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-30T08:36:51.531545Z","iopub.execute_input":"2022-07-30T08:36:51.531921Z","iopub.status.idle":"2022-07-30T08:36:51.538664Z","shell.execute_reply.started":"2022-07-30T08:36:51.531890Z","shell.execute_reply":"2022-07-30T08:36:51.537494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Extracting target\ntarget = train[\"target\"]\ntrain = train.drop(\"target\", axis=\"columns\")","metadata":{"papermill":{"duration":0.096975,"end_time":"2022-07-12T13:42:54.540693","exception":false,"start_time":"2022-07-12T13:42:54.443718","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-30T08:36:51.540547Z","iopub.execute_input":"2022-07-30T08:36:51.541011Z","iopub.status.idle":"2022-07-30T08:36:51.632945Z","shell.execute_reply.started":"2022-07-30T08:36:51.540967Z","shell.execute_reply":"2022-07-30T08:36:51.631838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Removing calc features\ntrain = train.drop(calc_features, axis=\"columns\")\ntest = test.drop(calc_features, axis=\"columns\")","metadata":{"papermill":{"duration":0.160122,"end_time":"2022-07-12T13:42:54.704868","exception":false,"start_time":"2022-07-12T13:42:54.544746","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-30T08:36:51.636522Z","iopub.execute_input":"2022-07-30T08:36:51.637026Z","iopub.status.idle":"2022-07-30T08:36:51.800379Z","shell.execute_reply.started":"2022-07-30T08:36:51.636979Z","shell.execute_reply":"2022-07-30T08:36:51.799034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Adding one-hot encoding of cat features\ntrain = pd.get_dummies(train, columns=cat_features)\ntest = pd.get_dummies(test, columns=cat_features)","metadata":{"papermill":{"duration":2.064956,"end_time":"2022-07-12T13:42:56.794724","exception":false,"start_time":"2022-07-12T13:42:54.729768","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-30T08:36:51.802200Z","iopub.execute_input":"2022-07-30T08:36:51.802558Z","iopub.status.idle":"2022-07-30T08:36:53.881784Z","shell.execute_reply.started":"2022-07-30T08:36:51.802526Z","shell.execute_reply":"2022-07-30T08:36:53.880483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert((train.columns==test.columns).all())","metadata":{"papermill":{"duration":0.013103,"end_time":"2022-07-12T13:42:56.811914","exception":false,"start_time":"2022-07-12T13:42:56.798811","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-30T08:36:53.883386Z","iopub.execute_input":"2022-07-30T08:36:53.883891Z","iopub.status.idle":"2022-07-30T08:36:53.890297Z","shell.execute_reply.started":"2022-07-30T08:36:53.883844Z","shell.execute_reply":"2022-07-30T08:36:53.889229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from numba import jit\n\n@jit\ndef eval_gini(y_true, y_pred):\n    y_true = np.asarray(y_true)\n    y_true = y_true[np.argsort(y_pred)]\n    ntrue = 0\n    gini = 0\n    delta = 0\n    n = len(y_true)\n    for i in range(n-1, -1, -1):\n        y_i = y_true[i]\n        ntrue += y_i\n        gini += y_i * delta\n        delta += 1 - y_i\n    gini = 1 - 2 * gini / (ntrue * (n - ntrue))\n    return gini\n\ndef gini_lgb(y_true, y_pred):\n    eval_name = 'normalized_gini_coef'\n    eval_result = eval_gini(y_true, y_pred)\n    is_higher_better = True\n    return eval_name, eval_result, is_higher_better","metadata":{"papermill":{"duration":0.015564,"end_time":"2022-07-12T13:42:56.831772","exception":false,"start_time":"2022-07-12T13:42:56.816208","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-30T08:36:53.891419Z","iopub.execute_input":"2022-07-30T08:36:53.892391Z","iopub.status.idle":"2022-07-30T08:36:54.888704Z","shell.execute_reply.started":"2022-07-30T08:36:53.892357Z","shell.execute_reply":"2022-07-30T08:36:54.887290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" if config.optuna_lgb:\n        \n    def objective(trial):\n        params = {\n                'learning_rate': trial.suggest_float(\"learning_rate\", 0.01, 1.0),\n                'num_leaves': trial.suggest_int(\"num_leaves\", 3, 255),\n                'min_child_samples': trial.suggest_int(\"min_child_samples\", 3, 3000),\n                'colsample_bytree': trial.suggest_float(\"colsample_bytree\", 0.1, 1.0),\n                'subsample_freq': trial.suggest_int(\"subsample_freq\", 0, 10),\n                'subsample': trial.suggest_float(\"subsample\", 0.1, 1.0),\n                'reg_alpha': trial.suggest_loguniform(\"reg_alpha\", 1e-9, 10.0),\n                'reg_lambda': trial.suggest_loguniform(\"reg_lambda\", 1e-9, 10.0),\n        }\n        \n        score = list()\n        skf = StratifiedKFold(n_splits=config.cv_folds, shuffle=True, random_state=config.random_state)\n\n        for train_idx, valid_idx in skf.split(train, target):\n            X_train, y_train = train.iloc[train_idx], target.iloc[train_idx]\n            X_valid, y_valid = train.iloc[valid_idx], target.iloc[valid_idx]\n\n            model = lgb.LGBMClassifier(**params,\n                                    n_estimators=1500,\n                                    early_stopping_round=150,\n                                    force_row_wise=True)\n\n            callbacks=[lgb.early_stopping(stopping_rounds=150, verbose=False)]\n            model.fit(X_train, y_train, eval_set=[(X_valid, y_valid)], eval_metric=gini_lgb, callbacks=callbacks)\n            score.append(model.best_score_['valid_0']['normalized_gini_coef'])\n\n        return np.mean(score)\n\n    study = optuna.create_study(direction=\"maximize\")\n    study.optimize(objective, n_trials=300)\n\n    print(\"Best Gini Normalized Score\", study.best_value)\n    print(\"Best parameters\", study.best_params)\n    \n    params = {'objective': 'binary',\n            'boosting_type': 'gbdt',\n            'verbosity': 0,\n            'random_state': 0}\n    \n    params.update(study.best_params)\n    \nelse:\n    params = config.params","metadata":{"papermill":{"duration":0.787336,"end_time":"2022-07-12T13:42:57.638456","exception":false,"start_time":"2022-07-12T13:42:56.851120","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-30T08:36:54.890712Z","iopub.execute_input":"2022-07-30T08:36:54.891409Z","iopub.status.idle":"2022-07-30T08:36:54.909661Z","shell.execute_reply.started":"2022-07-30T08:36:54.891360Z","shell.execute_reply":"2022-07-30T08:36:54.908812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = np.zeros(len(test))\noof = np.zeros(len(train))\nmetric_evaluations = list()\n\nskf = StratifiedKFold(n_splits=config.cv_folds, shuffle=True, random_state=config.random_state)\n\nfor idx, (train_idx, valid_idx) in enumerate(skf.split(train, target)):\n    print(f\"CV fold {idx}\")\n    X_train, y_train = train.iloc[train_idx], target.iloc[train_idx]\n    X_valid, y_valid = train.iloc[valid_idx], target.iloc[valid_idx]\n    \n    model = lgb.LGBMClassifier(**params,\n                               n_estimators=config.n_estimators,\n                               early_stopping_round=config.early_stopping_round,\n                               force_row_wise=True)\n    \n    callbacks=[lgb.early_stopping(stopping_rounds=150), \n               lgb.log_evaluation(period=100, show_stdv=False)]\n                                                                                           \n    model.fit(X_train, y_train, eval_set=[(X_valid, y_valid)], eval_metric=gini_lgb, callbacks=callbacks)\n    metric_evaluations.append(model.best_score_['valid_0']['normalized_gini_coef'])\n    preds += model.predict_proba(test, num_iteration=model.best_iteration_)[:,1] / skf.n_splits\n    oof[valid_idx] = model.predict_proba(X_valid, num_iteration=model.best_iteration_)[:,1]","metadata":{"papermill":{"duration":544.1184,"end_time":"2022-07-12T13:52:01.760880","exception":false,"start_time":"2022-07-12T13:42:57.642480","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-30T08:36:54.911143Z","iopub.execute_input":"2022-07-30T08:36:54.911617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"LightGBM CV Gini Normalized Score: {np.mean(metric_evaluations):0.3f} ({np.std(metric_evaluations):0.3f})\")","metadata":{"papermill":{"duration":0.015243,"end_time":"2022-07-12T13:52:01.783358","exception":false,"start_time":"2022-07-12T13:52:01.768115","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['target'] = preds\nsubmission.to_csv('lgb_submission.csv')","metadata":{"papermill":{"duration":2.318411,"end_time":"2022-07-12T13:52:04.108584","exception":false,"start_time":"2022-07-12T13:52:01.790173","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oofs = target.to_frame()\noofs['target'] = oof\noofs.to_csv('lgb_oof.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}