{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook have been prepared on the basis of LGBM baseline https://www.kaggle.com/ajcostarino/ingv-volcanic-eruption-prediction-lgbm-baseline, thanks to the author for sharing it. In my version, I've used the offered features, added some new and tuned xgboost regressor using scikit-optimizer. Hope this notebook will be useful for somebody.   "},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2020-10-13T17:35:57.667616Z","iopub.status.busy":"2020-10-13T17:35:57.666853Z","iopub.status.idle":"2020-10-13T17:35:57.670266Z","shell.execute_reply":"2020-10-13T17:35:57.669645Z"},"papermill":{"duration":0.021333,"end_time":"2020-10-13T17:35:57.670401","exception":false,"start_time":"2020-10-13T17:35:57.649068","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport numpy as np\nimport pandas as pd\n\nfrom time import time\nfrom time import ctime\n\nfrom sklearn.linear_model import LinearRegression\nimport scipy.stats as spstats\nfrom scipy.signal import hilbert\nfrom scipy.stats import iqr, gmean, sem, entropy\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom tqdm import tqdm_notebook\nfrom tqdm import tqdm\n\nimport joblib\nfrom joblib import Parallel, delayed\nimport multiprocessing\nnum_cores = multiprocessing.cpu_count()-1\n\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.metrics import r2_score\n\nRANDOM_STATE = 12061985\n\nimport matplotlib.pyplot as plt\n\nimport xgboost\nimport lightgbm as lgbm\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.model_selection import KFold,StratifiedKFold, RepeatedKFold\n\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import cross_validate\nfrom sklearn.model_selection import learning_curve\nfrom sklearn.model_selection import validation_curve\nfrom sklearn.model_selection import ShuffleSplit\nfrom sklearn.model_selection import StratifiedShuffleSplit\nfrom sklearn.model_selection import KFold\n\n# HPO\nfrom skopt.space import Integer, Categorical, Real\nfrom skopt.utils import use_named_args\nfrom skopt import gp_minimize, gbrt_minimize, forest_minimize\nfrom skopt.plots import plot_convergence\nfrom skopt.callbacks import DeltaXStopper, DeadlineStopper, DeltaYStopper\nfrom skopt.callbacks import EarlyStopper","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Additional methods and classes"},{"metadata":{},"cell_type":"markdown","source":"**Methods for features calculation**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def ChangeRate(x):\n    change = (np.diff(x) / x[:-1]).values\n    change = change[np.nonzero(change)[0]]\n    change = change[~np.isnan(change)]\n    change = change[change != -np.inf]\n    change = change[change != np.inf]\n    return np.mean(change)\n\ndef basic_statistics(t_X, x, s, sensor, postfix=''):\n    \"\"\"Computes basic statistics for the training feature set.\n    \n    Args:\n        t_X (pandas.DataFrame): The feature set being built.\n        x (pandas.Series): The signal values.\n        s (int): The integer number of the segment.\n        postfix (str): The postfix string value.\n    Return:\n        t_X (pandas.DataFrame): The feature set being built.\n    \"\"\"\n    t_X.loc[s, f'{sensor}_sum{postfix}']       = x.sum()\n    t_X.loc[s, f'{sensor}_mean{postfix}']      = x.mean()\n    t_X.loc[s, f'{sensor}_std{postfix}']       = x.std()\n    t_X.loc[s, f'{sensor}_var{postfix}']       = x.var() \n    t_X.loc[s, f'{sensor}_max{postfix}']       = x.max()\n    t_X.loc[s, f'{sensor}_min{postfix}']       = x.min()\n    t_X.loc[s, f'{sensor}_median{postfix}']    = x.median()\n    t_X.loc[s, f'{sensor}_skew{postfix}']      = x.skew()\n    t_X.loc[s, f'{sensor}_mad{postfix}']       = x.mad()\n    t_X.loc[s, f'{sensor}_kurtosis{postfix}']  = x.kurtosis()\n    \n    t_X.loc[s, f'{sensor}_ptp{postfix}']  = x.values.ptp()\n    t_X.loc[s, f'{sensor}_prod{postfix}'] = x.prod()\n    t_X.loc[s, f'{sensor}_iqr{postfix}']  = iqr(x)\n    t_X.loc[s, f'{sensor}_sem{postfix}']  = x.sem()\n    t_X.loc[s, f'{sensor}_gmean{postfix}']  = gmean(x)\n    t_X.loc[s, f'{sensor}_entropy{postfix}']  = entropy(x)\n    t_X.loc[s, f'{sensor}_chrate{postfix}']  = ChangeRate(x)\n\n    t_X.loc[s, f'{sensor}_hilmean{postfix}']  = np.abs(hilbert(x)).mean()\n    t_X.loc[s, f'{sensor}_countbig{postfix}']  = len(x[np.abs(x) > x.mean()])\n    t_X.loc[s, f'{sensor}_maxmindiff{postfix}']  = x.max() - np.abs(x.min())\n    t_X.loc[s, f'{sensor}_maxtomin{postfix}']  = x.max() / np.abs(x.min())\n    t_X.loc[s, f'{sensor}_meanchabs{postfix}']  = np.mean(np.diff(x))   \n\n    return t_X\n\n\ndef quantiles(t_X, x, s, sensor, postfix=''):\n    \"\"\"Calculates quantile features for the training feature set.\n    Args:\n        t_X (pandas.DataFrame): The feature set being built.\n        x (pandas.Series): The signal values.\n        s (int): The integer number of the segment.\n        postfix (str): The postfix string value.\n    Return:\n        t_X (pandas.DataFrame): The feature set being built.\n    \"\"\"\n    t_X.loc[s, f'{sensor}_q999{postfix}']     = np.quantile(x ,0.999)\n    t_X.loc[s, f'{sensor}_q99{postfix}']      = np.quantile(x, 0.99)\n    t_X.loc[s, f'{sensor}_q95{postfix}']      = np.quantile(x, 0.95)\n    t_X.loc[s, f'{sensor}_q87{postfix}']      = np.quantile(x, 0.87)\n    t_X.loc[s, f'{sensor}_q13{postfix}']      = np.quantile(x, 0.13)  \n    t_X.loc[s, f'{sensor}_q05{postfix}']      = np.quantile(x, 0.05)\n    t_X.loc[s, f'{sensor}_q01{postfix}']      = np.quantile(x, 0.01)\n    t_X.loc[s, f'{sensor}_q001{postfix}']     = np.quantile(x ,0.001)\n    \n    x_abs = np.abs(x)\n    t_X.loc[s, f'{sensor}_q999_abs{postfix}'] = np.quantile(x_abs, 0.999)\n    t_X.loc[s, f'{sensor}_q99_abs{postfix}']  = np.quantile(x_abs, 0.99)\n    t_X.loc[s, f'{sensor}_q95_abs{postfix}']  = np.quantile(x_abs, 0.95)\n    t_X.loc[s, f'{sensor}_q87_abs{postfix}']  = np.quantile(x_abs, 0.87)\n    t_X.loc[s, f'{sensor}_q13_abs{postfix}']  = np.quantile(x_abs, 0.13)\n    t_X.loc[s, f'{sensor}_q05_abs{postfix}']  = np.quantile(x_abs, 0.05)\n    t_X.loc[s, f'{sensor}_q01_abs{postfix}']  = np.quantile(x_abs, 0.01)\n    t_X.loc[s, f'{sensor}_q001_abs{postfix}'] = np.quantile(x_abs, 0.001)\n    \n    t_X.loc[s, f'{sensor}_iqr']     = np.subtract(*np.percentile(x, [75, 25]))\n    t_X.loc[s, f'{sensor}_iqr_abs'] = np.subtract(*np.percentile(x_abs, [75, 25]))\n\n    return t_X\n\ndef __linear_regression(arr, abs_v=False):\n    \"\"\"\n    \"\"\"\n    idx = np.array(range(len(arr)))\n    if abs_v:\n        arr = np.abs(arr)\n    lr = LinearRegression()\n    fit_X = idx.reshape(-1, 1)\n    lr.fit(fit_X, arr)\n    return lr.coef_[0]\n\n\ndef __classic_sta_lta(x, length_sta, length_lta):\n    sta = np.cumsum(x ** 2)\n    # Convert to float\n    sta = np.require(sta, dtype=np.float)\n    # Copy for LTA\n    lta = sta.copy()\n    # Compute the STA and the LTA\n    sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]\n    sta /= length_sta\n    lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]\n    lta /= length_lta\n    # Pad zeros\n    sta[:length_lta - 1] = 0\n    # Avoid division by zero by setting zero values to tiny float\n    dtiny = np.finfo(0.0).tiny\n    idx = lta < dtiny\n    lta[idx] = dtiny\n    return sta / lta\n\ndef linear_regression(t_X, x, s, sensor, postfix=''):\n    t_X.loc[s, f'{sensor}_lr_coef{postfix}'] = __linear_regression(x)\n    t_X.loc[s, f'{sensor}_lr_coef_abs{postfix}'] = __linear_regression(x, True)\n    return t_X\n\n\ndef classic_sta_lta(t_X, x, sensor, s):\n    t_X.loc[s, f'{sensor}_classic_sta_lta1_mean'] = __classic_sta_lta(x, 500, 10000).mean()\n    t_X.loc[s, f'{sensor}_classic_sta_lta2_mean'] = __classic_sta_lta(x, 5000, 100000).mean()\n    t_X.loc[s, f'{sensor}_classic_sta_lta3_mean'] = __classic_sta_lta(x, 3333, 6666).mean()\n    t_X.loc[s, f'{sensor}_classic_sta_lta4_mean'] = __classic_sta_lta(x, 10000, 25000).mean()\n    return t_X\n\n\ndef fft(t_X, x, s, sensor, postfix=''):\n    \"\"\"Generates basic statistics over the fft of the signal\"\"\"\n    z = np.fft.fft(x)\n    fft_real = np.real(z)\n    fft_imag = np.imag(z)\n\n    t_X.loc[s, f'fft_A0']             = abs(z[0])\n    t_X.loc[s, f'{sensor}_fft_real_mean{postfix}']      = fft_real.mean()\n    t_X.loc[s, f'{sensor}_fft_real_std{postfix}']       = fft_real.std()\n    t_X.loc[s, f'{sensor}_fft_real_max{postfix}']       = fft_real.max()\n    t_X.loc[s, f'{sensor}_fft_real_min{postfix}']       = fft_real.min()\n    t_X.loc[s, f'{sensor}_fft_real_median{postfix}']    = np.median(fft_real)\n    t_X.loc[s, f'{sensor}_fft_real_skew{postfix}']      = spstats.skew(fft_real)\n    t_X.loc[s, f'{sensor}_fft_real_kurtosis{postfix}']  = spstats.kurtosis(fft_real)\n    \n    t_X.loc[s, f'{sensor}_fft_imag_mean{postfix}']      = fft_imag.mean()\n    t_X.loc[s, f'{sensor}_fft_imag_std{postfix}']       = fft_imag.std()\n    t_X.loc[s, f'{sensor}_fft_imag_max{postfix}']       = fft_imag.max()\n    t_X.loc[s, f'{sensor}_fft_imag_min{postfix}']       = fft_imag.min()\n    t_X.loc[s, f'{sensor}_fft_imag_median{postfix}']    = np.median(fft_imag)\n    t_X.loc[s, f'{sensor}_fft_imag_skew{postfix}']      = spstats.skew(fft_imag)\n    t_X.loc[s, f'{sensor}_fft_imag_kurtosis{postfix}']  = spstats.kurtosis(fft_imag)\n    \n    return t_X","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Other methods**"},{"metadata":{"execution":{"iopub.execute_input":"2020-10-13T17:35:57.745442Z","iopub.status.busy":"2020-10-13T17:35:57.735014Z","iopub.status.idle":"2020-10-13T17:35:58.903426Z","shell.execute_reply":"2020-10-13T17:35:58.902715Z"},"papermill":{"duration":1.199203,"end_time":"2020-10-13T17:35:58.903551","exception":false,"start_time":"2020-10-13T17:35:57.704348","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def get_params_SKopt(model, X, Y, space, cv_search, alg = 'catboost', cat_features = None, eval_dataset = None, UBM = False, opt_method =\n                     'gbrt_minimize', verbose = True,  multi = False, scoring = 'neg_mean_squared_error', n_best = 50, total_time = 7200):\n    \"\"\"The method performs parameters tuning of an algorithm using scikit-optimize library.\n    Parameters:\n    1.\n    2.\n    3. multi - boolean, is used when a multioutput algorithm is tuned\n    UPDATES:\n    1. In this current version, the support of the catboost algorithms is added\n    \"\"\"\n    if alg == 'catboost':\n        fitparam = { 'eval_set' : eval_dataset,\n                     'use_best_model' : UBM,\n                     'cat_features' : cat_features,\n                     'early_stopping_rounds': 10 }\n    else:\n        fitparam = {}\n        \n    @use_named_args(space)\n    def objective(**params):\n        model.set_params(**params)\n        return -np.mean(cross_val_score(model, \n                                        X, Y, \n                                        cv=cv_search, \n                                        scoring= scoring,\n                                        fit_params=fitparam))\n    \n    if opt_method == 'gbrt_minimize':\n        \n        HPO_PARAMS = {'n_calls':1000,\n                      'n_random_starts':20,\n                      'acq_func':'EI',}\n        \n        reg_gp = gbrt_minimize(objective, \n                               space, \n                               n_jobs = -1,\n                               verbose = verbose,\n                               callback = [DeltaYStopper(delta = 0.01, n_best = 5), RepeatedMinStopper(n_best = n_best), DeadlineStopper(total_time = total_time)],\n                               **HPO_PARAMS,\n                               random_state = RANDOM_STATE)\n        \n\n    elif opt_method == 'forest_minimize':\n        \n        HPO_PARAMS = {'n_calls':1000,\n                      'n_random_starts':20,\n                      'acq_func':'EI',}\n        \n        reg_gp = forest_minimize(objective, \n                               space, \n                               n_jobs = -1,\n                               verbose = verbose,\n                               callback = [RepeatedMinStopper(n_best = n_best), DeadlineStopper(total_time = total_time)],\n                               **HPO_PARAMS,\n                               random_state = RANDOM_STATE)\n        \n    elif opt_method == 'gp_minimize':\n        \n        HPO_PARAMS = {'n_calls':1000,\n                      'n_random_starts':20,\n                      'acq_func':'gp_hedge',}        \n        \n        reg_gp = gp_minimize(objective, \n                               space, \n                               n_jobs = -1,\n                               verbose = verbose,\n                               callback = [RepeatedMinStopper(n_best = n_best), DeadlineStopper(total_time = total_time)],\n                               **HPO_PARAMS,\n                               random_state = RANDOM_STATE)\n    \n    TUNED_PARAMS = {} \n    for i, item in enumerate(space):\n        if multi:\n            TUNED_PARAMS[item.name.split('__')[1]] = reg_gp.x[i]\n        else:\n            TUNED_PARAMS[item.name] = reg_gp.x[i]\n    \n    return [TUNED_PARAMS,reg_gp]\n\nclass RepeatedMinStopper(EarlyStopper):\n    \"\"\"Stop the optimization when there is no improvement in the minimum.\n    Stop the optimization when there is no improvement in the minimum\n    achieved function evaluation after `n_best` iterations.\n    \"\"\"\n    def __init__(self, n_best=50):\n        super(EarlyStopper, self).__init__()\n        self.n_best = n_best\n        self.count = 0\n        self.minimum = np.finfo(np.float).max\n\n    def _criterion(self, result):\n        if result.fun < self.minimum:\n            self.minimum = result.fun\n            self.count = 0\n        elif result.fun > self.minimum:\n            self.count = 0\n        else:\n            self.count += 1\n\n        return self.count >= self.n_best\n\ndef plotfig (ypred, yactual, strtitle, y_max):\n    plt.scatter(ypred, yactual.values.ravel())\n    plt.title(strtitle)\n    plt.plot([(0, 0), (y_max, y_max)], [(0, 0), (y_max, y_max)])\n    plt.xlim(0, y_max)\n    plt.ylim(0, y_max)\n    plt.xlabel('Predicted', fontsize=12)\n    plt.ylabel('Actual', fontsize=12)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.011327,"end_time":"2020-10-13T17:35:58.926548","exception":false,"start_time":"2020-10-13T17:35:58.915221","status":"completed"},"tags":[]},"cell_type":"markdown","source":"### Prepare train dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"n_files = None\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def features_generator(path_to_file):\n    signals = pd.read_csv(path_to_file)\n    seg = int(path_to_file.split('/')[-1].split('.')[0])\n    row = pd.DataFrame([])\n    \n    signals = signals.rolling(10).mean().iloc[list(np.arange(10,60001,10))]\n    \n    for i in range(1, 11):\n        sensor_id = f'sensor_{i}'\n\n#         row = basic_statistics(row, signals[sensor_id].interpolate(method='polynomial', order=2), seg, sensor_id, postfix='')\n#         row = basic_statistics(row, signals[sensor_id].interpolate(method='polynomial', order=2), seg, sensor_id, postfix='')\n#         row = basic_statistics(row, signals[sensor_id].interpolate(method='polynomial', order=2), seg, sensor_id, postfix='')\n#         row = basic_statistics(row, signals[sensor_id].interpolate(method='polynomial', order=2), seg, sensor_id, postfix='')\n        \n        \n        row = basic_statistics(row, signals[sensor_id].fillna(0), seg, sensor_id, postfix='')\n        row = basic_statistics(row, signals[sensor_id].fillna(0), seg, sensor_id, postfix='')\n        row = basic_statistics(row, signals[sensor_id].fillna(0), seg, sensor_id, postfix='')\n        row = basic_statistics(row, signals[sensor_id].fillna(0), seg, sensor_id, postfix='')\n    return row","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path_to_signals = '/kaggle/input/predict-volcanic-eruptions-ingv-oe/train/'\ntrain_files_list = [os.path.join(train_path_to_signals, file) for file in os.listdir(train_path_to_signals)]\nrows = Parallel(n_jobs=-1)(delayed(features_generator)(ex) for ex in tqdm(train_files_list[:n_files]))  \ntrain_set = pd.concat(rows, axis=0)\ntrain_set.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/predict-volcanic-eruptions-ingv-oe/train.csv')\ntrain = train.set_index('segment_id')\ntrain_set = pd.concat([train_set, train], axis = 1)\ntrain_set.head()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.01143,"end_time":"2020-10-13T18:48:23.536923","exception":false,"start_time":"2020-10-13T18:48:23.525493","status":"completed"},"tags":[]},"cell_type":"markdown","source":"### Prepare test dataset"},{"metadata":{"execution":{"iopub.execute_input":"2020-10-13T18:48:23.575482Z","iopub.status.busy":"2020-10-13T18:48:23.574294Z","iopub.status.idle":"2020-10-13T20:02:37.982764Z","shell.execute_reply":"2020-10-13T20:02:37.980843Z"},"papermill":{"duration":4454.434516,"end_time":"2020-10-13T20:02:37.98308","exception":false,"start_time":"2020-10-13T18:48:23.548564","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"test_path_to_signals = '/kaggle/input/predict-volcanic-eruptions-ingv-oe/test/'\ntest_files_list = [os.path.join(test_path_to_signals, file) for file in os.listdir(test_path_to_signals)]\nrows = Parallel(n_jobs=-1)(delayed(features_generator)(ex) for ex in tqdm(test_files_list[:n_files]))  \ntest_set = pd.concat(rows, axis=0)\ntest_set.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/predict-volcanic-eruptions-ingv-oe/sample_submission.csv')\ntest = test.set_index('segment_id')\ntest_set = pd.concat([test_set, test], axis = 1)\ntest_set.head()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.011902,"end_time":"2020-10-13T20:02:38.010605","exception":false,"start_time":"2020-10-13T20:02:37.998703","status":"completed"},"tags":[]},"cell_type":"markdown","source":"### Tune and train xgb regressor "},{"metadata":{"execution":{"iopub.execute_input":"2020-10-13T20:02:38.062933Z","iopub.status.busy":"2020-10-13T20:02:38.058956Z","iopub.status.idle":"2020-10-13T23:08:15.923361Z","shell.execute_reply":"2020-10-13T23:08:15.922504Z"},"papermill":{"duration":11137.901111,"end_time":"2020-10-13T23:08:15.923529","exception":false,"start_time":"2020-10-13T20:02:38.022418","status":"completed"},"scrolled":false,"tags":[],"trusted":true},"cell_type":"code","source":"%%time\n\nY = train_set['time_to_eruption']\nX = train_set.drop(['time_to_eruption'], axis = 1)\n\nX_test = test_set.drop(['time_to_eruption'], axis = 1)\n\nSTATIC_PARAMS = {\n                    'n_estimators': 250,\n                    'objective' : 'reg:squarederror',\n                    'random_state' : RANDOM_STATE,\n                    'n_jobs': -1, \n                }\n\nspace_SKopt = [\n                 Integer(2, 50, name='max_depth'),\n                 Integer(2, 500, name='min_child_weight'),\n                 Real(0.005, .05, name='learning_rate'),\n                 Real(0.1, 1, name='subsample'),\n                 Real(0.1, 1, name='colsample_bytree'),\n                 Real(0.1, 10, name='reg_alpha'),\n                 Real(0.1, 10, name='reg_lambda')\n               ]\n\nmodel = xgboost.XGBRegressor(**STATIC_PARAMS)\n\nX_train_tune, X_test_tune, y_train_tune, y_test_tune = train_test_split(X, Y, \n                                                                        test_size=0.3, \n                                                                        shuffle = True,\n                                                                        random_state=RANDOM_STATE)\nbest_alg_params = {}\n\neval_dataset = [(X_test_tune, y_test_tune)]\n\nn_fold = 5\ncv_tune = KFold(n_splits=n_fold, shuffle=True, random_state=RANDOM_STATE)\n\nstart_time = time()\n[TUNED_alg_PARAMS,reg_gp] = get_params_SKopt(\n                                                model, \n                                                 X_train_tune, y_train_tune, \n                                                 space_SKopt, \n                                                 cv_tune,\n                                                 alg = 'xgboost',\n                                                 cat_features = [],\n                                                 eval_dataset = eval_dataset,\n                                                 UBM = True,\n                                                 opt_method = 'forest_minimize',\n                                                 verbose = True,\n                                                 multi = False, \n                                                 scoring = 'neg_mean_absolute_error', \n                                                 n_best = 30,\n                                                 total_time = 10800,                                       \n                                            )\n\nprint('\\nTime for tuning: {0:.2f} minutes'.format((time() - start_time)/60))\nSTATIC_PARAMS['n_estimators'] = 2000\nNEW_PARAMS = {**STATIC_PARAMS, **TUNED_alg_PARAMS} \nprint(NEW_PARAMS)\n\nprint('Достигнутое значение метрики: ', reg_gp.fun)\n\nn_fold = 5\ncv = KFold(n_splits=n_fold, shuffle=True, random_state=RANDOM_STATE)\n\noof = np.zeros(len(X))\nprediction = np.zeros(len(X_test))\nmae, r2 = [], []\n\nfor fold_n, (train_index, valid_index) in enumerate(cv.split(X)):\n    print('\\nFold', fold_n, 'started at', ctime())\n\n    X_train = X.iloc[train_index,:]\n    X_valid = X.iloc[valid_index,:]\n    \n    Y_train = Y.iloc[train_index]\n    Y_valid = Y.iloc[valid_index]\n          \n    best_model = xgboost.XGBRegressor(**NEW_PARAMS)\n    \n    best_model.fit(X_train, Y_train, \n           eval_metric='mae',    \n           eval_set=[(X_train, Y_train), (X_valid, Y_valid)],\n           verbose=False,\n           early_stopping_rounds = 100)\n      \n    y_pred = best_model.predict(X_valid, \n                               ntree_limit = best_model.best_ntree_limit)\n\n    mae.append(mean_absolute_error(Y_valid, y_pred))\n    r2.append(r2_score(Y_valid, y_pred))\n\n    print('MAE: ', mean_absolute_error(Y_valid, y_pred))\n    print('R2: ', r2_score(Y_valid, y_pred))\n\n    prediction += best_model.predict(X_test,\n                                    ntree_limit = best_model.best_ntree_limit)\n        \nprediction /= n_fold\n\nprint('='*45)\nprint('CV mean MAE: {0:.4f}, std: {1:.4f}.'.format(np.mean(mae), np.std(mae)))\nprint('CV mean R2:  {0:.4f}, std: {1:.4f}.'.format(np.mean(r2), np.std(r2)))\n\nplotfig(best_model.predict(X), Y, 'Predicted vs. Actual responses for XGB', max(Y) + 0.1*max(Y))\n","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.068939,"end_time":"2020-10-13T23:08:41.398725","exception":false,"start_time":"2020-10-13T23:08:41.329786","status":"completed"},"tags":[]},"cell_type":"markdown","source":"### Submission"},{"metadata":{"execution":{"iopub.execute_input":"2020-10-13T23:08:41.545842Z","iopub.status.busy":"2020-10-13T23:08:41.545081Z","iopub.status.idle":"2020-10-13T23:08:42.090469Z","shell.execute_reply":"2020-10-13T23:08:42.089643Z"},"papermill":{"duration":0.623361,"end_time":"2020-10-13T23:08:42.090599","exception":false,"start_time":"2020-10-13T23:08:41.467238","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame()\nsubmission['segment_id'] = test_set.index\nsubmission['time_to_eruption'] = prediction\nsubmission.to_csv('submission.csv', header=True, index=False)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.068863,"end_time":"2020-10-13T23:08:42.229149","exception":false,"start_time":"2020-10-13T23:08:42.160286","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}