{"cells":[{"metadata":{},"cell_type":"markdown","source":"## General information\n\nIn this kernel I'll try various technics for models interpretability and feature selection. Also I'll compare various models.\n\nI use the features from my dataset: https://www.kaggle.com/artgor/lanl-features\n\nThis dataset was created using this kernel: https://www.kaggle.com/artgor/even-more-features/\n\n**UPD**: Thanks to the new kaggle update we can write code in kernels and import it. This is much more convenient and useful.\nI'm moving all the functions I can into this script: https://www.kaggle.com/artgor/artgor-utils\nSo if you see somewhere code like `artgot_utils.function_name(parameters)` - it is from this script\n\n![](https://torontoseoulcialite.com/wp-content/uploads/2016/02/zimbiocom.jpg)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom tqdm import tqdm_notebook\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.svm import NuSVR, SVR\nfrom sklearn.metrics import mean_absolute_error\npd.options.display.precision = 15\nfrom sklearn.model_selection import train_test_split, KFold, cross_val_score, StratifiedKFold, GridSearchCV\nfrom sklearn import svm\nimport lightgbm as lgb\nimport xgboost as xgb\nimport time\nimport datetime\nfrom catboost import CatBoostRegressor\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn import neighbors\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn import linear_model\nimport gc\nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom sklearn.ensemble import ExtraTreesRegressor, AdaBoostRegressor\n\nfrom scipy.signal import hilbert\nfrom scipy.signal import hann\nfrom scipy.signal import convolve\nfrom scipy import stats\nfrom sklearn.kernel_ridge import KernelRidge\nfrom sklearn.neighbors import NearestNeighbors\nimport librosa, librosa.display\nimport builtins\nfrom sklearn.ensemble import RandomForestRegressor\nimport eli5\nimport shap\nfrom sklearn.feature_selection import GenericUnivariateSelect, SelectPercentile, SelectKBest, f_classif, mutual_info_classif, RFE\n\nfrom IPython.display import HTML\nimport json\nimport altair as alt\n\nimport artgor_utils\n\n# setting up altair\nworkaround = artgor_utils.prepare_altair()\nHTML(\"\".join((\n    \"<script>\",\n    workaround,\n    \"</script>\",\n)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir('../input/lanl-features')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* 'train_features.csv' - train features generated on original data\n* 'train_features_denoised.csv' - train features generated on denoised data\n* 'test_features.csv' - test features generated on original data\n* 'test_features_denoised.csv' - test features generated on denoised data\n* 'submission_1.csv' - one of my local submissions\n* 'y.csv' - train target"},{"metadata":{},"cell_type":"markdown","source":"## Loading data\nLet's load features!"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_features = pd.read_csv('../input/lanl-features/train_features.csv')\ntest_features = pd.read_csv('../input/lanl-features/test_features.csv')\ntrain_features_denoised = pd.read_csv('../input/lanl-features/train_features_denoised.csv')\ntest_features_denoised = pd.read_csv('../input/lanl-features/test_features_denoised.csv')\ntrain_features_denoised.columns = [f'{i}_denoised' for i in train_features_denoised.columns]\ntest_features_denoised.columns = [f'{i}_denoised' for i in test_features_denoised.columns]\ny = pd.read_csv('../input/lanl-features/y.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = pd.concat([train_features, train_features_denoised], axis=1).drop(['seg_id_denoised', 'target_denoised'], axis=1)\nX_test = pd.concat([test_features, test_features_denoised], axis=1).drop(['seg_id_denoised', 'target_denoised'], axis=1)\nX = X[:-1]\ny = y[:-1]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We have almost 2000 features here!"},{"metadata":{"trusted":true},"cell_type":"code","source":"n_fold = 10\nfolds = KFold(n_splits=n_fold, shuffle=True, random_state=11)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Basic model\n\nTraining function is imported from my script. Important changes from the code, which I used previously:\n- function returns dictionary with oof, test predictions and scores. Also with feature importances, if necessary;\n- in future it will be easier to change metrics."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"params = {'num_leaves': 128,\n          'min_child_samples': 79,\n          'objective': 'gamma',\n          'max_depth': -1,\n          'learning_rate': 0.01,\n          \"boosting_type\": \"gbdt\",\n          \"subsample_freq\": 5,\n          \"subsample\": 0.9,\n          \"bagging_seed\": 11,\n          \"metric\": 'mae',\n          \"verbosity\": -1,\n          'reg_alpha': 0.1302650970728192,\n          'reg_lambda': 0.3603427518866501,\n          'colsample_bytree': 0.1\n         }\nresult_dict_lgb = artgor_utils.train_model_regression(X=X, X_test=X_test, y=y, params=params, folds=folds, model_type='lgb',\n                                                                                  eval_metric='mae', plot_feature_importance=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/LANL-Earthquake-Prediction/sample_submission.csv', index_col='seg_id')\nsubmission['time_to_failure'] = result_dict_lgb['prediction']\nprint(submission.head())\nsubmission.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub1 = pd.read_csv('../input/lanl-features/submission_1.csv')\nsub1.to_csv('submission_1.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## NN features\nHere I normalize the data and create features using NearestNeighbors. The idea is to find samples which are similar and use it to generate features."},{"metadata":{"trusted":true},"cell_type":"code","source":"scaler = StandardScaler()\nscaler.fit(X)\nX_train_scaled = pd.DataFrame(scaler.transform(X), columns=X.columns)\nX_test_scaled = pd.DataFrame(scaler.transform(X_test), columns=X_test.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nn = 10\nneigh = NearestNeighbors(n, n_jobs=-1)\nneigh.fit(X_train_scaled)\n\ndists, _ = neigh.kneighbors(X_train_scaled, n_neighbors=n)\nmean_dist = dists.mean(axis=1)\nmax_dist = dists.max(axis=1)\nmin_dist = dists.min(axis=1)\n\nX_train_scaled['mean_dist'] = mean_dist\nX_train_scaled['max_dist'] = max_dist\nX_train_scaled['min_dist'] = min_dist\n\ntest_dists, _ = neigh.kneighbors(X_test_scaled, n_neighbors=n)\n\ntest_mean_dist = test_dists.mean(axis=1)\ntest_max_dist = test_dists.max(axis=1)\ntest_min_dist = test_dists.min(axis=1)\n\nX_test_scaled['mean_dist'] = test_mean_dist\nX_test_scaled['max_dist'] = test_max_dist\nX_test_scaled['min_dist'] = test_min_dist\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params = {'num_leaves': 32,\n          'min_data_in_leaf': 79,\n          'objective': 'gamma',\n          'max_depth': -1,\n          'learning_rate': 0.01,\n          \"boosting\": \"gbdt\",\n          \"bagging_freq\": 5,\n          \"bagging_fraction\": 0.8126672064208567,\n          \"bagging_seed\": 11,\n          \"metric\": 'mae',\n          \"verbosity\": -1,\n          'reg_alpha': 0.1302650970728192,\n          'reg_lambda': 0.3603427518866501,\n          'feature_fraction': 0.1\n         }\nresult_dict_lgb = artgor_utils.train_model_regression(X=X, X_test=X_test, y=y, params=params, folds=folds, model_type='lgb',\n                                                                                  eval_metric='mae', plot_feature_importance=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['time_to_failure'] = result_dict_lgb['prediction']\nsubmission.to_csv('submission_nn.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model interpretation\n"},{"metadata":{},"cell_type":"markdown","source":"### ELI5 and permutation importance\nELI5 is a package with provides explanations for ML models. It can do this not only for linear models, but also for tree based like Random Forest or lightgbm.\n\n**Important notice**: running eli5 on all features takes a lot of time, so I run the cell below in `version 14` and printed the top-50 features. In the following versions I'll use these 50 columns and use eli5 to find top-40 of them so that it takes less time"},{"metadata":{"trusted":true},"cell_type":"code","source":"top_columns = ['iqr1_denoised', 'percentile_5_denoised', 'abs_percentile_90_denoised', 'percentile_95_denoised', 'ave_roll_std_10', 'num_peaks_10', 'percentile_roll_std_20',\n               'ratio_unique_values_denoised', 'fftr_percentile_roll_std_75_denoised', 'num_crossing_0_denoised', 'percentile_95', 'ffti_percentile_roll_std_75_denoised',\n               'min_roll_std_10000', 'percentile_roll_std_1', 'percentile_roll_std_10', 'fftr_percentile_roll_std_70_denoised', 'ave_roll_std_50', 'ffti_percentile_roll_std_70_denoised',\n               'exp_Moving_std_300_mean_denoised', 'ffti_percentile_roll_std_30_denoised', 'mean_change_rate', 'percentile_roll_std_5', 'range_-1000_0', 'mad',\n               'fftr_range_1000_2000_denoised', 'percentile_10_denoised', 'ffti_percentile_roll_std_80', 'percentile_roll_std_25', 'fftr_percentile_10_denoised',\n               'ffti_range_-2000_-1000_denoised', 'autocorrelation_5', 'min_roll_std_100', 'fftr_percentile_roll_std_80', 'min_roll_std_500', 'min_roll_std_50', 'min_roll_std_1000',\n               'ffti_percentile_20_denoised', 'iqr1', 'classic_sta_lta5_mean_denoised', 'classic_sta_lta6_mean_denoised', 'percentile_roll_std_10_denoised',\n               'fftr_percentile_70_denoised', 'ffti_c3_50_denoised', 'ffti_percentile_roll_std_75', 'abs_percentile_90', 'range_0_1000', 'spkt_welch_density_50_denoised',\n               'ffti_percentile_roll_std_40_denoised', 'ffti_range_-4000_-3000', 'mean_change_rate_last_50000']\n\n\nX_train, X_valid, y_train, y_valid = train_test_split(X[top_columns], y, test_size=0.1)\nmodel = lgb.LGBMRegressor(**params, n_estimators = 50000, n_jobs = -1, verbose=-1)\nmodel.fit(X_train, y_train, \n        eval_set=[(X_train, y_train), (X_valid, y_valid)], eval_metric='mae',\n        verbose=10000, early_stopping_rounds=200)\n\nperm = eli5.sklearn.PermutationImportance(model, random_state=1).fit(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"eli5.show_weights(perm, top=50, feature_names=top_columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"top_features = [i for i in eli5.formatters.as_dataframe.explain_weights_df(model).feature if 'BIAS' not in i][:40]\nresult_dict_lgb = artgor_utils.train_model_regression(X, X_test, y, params=params, folds=folds, model_type='lgb', plot_feature_importance=True, columns=top_features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['time_to_failure'] = result_dict_lgb['prediction']\nsubmission.to_csv('submission_eli5.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Feature selection\n\nHere I try various approaches to feature selection.\n\n**Important notice**: running feature selection on all features takes a lot of time, so I'll run some of feature selection methods and print the result, which I'll use in the following versions of the kernel, so that I can explore more approaches."},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"params = {'num_leaves': 32,\n          'min_child_samples': 79,\n          'objective': 'gamma',\n          'max_depth': -1,\n          'learning_rate': 0.01,\n          \"boosting_type\": \"gbdt\",\n          \"subsample_freq\": 5,\n          \"subsample\": 0.9,\n          \"bagging_seed\": 11,\n          \"metric\": 'mae',\n          \"verbosity\": -1,\n          'reg_alpha': 0.1302650970728192,\n          'reg_lambda': 0.3603427518866501,\n          'colsample_bytree': 1.0\n         }","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### SelectPercentile\n\n**Important notice**:  I run the cell below in `version 14` and printed the scores_dict. In the following versions I'll use `scores_dict` and plot the results instead of running feature selection each time"},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"# %%time\n# scores_dict = {'f_classif': [], 'mutual_info_classif': [], 'n_features': []}\n# for i in range(5, 105, 5):\n#     print(i)\n#     s1 = SelectPercentile(f_classif, percentile=i)\n#     X_train1 = s1.fit_transform(X, y.values.astype(int))\n#     X_test1 = s1.transform(X_test)\n#     result_dict_lgb = artgor_utils.train_model_regression(X_train1, X_test1, y.values.reshape(-1, ), params=params, folds=folds, model_type='lgb', plot_feature_importance=False)\n#     scores_dict['f_classif'].append(np.mean(result_dict_lgb['scores']))\n    \n#     s2 = SelectPercentile(mutual_info_classif, percentile=i)\n#     X_train1 = s2.fit_transform(X, y.values.astype(int))\n#     X_test1 = s2.transform(X_test)\n#     result_dict_lgb = artgor_utils.train_model_regression(X_train1, X_test1, y.values.reshape(-1, ), params=params, folds=folds, model_type='lgb', plot_feature_importance=False)\n#     scores_dict['mutual_info_classif'].append(np.mean(result_dict_lgb['scores']))\n    \n#     scores_dict['n_features'].append(X_train1.shape[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scores_dict = {'f_classif': [2.0746468465171377, 2.0753843541953687, 2.062191535440333, 2.0654327826583034, 2.0643551320704936, 2.0617560048382675,\n                             2.061565197738015, 2.0598878198917494, 2.0654865223333143, 2.0632788555735777, 2.058002635080971, 2.051075689018734,\n                             2.0472543961304583, 2.052401474353084, 2.055924154798443, 2.0561794619762352, 2.0549680611994963, 2.057123777802326,\n                             2.0591868861136904, 2.0577745274024553],\n               'mutual_info_classif': [2.0866763775014006, 2.0745431497064324, 2.0564324832516427, 2.060125564781158, 2.067334544167612, 2.0665943783246448,\n                                       2.063891669849029, 2.070194051004794, 2.0667490707700447, 2.0681653852378354, 2.0592743636982345, 2.061260741522344,\n                                       2.05680667824411, 2.0565047875243003, 2.058252567141659, 2.0554927194831922, 2.0562776429736873, 2.0618179277444084,\n                                       2.06364125584214, 2.0577745274024553],\n               'n_features': [98, 196, 294, 392, 490, 588, 685, 783, 881, 979, 1077, 1175, 1273, 1370, 1468, 1566, 1664, 1762, 1860, 1958]}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scores_df = pd.DataFrame(scores_dict)\nscores_df = scores_df.melt(id_vars=['n_features'], value_vars=['mutual_info_classif', 'f_classif'], var_name='metric', value_name='mae')\nmax_value = scores_df['mae'].max() * 1.05\nmin_value = scores_df['mae'].min() * 0.95\nartgor_utils.render(alt.Chart(scores_df).mark_line().encode(\n    y=alt.Y('mae:Q', scale=alt.Scale(domain=(min_value, max_value))),\n    x='n_features:O',\n    color='metric:N',\n    tooltip=['metric:N', 'n:O', 'mae:Q']\n).properties(\n    title='Top N features by SelectPercentile vs CV'\n).interactive())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### SelectKBest\n\n**Important notice**:  I run the cell below in `version 14` and printed the scores_dict. In the following versions I'll use `scores_dict` and plot the results instead of running feature selection each time"},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"# %%time\n# scores_dict = {'f_classif': [], 'mutual_info_classif': [], 'n_features': []}\n# for i in np.arange(10, 1958, 100):\n#     print(i)\n#     s1 = SelectKBest(f_classif, k=i)\n#     X_train1 = s1.fit_transform(X, y.values.astype(int))\n#     X_test1 = s1.transform(X_test)\n#     result_dict_lgb = artgor_utils.train_model_regression(X_train1, X_test1, y.values.reshape(-1, ), params=params, folds=folds, model_type='lgb', plot_feature_importance=False)\n#     scores_dict['f_classif'].append(np.mean(result_dict_lgb['scores']))\n    \n#     s2 = SelectKBest(mutual_info_classif, k=i)\n#     X_train1 = s2.fit_transform(X, y.values.astype(int))\n#     X_test1 = s2.transform(X_test)\n#     result_dict_lgb = artgor_utils.train_model_regression(X_train1, X_test1, y.values.reshape(-1, ), params=params, folds=folds, model_type='lgb', plot_feature_importance=False)\n#     scores_dict['mutual_info_classif'].append(np.mean(result_dict_lgb['scores']))\n    \n#     scores_dict['n_features'].append(X_train1.shape[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scores_dict = {'f_classif': [2.1495892622081354, 2.0778182269587147, 2.0716153738740006, 2.06152950679902, 2.0645162758752553, 2.0627705797004032, 2.0610992303725157,\n                             2.057762113735462, 2.0618360883613627, 2.0603197111525984, 2.06081274633874, 2.0580767195278056, 2.0527646572747127, 2.0498353445032533,\n                             2.052442594925, 2.0564456881902133, 2.0582284644115365, 2.0558612960548635, 2.0580900016350094, 2.058218782401599],\n               'mutual_info_classif': [2.1235703196243687, 2.084958198672301, 2.0596822478390955, 2.053305869981444, 2.063468853227225, 2.0674399950434323, 2.0658618511287874,\n                                       2.063003703200445, 2.0653174905858664, 2.0644340327023656, 2.0748993062333523, 2.0587602096358113, 2.0601495560836076, 2.0559629138548603,\n                                       2.0553852701221134, 2.058022171415446, 2.060755947658241, 2.057916705462307, 2.056245795262636, 2.0580691870837056],\n               'n_features': [10, 110, 210, 310, 410, 510, 610, 710, 810, 910, 1010, 1110, 1210, 1310, 1410, 1510, 1610, 1710, 1810, 1910]}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scores_df = pd.DataFrame(scores_dict)\nscores_df = scores_df.melt(id_vars=['n_features'], value_vars=['mutual_info_classif', 'f_classif'], var_name='metric', value_name='mae')\nmax_value = scores_df['mae'].max() * 1.05\nmin_value = scores_df['mae'].min() * 0.95\nartgor_utils.render(alt.Chart(scores_df).mark_line().encode(\n    y=alt.Y('mae:Q', scale=alt.Scale(domain=(min_value, max_value))),\n    x='n_features:O',\n    color='metric:N',\n    tooltip=['metric:N', 'n:O', 'mae:Q']\n).properties(\n    title='Top N features by SelectKBest vs CV'\n).interactive())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dropping highly correlated features\n\nDue to the huge number of features there are certainly some highly correlated features, let's try droping them!"},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://chrisalbon.com/machine_learning/feature_selection/drop_highly_correlated_features/\ncorr_matrix = X.corr().abs()\n\n# Select upper triangle of correlation matrix\nupper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))\n\n# Find index of feature columns with correlation greater than 0.95\nto_drop = [column for column in upper.columns if any(upper[column] > 0.99)]\nX = X.drop(to_drop, axis=1)\nX_test = X_test.drop(to_drop, axis=1)\nresult_dict_lgb_lgb = artgor_utils.train_model_regression(X, X_test, y, params=params, folds=folds, model_type='lgb', plot_feature_importance=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['time_to_failure'] = result_dict_lgb['prediction']\nsubmission.to_csv('submission_no_corr.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Important**: from now I'll use the reduced dataset - without highly correlated features."},{"metadata":{},"cell_type":"markdown","source":"### RFE\n\n\n**Important notice**:  I run the cell below in `version 18` and printed the scores_dict. In the following versions I'll use `scores_dict` and plot the results instead of running feature selection each time"},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"# %%time\n# scores_dict = {'rfe_score': [], 'n_features': []}\n# for i in np.arange(10, 1958, 100)[:3]:\n#     print(i)\n#     s1 = RFE(model, i, step=100)\n#     X_train1 = s1.fit_transform(X, y.values.astype(int))\n#     X_test1 = s1.transform(X_test)\n#     result_dict_lgb = artgor_utils.train_model_regression(X_train1, X_test1, y.values.reshape(-1, ), params=params, folds=folds, model_type='lgb', plot_feature_importance=False)\n#     scores_dict['rfe_score'].append(np.mean(result_dict_lgb['scores']))\n    \n#     scores_dict['n_features'].append(X_train1.shape[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scores_dict = {'rfe_score': [2.103586938061856, 2.052535910798748, 2.053228199447811], 'n_features': [10, 110, 210]}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scores_df = pd.DataFrame(scores_dict)\nscores_df = scores_df.melt(id_vars=['n_features'], value_vars=['rfe_score'], var_name='metric', value_name='mae')\nmax_value = scores_df['mae'].max() * 1.05\nmin_value = scores_df['mae'].min() * 0.95\nartgor_utils.render(alt.Chart(scores_df).mark_line().encode(\n    y=alt.Y('mae:Q', scale=alt.Scale(domain=(min_value, max_value))),\n    x='n_features:O',\n    color='metric:N',\n    tooltip=['metric:N', 'n:O', 'mae:Q']\n).properties(\n    title='Top N features by RFE vs CV'\n).interactive())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model comparison\n\nIn this section I'll try variuos sklearn models and compair their score. Running GridSearchCV each time is too long, so I'll run it once for each model and use optimal parameters."},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nrfr = RandomForestRegressor()\n\n# parameter_grid = {'n_estimators': [50, 60],\n#                   'max_depth': [5, 10]\n#                  }\n\n# grid_search = GridSearchCV(rfr, param_grid=parameter_grid, cv=folds, scoring='neg_mean_absolute_error', n_jobs=-1)\n# grid_search.fit(X, y)\n# print('Best score: {}'.format(grid_search.best_score_))\n# print('Best parameters: {}'.format(grid_search.best_params_))\n# rfr = RandomForestRegressor(**grid_search.best_params_)\nrfr = RandomForestRegressor(n_estimators=50, max_depth=5)\nresult_dict_rfr = artgor_utils.train_model_regression(X, X_test, y, params=params, folds=folds, model_type='sklearn', model=rfr)\n# print(scores_rfr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nlinreg = linear_model.LinearRegression(normalize=False, copy_X=True, n_jobs=-1)\n\nresult_dict_linreg = artgor_utils.train_model_regression(X, X_test, y, params=None, folds=folds, model_type='sklearn', model=linreg)\n# print(scores_linreg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nridge = linear_model.Ridge(normalize=True)\n\nparameter_grid = {'alpha': [0.1, 1.0, 10.0]}\n\ngrid_search = GridSearchCV(ridge, param_grid=parameter_grid, cv=folds, scoring='neg_mean_absolute_error', n_jobs=-1)\ngrid_search.fit(X, y)\nprint('Best score: {}'.format(grid_search.best_score_))\nprint('Best parameters: {}'.format(grid_search.best_params_))\nridge = linear_model.Ridge(**grid_search.best_params_, normalize=True)\nresult_dict_ridge = artgor_utils.train_model_regression(X, X_test, y, params=params, folds=folds, model_type='sklearn', model=ridge)\n# print(scores_ridge)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nknn = neighbors.KNeighborsRegressor()\n\nparameter_grid = {'n_neighbors': [50, 100]}\n\ngrid_search = GridSearchCV(knn, param_grid=parameter_grid, cv=folds, scoring='neg_mean_absolute_error', n_jobs=-1)\ngrid_search.fit(X, y)\nprint('Best score: {}'.format(grid_search.best_score_))\nprint('Best parameters: {}'.format(grid_search.best_params_))\nknn = neighbors.KNeighborsRegressor(**grid_search.best_params_)\nresult_dict_knn = artgor_utils.train_model_regression(X, X_test, y, params=params, folds=folds, model_type='sklearn', model=knn)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nlasso = linear_model.Lasso(normalize=True)\n\nparameter_grid = {'alpha': [0.1, 1.0, 10.0]}\n\ngrid_search = GridSearchCV(lasso, param_grid=parameter_grid, cv=folds, scoring='neg_mean_absolute_error', n_jobs=-1)\ngrid_search.fit(X, y)\nprint('Best score: {}'.format(grid_search.best_score_))\nprint('Best parameters: {}'.format(grid_search.best_params_))\nlasso = linear_model.Lasso(**grid_search.best_params_, normalize=True)\nresult_dict_lasso = artgor_utils.train_model_regression(X, X_test, y, params=params, folds=folds, model_type='sklearn', model=lasso)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\netr = ExtraTreesRegressor()\n\n# parameter_grid = {'n_estimators': [500, 1000],\n#                   'max_depth': [5, 10, 20]\n#                  }\n\n# grid_search = GridSearchCV(rfr, param_grid=parameter_grid, cv=folds, scoring='neg_mean_absolute_error', n_jobs=-1)\n# grid_search.fit(X, y)\n# print('Best score: {}'.format(grid_search.best_score_))\n# print('Best parameters: {}'.format(grid_search.best_params_))\n# etr = ExtraTreesRegressor(**grid_search.best_params_)\netr = ExtraTreesRegressor(n_estimators=1000, max_depth=10)\nresult_dict_etr = artgor_utils.train_model_regression(X, X_test, y, params=params, folds=folds, model_type='sklearn', model=etr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nadr = AdaBoostRegressor()\n\nparameter_grid = {'n_estimators': [10, 50],\n                 }\n\ngrid_search = GridSearchCV(adr, param_grid=parameter_grid, cv=folds, scoring='neg_mean_absolute_error', n_jobs=-1)\ngrid_search.fit(X, y)\nprint('Best score: {}'.format(grid_search.best_score_))\nprint('Best parameters: {}'.format(grid_search.best_params_))\nadr = AdaBoostRegressor(**grid_search.best_params_)\nresult_dict_adr = artgor_utils.train_model_regression(X, X_test, y, params=params, folds=folds, model_type='sklearn', model=adr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12, 8));\nscores_df = pd.DataFrame({'RandomForestRegressor': result_dict_rfr['scores']})\nscores_df['ExtraTreesRegressor'] = result_dict_etr['scores']\nscores_df['AdaBoostRegressor'] = result_dict_adr['scores']\nscores_df['KNN'] = result_dict_knn['scores']\nscores_df['LinearRegression'] = result_dict_linreg['scores']\nscores_df['Ridge'] = result_dict_ridge['scores']\nscores_df['Lasso'] = result_dict_lasso['scores']\n\nsns.boxplot(data=scores_df);\nplt.xticks(rotation=45);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Blending\n\nLet's try training and blending several models."},{"metadata":{"trusted":true},"cell_type":"code","source":"params = {'num_leaves': 128,\n          'min_child_samples': 79,\n          'objective': 'gamma',\n          'max_depth': -1,\n          'learning_rate': 0.01,\n          \"boosting_type\": \"gbdt\",\n          \"subsample_freq\": 5,\n          \"subsample\": 0.9,\n          \"bagging_seed\": 11,\n          \"metric\": 'mae',\n          \"verbosity\": -1,\n          'reg_alpha': 0.1302650970728192,\n          'reg_lambda': 0.3603427518866501,\n          'colsample_bytree': 0.2\n         }\nresult_dict_lgb = artgor_utils.train_model_regression(X=X, X_test=X_test, y=y, params=params, folds=folds, model_type='lgb',\n                                                                                  eval_metric='mae', plot_feature_importance=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_params = {'eta': 0.03,\n              'max_depth': 9,\n              'subsample': 0.85,\n              'colsample_bytree': 0.3,\n              'objective': 'reg:linear',\n              'eval_metric': 'mae',\n              'silent': True,\n              'nthread': -1}\nresult_dict_xgb = artgor_utils.train_model_regression(X=X, X_test=X_test, y=y, params=xgb_params, folds=folds, model_type='xgb')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['time_to_failure'] = (result_dict_lgb['prediction'] + result_dict_etr['prediction'] + result_dict_xgb['prediction']) / 3\nprint(submission.head())\nsubmission.to_csv('blending.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(18, 8))\nplt.subplot(2, 2, 1)\nplt.plot(y, color='g', label='y_train')\nplt.plot(result_dict_lgb['oof'], color='b', label='lgb')\nplt.legend(loc=(1, 0.5));\nplt.title('lgb');\nplt.subplot(2, 2, 2)\nplt.plot(y, color='g', label='y_train')\nplt.plot(result_dict_etr['oof'], color='teal', label='xgb')\nplt.legend(loc=(1, 0.5));\nplt.title('xgb');\nplt.subplot(2, 2, 3)\nplt.plot(y, color='g', label='y_train')\nplt.plot(result_dict_xgb['oof'], color='red', label='etr')\nplt.legend(loc=(1, 0.5));\nplt.title('Extratrees');\nplt.subplot(2, 2, 4)\nplt.plot(y, color='g', label='y_train')\nplt.plot((result_dict_lgb['oof'] + result_dict_etr['oof'] + result_dict_xgb['oof']) / 3, color='gold', label='blend')\nplt.legend(loc=(1, 0.5));\nplt.title('blend');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## Stacking"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_stack = np.vstack([result_dict_rfr['oof'], result_dict_ridge['oof'], result_dict_knn['oof'], result_dict_lasso['oof'], result_dict_etr['oof'],\n                         result_dict_adr['oof'], result_dict_lgb['oof'], result_dict_xgb['oof'], result_dict_etr['oof']]).transpose()\ntrain_stack = pd.DataFrame(train_stack, columns=['rfr', 'ridge', 'knn', 'lasso', 'etr', 'adr', 'lgb', 'xgb', 'etr'])\n\ntest_stack = np.vstack([result_dict_rfr['prediction'], result_dict_ridge['prediction'], result_dict_knn['prediction'], result_dict_lasso['prediction'], result_dict_etr['prediction'],\n                        result_dict_adr['prediction'], result_dict_lgb['prediction'], result_dict_xgb['prediction'], result_dict_etr['prediction']]).transpose()\ntest_stack = pd.DataFrame(test_stack, columns=['rfr', 'ridge', 'knn', 'lasso', 'etr', 'adr', 'lgb', 'xgb', 'etr'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params = {'num_leaves': 8,\n         #'min_data_in_leaf': 20,\n         'objective': 'regression',\n         'max_depth': 2,\n         'learning_rate': 0.01,\n         \"boosting\": \"gbdt\",\n         \"bagging_seed\": 11,\n         \"metric\": 'rmse',\n        # \"lambda_l1\": 0.2,\n         \"verbosity\": -1}\nresult_dict_lgb_stack = artgor_utils.train_model_regression(X=train_stack, X_test=test_stack, y=y, params=params, folds=folds, model_type='lgb',\n                                                                                  eval_metric='mae', plot_feature_importance=False,\n                                                            columns=(list(train_stack.columns)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['time_to_failure'] = result_dict_lgb_stack['prediction']\nprint(submission.head())\nsubmission.to_csv('stacking.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}