{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# Importing core libraries\nimport os\nimport gc\nimport numpy as np\nimport pandas as pd\nfrom time import time\nimport pprint\nimport joblib\nimport logging\n\n\n# Suppressing warnings because of skopt verbosity\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# Boosting models\nimport catboost as cat\nimport xgboost as xgb\nimport lightgbm as lgb\n\n# Hyperparameters distributions\nfrom scipy.stats import randint\nfrom scipy.stats import uniform\n\n# Preprocesing\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import LabelEncoder\n\n# Utilities\nfrom sklearn.pipeline import Pipeline\nfrom tqdm import tqdm_notebook\nfrom itertools import chain\nfrom sklearn.model_selection import train_test_split\nfrom typing import TypeVar, List, Dict, Tuple\nPandasDataFrame = TypeVar('pandas.core.frame.DataFrame')\n\n# Visualizations\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n\n# Model selection\nfrom sklearn.model_selection import StratifiedKFold, KFold\nfrom sklearn.model_selection import GridSearchCV, RandomizedSearchCV\nfrom sklearn.model_selection import cross_val_score\nfrom bayes_opt import BayesianOptimization\n\n\n# Metrics\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import make_scorer\nfrom sklearn.metrics import mean_absolute_error\n\n# Skopt functions\nfrom skopt import BayesSearchCV\nfrom skopt import gp_minimize # Bayesian optimization using Gaussian Processes\nfrom skopt.space import Real, Categorical, Integer\nfrom skopt.utils import use_named_args # decorator to convert a list of parameters to named arguments\nfrom skopt.callbacks import DeadlineStopper # Stop the optimization before running out of a fixed budget of time.\nfrom skopt.callbacks import VerboseCallback # Callback to control the verbosity\nfrom skopt.callbacks import DeltaXStopper # Stop the optimization If the last two positions at which the objective has been evaluated are less than delt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IS_LOCAL = False\nif(IS_LOCAL):\n    PATH=\"../input/LANL/\"\nelse:\n    PATH=\"../input/\"\nos.listdir(PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X = pd.read_csv('../input/mlip-data/train_features.csv')\ntest_X = pd.read_csv('../input/mlip-data/test_features.csv')\ntrain_y = pd.read_csv('../input/mlip-data/y.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X.shape, test_X.shape, train_y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/LANL-Earthquake-Prediction/sample_submission.csv', index_col='seg_id')\nsubmission.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Converting average precision score into a scorer suitable for model selection\nmse_scoring = make_scorer(mean_squared_error, greater_is_better=False)\nmae_scoring = make_scorer(mean_absolute_error, greater_is_better=False)\n# Setting a 5-fold stratified cross-validation (note: shuffle=True)\n# skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=0)S\nk_fold = KFold(n_splits=7, shuffle=True, random_state=13)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef run_lgb(\n    X_tr: PandasDataFrame,\n    X_val: PandasDataFrame,\n    y_tr: PandasDataFrame,\n    y_val: PandasDataFrame,\n    test_data: PandasDataFrame,\n    params: Dict\n):\n    \"\"\"CV train lgb Booster.\n    \n    Args:\n        params: Params for Booster.\n        X_train: Training dataset.\n        X_test: Testing dataset.\n        \n    Returns:\n        model: Trained model.\n        oof_train_lgb:  Training CV predictions.\n        oof_test_lgb:  Testing CV predictions.\n    \"\"\"\n    \n    early_stop = 200\n    num_rounds = 10000\n    verbose_eval=1000\n\n    \n    d_train = lgb.Dataset(X_tr, label=y_tr)\n    d_valid = lgb.Dataset(X_val, label=y_val)\n    watchlist = [d_train, d_valid]\n        \n    model = lgb.train(\n        params,\n        train_set=d_train,\n        num_boost_round=num_rounds,\n        valid_sets=watchlist,\n        verbose_eval=verbose_eval,\n        early_stopping_rounds=early_stop\n    )\n\n    val_pred = model.predict(X_val, num_iteration=model.best_iteration)    \n    prediction = model.predict(test_data, num_iteration=model.best_iteration)\n   \n    return val_pred, prediction, model.feature_importance(importance_type=\"split\")\ndef run_cv_model(\n    train_data,\n    train_target,\n    test_data,\n    model_fn, params,\n    scoring,\n    label='model',\n    feature_imp=False,\n    n_folds = 5\n):\n    oof_val = np.zeros(len(train_data))\n    predictions = np.zeros(len(test_data))\n    oof_predict = np.zeros((n_folds, test_data.shape[0]))\n    scores = []\n\n    feature_importance_df = pd.DataFrame()\n    \n    folds = KFold(n_splits=n_folds, shuffle=True, random_state=42)\n    train_columns = train_data.columns.values\n    \n\n    for fold_, (trn_idx, val_idx) in enumerate(folds.split(train_data, train_target.values)):\n        strLog = \"fold {} for {}\".format(fold_, label)\n        print(strLog)\n        X_tr, X_val = train_data.iloc[trn_idx], train_data.iloc[val_idx]\n        y_tr, y_val = train_target.iloc[trn_idx], train_target.iloc[val_idx]\n        \n        val_pred, prediction, feature_importances = model_fn(\n            X_tr, X_val,\n            y_tr, y_val,\n            test_data,\n            params\n        )\n        score = scoring(y_val, val_pred)\n        scores.append(score)\n\n        \n        oof_val[val_idx] = val_pred\n        \n        #feature importance\n        if feature_imp == True:\n            fold_importance_df = pd.DataFrame()\n            fold_importance_df[\"feature\"] = train_columns\n            fold_importance_df[\"importance\"] = feature_importances[:len(train_columns)]\n            fold_importance_df[\"fold\"] = fold_ + 1\n            feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n        \n        #predictions\n        oof_predict[fold_] = prediction\n        predictions += prediction/folds.n_splits\n        \n        print('CV score: {0:.4f}, std: {1:.4f}.\\n'.format(np.mean(score), np.std(score)))  \n        print('CV mean score: {0:.4f}, std: {1:.4f}.\\n'.format(np.mean(scores), np.std(scores)))  \n    \n\n\n            \n    return oof_val, oof_predict, predictions, feature_importance_df, scores","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def LGB_bayesian(**lgb_bayes_params):\n    \n    \n\n    \n    lgb_bayes_params['num_leaves'] = int(lgb_bayes_params['num_leaves'] )\n    lgb_bayes_params['max_depth'] = int(lgb_bayes_params['max_depth'])\n    lgb_bayes_params['min_child_samples'] = int(lgb_bayes_params['min_child_samples'])\n    lgb_bayes_params['max_bin'] = int(lgb_bayes_params['max_bin'])\n    lgb_bayes_params['subsample_freq'] = int(lgb_bayes_params['subsample_freq'])\n    lgb_bayes_params['min_child_weight'] = int(lgb_bayes_params['min_child_weight'])\n    lgb_bayes_params['subsample_for_bin'] = int(lgb_bayes_params['subsample_for_bin'])\n    lgb_bayes_params['n_estimators'] = int(lgb_bayes_params['n_estimators'])\n\n    \n    \n    \n    fix_lgb_params = {\n        'objective':'regression',\n        \"boosting\": \"gbdt\",\n        \"metric\": 'mae',\n    }\n\n    lgb_params = {**fix_lgb_params, **lgb_bayes_params}\n\n\n\n\n    oof_val, oof_predict, predictions, feature_importance_df, scores = run_cv_model(\n    train_data=train_X,\n    train_target=train_y,\n    test_data=test_X,\n    model_fn=run_lgb,\n    params=lgb_params,\n    scoring=mean_absolute_error,\n    label=\"lgb\",\n    feature_imp=True,\n    n_folds=8\n)\n    \n    return -np.mean(scores)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nsearch_spaces = {\n        'learning_rate': (0.01, .5),\n        'num_leaves': (2, 350),\n        'max_depth': (50, 500),\n        'min_child_samples': (20, 150), # minimal number of data in one leaf\n        'max_bin': (30000, 100000), # max number of bins that feature values will be bucketed\n        'subsample': (0.01, 1.0),\n        'subsample_freq': (2, 20), # bagging fraction\n        'colsample_bytree': (0.01, 1.0), # enabler of bagging fraction\n        'min_child_weight': (0, 10), # minimal number of data in one leaf.\n        'subsample_for_bin': (100000, 800000), # number of data that sampled for histogram bins\n        'reg_lambda': (1e-9, 1000), # L2 regularization\n        'reg_alpha': (1e-9, 1.0), # L1 regularization\n        'scale_pos_weight': (1e-6, 1000), \n        'n_estimators': (1000, 10000),\n        'min_gain_to_split': (0.0, 1.0),\n        }\n\nlgb_bo = BayesianOptimization(LGB_bayesian,search_spaces, random_state=7)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# with warnings.catch_warnings():\n#     warnings.filterwarnings('ignore')\n#     lgb_bo.maximize(init_points=50, n_iter=50, acq='ucb')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_bayes_params = lgb_bo.max['params']\n\n\n\nlgb_bayes_params['num_leaves'] = int(lgb_bayes_params['num_leaves'] )\nlgb_bayes_params['max_depth'] = int(lgb_bayes_params['max_depth'])\nlgb_bayes_params['min_child_samples'] = int(lgb_bayes_params['min_child_samples'])\nlgb_bayes_params['max_bin'] = int(lgb_bayes_params['max_bin'])\nlgb_bayes_params['subsample_freq'] = int(lgb_bayes_params['subsample_freq'])\nlgb_bayes_params['min_child_weight'] = int(lgb_bayes_params['min_child_weight'])\nlgb_bayes_params['subsample_for_bin'] = int(lgb_bayes_params['subsample_for_bin'])\nlgb_bayes_params['n_estimators'] = int(lgb_bayes_params['n_estimators'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_bayes_params = {'colsample_bytree': 0.07731235427729943,\n 'learning_rate': 0.021390697326123965,\n 'max_bin': 52576,\n 'max_depth': 339,\n 'min_child_samples': 119,\n 'min_child_weight': 6,\n 'min_gain_to_split': 0.4223894968003159,\n 'n_estimators': 1560,\n 'num_leaves': 25,\n 'reg_alpha': 0.7079338662764831,\n 'reg_lambda': 368.73133802650744,\n 'scale_pos_weight': 363.3220921999365,\n 'subsample': 0.9910157826344941,\n 'subsample_for_bin': 693174,\n 'subsample_freq': 19}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FeatureSelector():\n    \"\"\"\n    Class for performing feature selection for machine learning or data preprocessing.\n    \n    Implements five different methods to identify features for removal \n    \n        1. Find columns with a missing percentage greater than a specified threshold\n        2. Find columns with a single unique value\n        3. Find collinear variables with a correlation greater than a specified correlation coefficient\n        4. Find features with 0.0 feature importance from a gradient boosting machine (gbm)\n        5. Find low importance features that do not contribute to a specified cumulative feature importance from the gbm\n        \n    Parameters\n    --------\n        data : dataframe\n            A dataset with observations in the rows and features in the columns\n        labels : array or series, default = None\n            Array of labels for training the machine learning model to find feature importances. These can be either binary labels\n            (if task is 'classification') or continuous targets (if task is 'regression').\n            If no labels are provided, then the feature importance based methods are not available.\n        \n    Attributes\n    --------\n    \n    ops : dict\n        Dictionary of operations run and features identified for removal\n        \n    missing_stats : dataframe\n        The fraction of missing values for all features\n    \n    record_missing : dataframe\n        The fraction of missing values for features with missing fraction above threshold\n        \n    unique_stats : dataframe\n        Number of unique values for all features\n    \n    record_single_unique : dataframe\n        Records the features that have a single unique value\n        \n    corr_matrix : dataframe\n        All correlations between all features in the data\n    \n    record_collinear : dataframe\n        Records the pairs of collinear variables with a correlation coefficient above the threshold\n        \n    feature_importances : dataframe\n        All feature importances from the gradient boosting machine\n    \n    record_zero_importance : dataframe\n        Records the zero importance features in the data according to the gbm\n    \n    record_low_importance : dataframe\n        Records the lowest importance features not needed to reach the threshold of cumulative importance according to the gbm\n    \n    \n    Notes\n    --------\n    \n        - All 5 operations can be run with the `identify_all` method.\n        - If using feature importances, one-hot encoding is used for categorical variables which creates new columns\n    \n    \"\"\"\n    \n    def __init__(self, data, labels=None):\n        \n        # Dataset and optional training labels\n        self.data = data\n        self.labels = labels\n\n        if labels is None:\n            print('No labels provided. Feature importance based methods are not available.')\n        \n        self.base_features = list(data.columns)\n        self.one_hot_features = None\n        \n        # Dataframes recording information about features to remove\n        self.record_missing = None\n        self.record_single_unique = None\n        self.record_collinear = None\n        self.record_zero_importance = None\n        self.record_low_importance = None\n        \n        self.missing_stats = None\n        self.unique_stats = None\n        self.corr_matrix = None\n        self.feature_importances = None\n        \n        # Dictionary to hold removal operations\n        self.ops = {}\n        \n        self.one_hot_correlated = False\n        \n    def identify_missing(self, missing_threshold):\n        \"\"\"Find the features with a fraction of missing values above `missing_threshold`\"\"\"\n        \n        self.missing_threshold = missing_threshold\n\n        # Calculate the fraction of missing in each column \n        missing_series = self.data.isnull().sum() / self.data.shape[0]\n        self.missing_stats = pd.DataFrame(missing_series).rename(columns = {'index': 'feature', 0: 'missing_fraction'})\n\n        # Sort with highest number of missing values on top\n        self.missing_stats = self.missing_stats.sort_values('missing_fraction', ascending = False)\n\n        # Find the columns with a missing percentage above the threshold\n        record_missing = pd.DataFrame(missing_series[missing_series > missing_threshold]).reset_index().rename(columns = \n                                                                                                               {'index': 'feature', \n                                                                                                                0: 'missing_fraction'})\n\n        to_drop = list(record_missing['feature'])\n\n        self.record_missing = record_missing\n        self.ops['missing'] = to_drop\n        \n        print('%d features with greater than %0.2f missing values.\\n' % (len(self.ops['missing']), self.missing_threshold))\n        \n    def identify_single_unique(self):\n        \"\"\"Finds features with only a single unique value. NaNs do not count as a unique value. \"\"\"\n\n        # Calculate the unique counts in each column\n        unique_counts = self.data.nunique()\n        self.unique_stats = pd.DataFrame(unique_counts).rename(columns = {'index': 'feature', 0: 'nunique'})\n        self.unique_stats = self.unique_stats.sort_values('nunique', ascending = True)\n        \n        # Find the columns with only one unique count\n        record_single_unique = pd.DataFrame(unique_counts[unique_counts == 1]).reset_index().rename(columns = {'index': 'feature', \n                                                                                                                0: 'nunique'})\n\n        to_drop = list(record_single_unique['feature'])\n    \n        self.record_single_unique = record_single_unique\n        self.ops['single_unique'] = to_drop\n        \n        print('%d features with a single unique value.\\n' % len(self.ops['single_unique']))\n    \n    def identify_collinear(self, correlation_threshold, one_hot=False):\n        \"\"\"\n        Finds collinear features based on the correlation coefficient between features. \n        For each pair of features with a correlation coefficient greather than `correlation_threshold`,\n        only one of the pair is identified for removal. \n        Using code adapted from: https://chrisalbon.com/machine_learning/feature_selection/drop_highly_correlated_features/\n        \n        Parameters\n        --------\n        correlation_threshold : float between 0 and 1\n            Value of the Pearson correlation cofficient for identifying correlation features\n        one_hot : boolean, default = False\n            Whether to one-hot encode the features before calculating the correlation coefficients\n        \"\"\"\n        \n        self.correlation_threshold = correlation_threshold\n        self.one_hot_correlated = one_hot\n        \n         # Calculate the correlations between every column\n        if one_hot:\n            \n            # One hot encoding\n            features = pd.get_dummies(self.data)\n            self.one_hot_features = [column for column in features.columns if column not in self.base_features]\n\n            # Add one hot encoded data to original data\n            self.data_all = pd.concat([features[self.one_hot_features], self.data], axis = 1)\n            \n            corr_matrix = pd.get_dummies(features).corr()\n\n        else:\n            corr_matrix = self.data.corr()\n        \n        self.corr_matrix = corr_matrix\n    \n        # Extract the upper triangle of the correlation matrix\n        upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k = 1).astype(np.bool))\n        \n        # Select the features with correlations above the threshold\n        # Need to use the absolute value\n        to_drop = [column for column in upper.columns if any(upper[column].abs() > correlation_threshold)]\n\n        # Dataframe to hold correlated pairs\n        record_collinear = pd.DataFrame(columns = ['drop_feature', 'corr_feature', 'corr_value'])\n\n        # Iterate through the columns to drop to record pairs of correlated features\n        for column in to_drop:\n\n            # Find the correlated features\n            corr_features = list(upper.index[upper[column].abs() > correlation_threshold])\n\n            # Find the correlated values\n            corr_values = list(upper[column][upper[column].abs() > correlation_threshold])\n            drop_features = [column for _ in range(len(corr_features))]    \n\n            # Record the information (need a temp df for now)\n            temp_df = pd.DataFrame.from_dict({'drop_feature': drop_features,\n                                             'corr_feature': corr_features,\n                                             'corr_value': corr_values})\n\n            # Add to dataframe\n            record_collinear = record_collinear.append(temp_df, ignore_index = True)\n\n        self.record_collinear = record_collinear\n        self.ops['collinear'] = to_drop\n        \n        print('%d features with a correlation magnitude greater than %0.2f.\\n' % (len(self.ops['collinear']), self.correlation_threshold))\n\n    def identify_zero_importance(self,  feature_importances):\n        \"\"\"\n        \n        Identify the features with zero importance according to a gradient boosting machine.\n        The gbm can be trained with early stopping using a validation set to prevent overfitting. \n        The feature importances are averaged over `n_iterations` to reduce variance. \n        \n        Parameters \n        --------\n        eval_metric : string\n            Evaluation metric to use for the gradient boosting machine for early stopping. Must be\n            provided if `early_stopping` is True\n        task : string\n            The machine learning task, either 'classification' or 'regression'\n        n_iterations : int, default = 10\n            Number of iterations to train the gradient boosting machine\n            \n        early_stopping : boolean, default = True\n            Whether or not to use early stopping with a validation set when training\n        \n        \n        Notes\n        --------\n        \n        - Features are one-hot encoded to handle the categorical variables before training.\n        - The gbm is not optimized for any particular task and might need some hyperparameter tuning\n        - Feature importances, including zero importance features, can change across runs\n        \"\"\"\n\n\n            \n        if self.labels is None:\n            raise ValueError(\"No training labels provided.\")\n        \n        # One hot encoding\n        features = pd.get_dummies(self.data)\n        self.one_hot_features = [column for column in features.columns if column not in self.base_features]\n\n        # Add one hot encoded data to original data\n        self.data_all = pd.concat([features[self.one_hot_features], self.data], axis = 1)\n\n        # Extract feature names\n        feature_names = list(features.columns)\n\n        # Convert to np array\n        features = np.array(features)\n        labels = np.array(self.labels).reshape((-1, ))\n\n\n        \n        \n\n     \n\n        # Sort features according to importance\n        feature_importances = feature_importances.sort_values('importance', ascending = False).reset_index(drop = True)\n\n        # Normalize the feature importances to add up to one\n        feature_importances['normalized_importance'] = feature_importances['importance'] / feature_importances['importance'].sum()\n        feature_importances['cumulative_importance'] = np.cumsum(feature_importances['normalized_importance'])\n\n        # Extract the features with zero importance\n        record_zero_importance = feature_importances[feature_importances['importance'] == 0.0]\n        \n        to_drop = list(record_zero_importance['feature'])\n\n        self.feature_importances = feature_importances\n        self.record_zero_importance = record_zero_importance\n        self.ops['zero_importance'] = to_drop\n        \n        print('\\n%d features with zero importance after one-hot encoding.\\n' % len(self.ops['zero_importance']))\n    \n    def identify_low_importance(self, cumulative_importance):\n        \"\"\"\n        Finds the lowest importance features not needed to account for `cumulative_importance` fraction\n        of the total feature importance from the gradient boosting machine. As an example, if cumulative\n        importance is set to 0.95, this will retain only the most important features needed to \n        reach 95% of the total feature importance. The identified features are those not needed.\n        Parameters\n        --------\n        cumulative_importance : float between 0 and 1\n            The fraction of cumulative importance to account for \n        \"\"\"\n\n        self.cumulative_importance = cumulative_importance\n        \n        # The feature importances need to be calculated before running\n        if self.feature_importances is None:\n            raise NotImplementedError(\"\"\"Feature importances have not yet been determined. \n                                         Call the `identify_zero_importance` method first.\"\"\")\n            \n        # Make sure most important features are on top\n        self.feature_importances = self.feature_importances.sort_values('cumulative_importance')\n\n        # Identify the features not needed to reach the cumulative_importance\n        record_low_importance = self.feature_importances[self.feature_importances['cumulative_importance'] > cumulative_importance]\n\n        to_drop = list(record_low_importance['feature'])\n\n        self.record_low_importance = record_low_importance\n        self.ops['low_importance'] = to_drop\n    \n        print('%d features required for cumulative importance of %0.2f after one hot encoding.' % (len(self.feature_importances) -\n                                                                            len(self.record_low_importance), self.cumulative_importance))\n        print('%d features do not contribute to cumulative importance of %0.2f.\\n' % (len(self.ops['low_importance']),\n                                                                                               self.cumulative_importance))\n        \n    def identify_all(self, selection_params):\n        \"\"\"\n        Use all five of the methods to identify features to remove.\n        \n        Parameters\n        --------\n            \n        selection_params : dict\n           Parameters to use in the five feature selection methhods.\n           Params must contain the keys ['missing_threshold', 'correlation_threshold', 'eval_metric', 'task', 'cumulative_importance']\n        \n        \"\"\"\n        \n        # Check for all required parameters\n        for param in ['missing_threshold', 'correlation_threshold', 'eval_metric', 'task', 'cumulative_importance']:\n            if param not in selection_params.keys():\n                raise ValueError('%s is a required parameter for this method.' % param)\n        \n        # Implement each of the five methods\n        self.identify_missing(selection_params['missing_threshold'])\n        self.identify_single_unique()\n        self.identify_collinear(selection_params['correlation_threshold'])\n        self.identify_zero_importance(task = selection_params['task'], eval_metric = selection_params['eval_metric'])\n        self.identify_low_importance(selection_params['cumulative_importance'])\n        \n        # Find the number of features identified to drop\n        self.all_identified = set(list(chain(*list(self.ops.values()))))\n        self.n_identified = len(self.all_identified)\n        \n        print('%d total features out of %d identified for removal after one-hot encoding.\\n' % (self.n_identified, \n                                                                                                  self.data_all.shape[1]))\n        \n    def check_removal(self, keep_one_hot=True):\n        \n        \"\"\"Check the identified features before removal. Returns a list of the unique features identified.\"\"\"\n        \n        self.all_identified = set(list(chain(*list(self.ops.values()))))\n        print('Total of %d features identified for removal' % len(self.all_identified))\n        \n        if not keep_one_hot:\n            if self.one_hot_features is None:\n                print('Data has not been one-hot encoded')\n            else:\n                one_hot_to_remove = [x for x in self.one_hot_features if x not in self.all_identified]\n                print('%d additional one-hot features can be removed' % len(one_hot_to_remove))\n        \n        return list(self.all_identified)\n        \n    \n    def remove(self, methods, keep_one_hot = True):\n        \"\"\"\n        Remove the features from the data according to the specified methods.\n        \n        Parameters\n        --------\n            methods : 'all' or list of methods\n                If methods == 'all', any methods that have identified features will be used\n                Otherwise, only the specified methods will be used.\n                Can be one of ['missing', 'single_unique', 'collinear', 'zero_importance', 'low_importance']\n            keep_one_hot : boolean, default = True\n                Whether or not to keep one-hot encoded features\n                \n        Return\n        --------\n            data : dataframe\n                Dataframe with identified features removed\n                \n        \n        Notes \n        --------\n            - If feature importances are used, the one-hot encoded columns will be added to the data (and then may be removed)\n            - Check the features that will be removed before transforming data!\n        \n        \"\"\"\n        \n        \n        features_to_drop = []\n      \n        if methods == 'all':\n            \n            # Need to use one-hot encoded data as well\n            data = self.data_all\n                                          \n            print('{} methods have been run\\n'.format(list(self.ops.keys())))\n            \n            # Find the unique features to drop\n            features_to_drop = set(list(chain(*list(self.ops.values()))))\n            \n        else:\n            # Need to use one-hot encoded data as well\n            if 'zero_importance' in methods or 'low_importance' in methods or self.one_hot_correlated:\n                data = self.data_all\n                \n            else:\n                data = self.data\n                \n            # Iterate through the specified methods\n            for method in methods:\n                \n                # Check to make sure the method has been run\n                if method not in self.ops.keys():\n                    raise NotImplementedError('%s method has not been run' % method)\n                    \n                # Append the features identified for removal\n                else:\n                    features_to_drop.append(self.ops[method])\n        \n            # Find the unique features to drop\n            features_to_drop = set(list(chain(*features_to_drop)))\n            \n        features_to_drop = list(features_to_drop)\n            \n        if not keep_one_hot:\n            \n            if self.one_hot_features is None:\n                print('Data has not been one-hot encoded')\n            else:\n                             \n                features_to_drop = list(set(features_to_drop) | set(self.one_hot_features))\n       \n        # Remove the features and return the data\n        data = data.drop(columns = features_to_drop)\n        self.removed_features = features_to_drop\n        \n        if not keep_one_hot:\n        \tprint('Removed %d features including one-hot features.' % len(features_to_drop))\n        else:\n        \tprint('Removed %d features.' % len(features_to_drop))\n        \n        return data\n    \n    def plot_missing(self):\n        \"\"\"Histogram of missing fraction in each feature\"\"\"\n        if self.record_missing is None:\n            raise NotImplementedError(\"Missing values have not been calculated. Run `identify_missing`\")\n        \n        self.reset_plot()\n        \n        # Histogram of missing values\n        plt.style.use('seaborn-white')\n        plt.figure(figsize = (7, 5))\n        plt.hist(self.missing_stats['missing_fraction'], bins = np.linspace(0, 1, 11), edgecolor = 'k', color = 'red', linewidth = 1.5)\n        plt.xticks(np.linspace(0, 1, 11));\n        plt.xlabel('Missing Fraction', size = 14); plt.ylabel('Count of Features', size = 14); \n        plt.title(\"Fraction of Missing Values Histogram\", size = 16);\n        \n    \n    def plot_unique(self):\n        \"\"\"Histogram of number of unique values in each feature\"\"\"\n        if self.record_single_unique is None:\n            raise NotImplementedError('Unique values have not been calculated. Run `identify_single_unique`')\n        \n        self.reset_plot()\n\n        # Histogram of number of unique values\n        self.unique_stats.plot.hist(edgecolor = 'k', figsize = (7, 5))\n        plt.ylabel('Frequency', size = 14); plt.xlabel('Unique Values', size = 14); \n        plt.title('Number of Unique Values Histogram', size = 16);\n        \n    \n    def plot_collinear(self, plot_all = False):\n        \"\"\"\n        Heatmap of the correlation values. If plot_all = True plots all the correlations otherwise\n        plots only those features that have a correlation above the threshold\n        \n        Notes\n        --------\n            - Not all of the plotted correlations are above the threshold because this plots\n            all the variables that have been idenfitied as having even one correlation above the threshold\n            - The features on the x-axis are those that will be removed. The features on the y-axis\n            are the correlated features with those on the x-axis\n        \n        Code adapted from https://seaborn.pydata.org/examples/many_pairwise_correlations.html\n        \"\"\"\n        \n        if self.record_collinear is None:\n            raise NotImplementedError('Collinear features have not been idenfitied. Run `identify_collinear`.')\n        \n        if plot_all:\n        \tcorr_matrix_plot = self.corr_matrix\n        \ttitle = 'All Correlations'\n        \n        else:\n\t        # Identify the correlations that were above the threshold\n\t        # columns (x-axis) are features to drop and rows (y_axis) are correlated pairs\n\t        corr_matrix_plot = self.corr_matrix.loc[list(set(self.record_collinear['corr_feature'])), \n\t                                                list(set(self.record_collinear['drop_feature']))]\n\n\t        title = \"Correlations Above Threshold\"\n\n       \n        f, ax = plt.subplots(figsize=(10, 8))\n        \n        # Diverging colormap\n        cmap = sns.diverging_palette(220, 10, as_cmap=True)\n\n        # Draw the heatmap with a color bar\n        sns.heatmap(corr_matrix_plot, cmap=cmap, center=0,\n                    linewidths=.25, cbar_kws={\"shrink\": 0.6})\n\n        # Set the ylabels \n        ax.set_yticks([x + 0.5 for x in list(range(corr_matrix_plot.shape[0]))])\n        ax.set_yticklabels(list(corr_matrix_plot.index), size = int(160 / corr_matrix_plot.shape[0]));\n\n        # Set the xlabels \n        ax.set_xticks([x + 0.5 for x in list(range(corr_matrix_plot.shape[1]))])\n        ax.set_xticklabels(list(corr_matrix_plot.columns), size = int(160 / corr_matrix_plot.shape[1]));\n        plt.title(title, size = 14)\n        \n    def plot_feature_importances(self, plot_n = 15, threshold = None):\n        \"\"\"\n        Plots `plot_n` most important features and the cumulative importance of features.\n        If `threshold` is provided, prints the number of features needed to reach `threshold` cumulative importance.\n        Parameters\n        --------\n        \n        plot_n : int, default = 15\n            Number of most important features to plot. Defaults to 15 or the maximum number of features whichever is smaller\n        \n        threshold : float, between 0 and 1 default = None\n            Threshold for printing information about cumulative importances\n        \"\"\"\n        \n        if self.record_zero_importance is None:\n            raise NotImplementedError('Feature importances have not been determined. Run `idenfity_zero_importance`')\n            \n        # Need to adjust number of features if greater than the features in the data\n        if plot_n > self.feature_importances.shape[0]:\n            plot_n = self.feature_importances.shape[0] - 1\n\n        self.reset_plot()\n        \n        # Make a horizontal bar chart of feature importances\n        plt.figure(figsize = (10, 6))\n        ax = plt.subplot()\n\n        # Need to reverse the index to plot most important on top\n        # There might be a more efficient method to accomplish this\n        ax.barh(list(reversed(list(self.feature_importances.index[:plot_n]))), \n                self.feature_importances['normalized_importance'][:plot_n], \n                align = 'center', edgecolor = 'k')\n\n        # Set the yticks and labels\n        ax.set_yticks(list(reversed(list(self.feature_importances.index[:plot_n]))))\n        ax.set_yticklabels(self.feature_importances['feature'][:plot_n], size = 12)\n\n        # Plot labeling\n        plt.xlabel('Normalized Importance', size = 16); plt.title('Feature Importances', size = 18)\n        plt.show()\n\n        # Cumulative importance plot\n        plt.figure(figsize = (6, 4))\n        plt.plot(list(range(1, len(self.feature_importances) + 1)), self.feature_importances['cumulative_importance'], 'r-')\n        plt.xlabel('Number of Features', size = 14); plt.ylabel('Cumulative Importance', size = 14); \n        plt.title('Cumulative Feature Importance', size = 16);\n\n        if threshold:\n\n            # Index of minimum number of features needed for cumulative importance threshold\n            # np.where returns the index so need to add 1 to have correct number\n            importance_index = np.min(np.where(self.feature_importances['cumulative_importance'] > threshold))\n            plt.vlines(x = importance_index + 1, ymin = 0, ymax = 1, linestyles='--', colors = 'blue')\n            plt.show();\n\n            print('%d features required for %0.2f of cumulative importance' % (importance_index + 1, threshold))\n\n    def reset_plot(self):\n        plt.rcParams = plt.rcParamsDefault","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fix_lgb_params = {\n    'objective':'regression',\n    \"boosting\": \"gbdt\",\n    \"metric\": 'mae',\n}\n\nlgb_params = {**fix_lgb_params, **lgb_bayes_params}\n\n\noof_val, oof_predict, predictions, feature_importance_df, scores = run_cv_model(\n    train_data=train_X,\n    train_target=train_y,\n    test_data=test_X,\n    model_fn=run_lgb,\n    params=lgb_params,\n    scoring=mean_absolute_error,\n    label=\"lgb\",\n    feature_imp=True,\n    n_folds=8\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv',index=True)\nfs = FeatureSelector(data = train_X, labels = train_y)\n\nfeature_importance = feature_importance_df[[\"feature\", \"importance\"]].groupby(\"feature\").mean()\nfeature_importance['feature'] = feature_importance.index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfs.identify_zero_importance(feature_importance)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"one_hot_features = fs.one_hot_features\nbase_features = fs.base_features\nprint('There are %d original features' % len(base_features))\nprint('There are %d one-hot features' % len(one_hot_features))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"zero_importance_features = fs.ops['zero_importance']\nzero_importance_features[0:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfs.identify_low_importance(cumulative_importance = 0.99)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_to_remove = fs.check_removal()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_removed_all = fs.remove(methods = 'all', keep_one_hot=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_removed_all.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature = train_removed_all.columns\ntest_df = test_X[feature]\ntrain_df = train_removed_all","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def LGB_bayesian_sec(**lgb_bayes_params):\n    \n    \n\n    \n    lgb_bayes_params['num_leaves'] = int(lgb_bayes_params['num_leaves'] )\n    lgb_bayes_params['max_depth'] = int(lgb_bayes_params['max_depth'])\n    lgb_bayes_params['min_child_samples'] = int(lgb_bayes_params['min_child_samples'])\n    lgb_bayes_params['max_bin'] = int(lgb_bayes_params['max_bin'])\n    lgb_bayes_params['subsample_freq'] = int(lgb_bayes_params['subsample_freq'])\n    lgb_bayes_params['min_child_weight'] = int(lgb_bayes_params['min_child_weight'])\n    lgb_bayes_params['subsample_for_bin'] = int(lgb_bayes_params['subsample_for_bin'])\n    lgb_bayes_params['n_estimators'] = int(lgb_bayes_params['n_estimators'])\n\n    \n    \n    \n    fix_lgb_params = {\n        'objective':'regression',\n        \"boosting\": \"gbdt\",\n        \"metric\": 'mae',\n    }\n\n    lgb_params = {**fix_lgb_params, **lgb_bayes_params}\n\n\n\n\n    oof_val, oof_predict, predictions, feature_importance_df, scores = run_cv_model(\n    train_data=train_df,\n    train_target=train_y,\n    test_data=test_df,\n    model_fn=run_lgb,\n    params=lgb_params,\n    scoring=mean_absolute_error,\n    label=\"lgb\",\n    feature_imp=True,\n    n_folds=8\n)\n    \n    return -np.mean(scores)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_bo = BayesianOptimization(LGB_bayesian_sec,search_spaces, random_state=7)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with warnings.catch_warnings():\n    warnings.filterwarnings('ignore')\n    lgb_bo.maximize(init_points=50, n_iter=50, acq='ucb')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_bayes_params = lgb_bo.max['params']\n\n\n\nlgb_bayes_params['num_leaves'] = int(lgb_bayes_params['num_leaves'] )\nlgb_bayes_params['max_depth'] = int(lgb_bayes_params['max_depth'])\nlgb_bayes_params['min_child_samples'] = int(lgb_bayes_params['min_child_samples'])\nlgb_bayes_params['max_bin'] = int(lgb_bayes_params['max_bin'])\nlgb_bayes_params['subsample_freq'] = int(lgb_bayes_params['subsample_freq'])\nlgb_bayes_params['min_child_weight'] = int(lgb_bayes_params['min_child_weight'])\nlgb_bayes_params['subsample_for_bin'] = int(lgb_bayes_params['subsample_for_bin'])\nlgb_bayes_params['n_estimators'] = int(lgb_bayes_params['n_estimators'])\n\nlgb_params = {**fix_lgb_params, **lgb_bayes_params}\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof_val, oof_predict, predictions, feature_importance_df, scores = run_cv_model(\n    train_data=train_df,\n    train_target=train_y,\n    test_data=test_df,\n    model_fn=run_lgb,\n    params=lgb_params,\n    scoring=mean_absolute_error,\n    label=\"lgb\",\n    feature_imp=True,\n    n_folds=8\n\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols = (feature_importance_df[[\"feature\", \"importance\"]]\n        .groupby(\"feature\")\n        .mean()\n        .sort_values(by=\"importance\", ascending=False)[:50].index)\nbest_features = feature_importance_df.loc[feature_importance_df.feature.isin(cols)]\n\nplt.figure(figsize=(14,26))\nsns.barplot(x=\"importance\", y=\"feature\", data=best_features.sort_values(by=\"importance\",ascending=False))\nplt.title('LightGBM Features (averaged over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importances.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.time_to_failure = predictions\nsubmission.to_csv('clean_submission.csv',index=True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}