{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport seaborn as sns\nimport os\n\nfrom sklearn.model_selection import train_test_split\nfrom lightgbm import LGBMRegressor\nfrom sklearn.metrics import mean_squared_error as mse\nfrom sklearn.metrics import mean_absolute_error as mae\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# ACKNOWLEDGEMENTS: IMPORTANT\n    \nI have used the great features created here:\nhttps://www.kaggle.com/amanooo/ingv-volcanic-basic-solution-stft/\n\nI have used the ideas / code on tuning the model from here:\nhttps://www.kaggle.com/isaienkov/top-3-efficient-ensembling-in-few-lines-of-code\n\nI am just aiming to optimise the great features with some folds and testing model parameters."},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv('/kaggle/input/predict-volcanic-eruptions-ingv-oe/sample_submission.csv')\nsample_submission.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_set = pd.read_csv('/kaggle/input/volcano-stft-data/output_train_set.csv', index_col=0).reset_index(drop=True)\ntrain_set.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_set = pd.read_csv('/kaggle/input/volcano-stft-data/output_test_set.csv', index_col=0).reset_index(drop=True)\ntest_set.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#check that sample submission is lined up\nsum(sample_submission['segment_id']==test_set['segment_id']) / len(test_set)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#create some stratification for cross validation\ntrain_set['time_to_eruption_pc'] = train_set['time_to_eruption'] / train_set['time_to_eruption'].max()\n\nfrom sklearn.model_selection import StratifiedKFold\nNFOLDS=10\nskf = StratifiedKFold(n_splits=NFOLDS)\n\ntrain_set['label_strat'] = np.round(train_set['time_to_eruption_pc'] * 20, 0)\n\ntrain_set['fold']=0\nf=0\nfor trn_idx, val_idx in skf.split(train_set[['segment_id']], train_set['label_strat']):\n    train_set.loc[val_idx, 'fold']=f\n    f+=1\ntrain_set['fold'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DROP_FTS = ['segment_id', 'time_to_eruption','h:m:s', 'label_strat', 'fold', 'time_to_eruption_pc',]\nSEL_FTS = [x for x in train_set.columns if x not in DROP_FTS]\nLABEL = 'time_to_eruption'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_set[LABEL]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(SEL_FTS)\n\nnc=5\nnr=len(SEL_FTS)//nc+1\n\nfig,axes=plt.subplots(nrows=nr, ncols=nc, figsize=(25,nr*4))\n\nfor count,sf in enumerate(SEL_FTS):\n    sns.distplot(train_set[sf], ax=axes[count//nc, count%nc], color='Green')\n    sns.distplot(test_set[sf], ax=axes[count//nc, count%nc], color='Red')\n    \n    axes[count//nc, count%nc].set_title(sf)\n    sns.despine(ax=axes[count//nc, count%nc])\n    \nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import optuna\nfrom optuna.samplers import TPESampler\noptuna.logging.set_verbosity(optuna.logging.WARNING)\nimport xgboost as xgb\n\nOPTUNA_TRIALS = 30\n\nclass Optimizer:\n    def __init__(self, metric, trials=OPTUNA_TRIALS):\n        self.metric = metric\n        self.trials = trials\n        self.sampler = TPESampler(seed=42)\n        \n    def objective(self, trial):\n        model = create_model(trial)\n        model.fit(X_train, y_train)\n        preds = model.predict(X_val)\n        if self.metric == 'mae':\n            return -mae(y_val, preds)\n        else:\n            return -np.sqrt(mse(y_val, preds))\n        \n            \n    def optimize(self):\n        study = optuna.create_study(direction=\"maximize\", sampler=self.sampler)\n        study.optimize(self.objective, n_trials=self.trials)\n        return study.best_params\n\ndef create_model(trial):\n    #max_depth = trial.suggest_int(\"max_depth\", 2, 6)\n    #n_estimators = trial.suggest_int(\"n_estimators\", 2, 150)\n    colsample_bytree = trial.suggest_uniform(\"colsample_bytree\", 0.2, 0.95)\n    subsample = trial.suggest_uniform(\"subsample\", 0.2, 0.9)\n    reg_alpha = trial.suggest_uniform(\"reg_alpha\", 0.001, 100)\n    max_depth = trial.suggest_int(\"max_depth\", 1, 15)\n    min_child_samples = trial.suggest_int('min_child_samples', 1, 15)\n    n_estimators = trial.suggest_int('n_estimators', 100, 250)\n    \n    model = LGBMRegressor(random_state = 42,\n                    max_depth = max_depth,\n                    n_estimators = n_estimators, \n                          colsample_bytree=colsample_bytree,\n                          subsample=subsample,\n                          reg_alpha=reg_alpha,\n                    learning_rate = 0.05)\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"param_outputs = [] \n\nFOLD_VALUES = [x for x in train_set['fold'].unique()] \n\nfor fold in train_set['fold'].unique():\n    \n    trn_idx = train_set['fold']!=fold\n    val_idx = train_set['fold']==fold\n    \n    X_train = train_set.loc[trn_idx,SEL_FTS].values\n    y_train = train_set.loc[trn_idx, LABEL].values\n\n    X_val = train_set.loc[val_idx,SEL_FTS].values\n    y_val = train_set.loc[val_idx, LABEL].values\n\n    optimizer = Optimizer('mse')\n\n    lgb_params = optimizer.optimize()\n    param_outputs += [lgb_params]\n\n    lgb_params['random_state'] = 42\n    model_output = LGBMRegressor(**lgb_params)\n\n    print(fold, lgb_params)\n\n    model_output.fit(X_train, y_train)\n\n    preds = model_output.predict(X_val)\n\n    print(mae(y_val, preds))\n    \n    \nparam_headings = ['colsample_bytree','subsample', 'reg_alpha', \n                  'max_depth', 'min_child_samples', 'n_estimators', 'random_state']\n\nparams_output_df = pd.DataFrame(columns=param_headings, index=FOLD_VALUES, data=0.0)\n\nfor idx in params_output_df.index:\n    for h in param_headings:\n        params_output_df.at[idx, h] = param_outputs[idx][h]\n    \nprint('saving CSV')\nparams_output_df.to_csv('optuna_lgbm_params.csv', index=True)\n\nparam_means = params_output_df.mean(axis=0)\nparam_means","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FOLD_VALUES = [x for x in train_set['fold'].unique()] \nprint('FOLDS', FOLD_VALUES)\n\noof = np.zeros((len(train_set),))\ntest_predictions = np.zeros((len(sample_submission),))\n\nft_importances = pd.Series(index=SEL_FTS, data=0.0)\n\nRANDOM_SEEDS = [42, 0, 1, 2, 3, 4, 5, 6]\n\nfor RS in RANDOM_SEEDS:\n    \n    print('running random seed', RS)\n    \n    for fold in train_set['fold'].unique():\n\n        trn_idx = train_set['fold']!=fold\n        val_idx = train_set['fold']==fold\n\n        X_train = train_set.loc[trn_idx,SEL_FTS].values\n        y_train = train_set.loc[trn_idx, LABEL].values\n\n        X_val = train_set.loc[val_idx,SEL_FTS].values\n        y_val = train_set.loc[val_idx, LABEL].values\n\n        #print(X_train.shape, y_train.shape, X_val.shape, y_val.shape)\n\n        lgb = LGBMRegressor(random_state = RS,\n                            n_estimators = int(param_means['n_estimators']),\n                             subsample = param_means['subsample'],\n                             reg_alpha = param_means['reg_alpha'],\n                             max_depth = int(param_means['max_depth']),\n                             min_child_samples = int(param_means['min_child_samples']),\n                             colsample_bytree = param_means['colsample_bytree'],\n                        )\n\n        lgb.fit(X_train, y_train)\n        preds = lgb.predict(X_val)\n        oof[val_idx]+=preds\n\n        test_predictions+=lgb.predict(test_set[SEL_FTS])\n\n        ft_importances[:] += lgb.feature_importances_\n\n        print('Random Seed', RS, 'Fold ', fold, 'Error', np.sqrt(mse(y_val, preds)))\n\noof = oof / len(RANDOM_SEEDS)\n        \nprint('final OOF MSE', np.sqrt(mse(train_set[LABEL], oof)))\n\ntest_predictions = test_predictions/(len(FOLD_VALUES) * len(RANDOM_SEEDS))\n\nft_importances[:]  = ft_importances/(len(FOLD_VALUES) * len(RANDOM_SEEDS))\n\nsns.kdeplot(test_predictions, color='Red')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#feature importances - distribution\n\nsns.kdeplot(ft_importances, color='Blue')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#feature importances - visualise top results\n\nfig,axes=plt.subplots(figsize=(8,20))\nft_importances = ft_importances.sort_values(ascending=False)\naxes.barh(width=ft_importances[0:20],y=ft_importances.index[0:20])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#review distributions of train vs test for highest importance features\n\nTOP_FTS = [x for x in ft_importances.index[0:20]]\n\nnc=5\nnr=len(TOP_FTS)//nc\n\nfig,axes=plt.subplots(nrows=nr, ncols=nc, figsize=(25,nr*4))\n\nfor count,sf in enumerate(TOP_FTS):\n    sns.distplot(train_set[sf], ax=axes[count//nc, count%nc], color='Green')\n    sns.distplot(test_set[sf], ax=axes[count//nc, count%nc], color='Red')\n    \n    axes[count//nc, count%nc].set_title(sf)\n    sns.despine(ax=axes[count//nc, count%nc])\n    \nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#total summed feature importance for each measure e.g. 'BH_pow' across all sensors\n\nmeasure_list = [x.lstrip('s9_') for x in SEL_FTS if 's9_' in x]\n\nmeasure_list = pd.Series(index=measure_list, data=0.0)\n\nfor sc in measure_list.index:\n    cols_ = [x for x in ft_importances.index if sc in x]\n    \n    measure_list[sc] = ft_importances[cols_].sum()\n\nfig,axes=plt.subplots(figsize=(8,6))\naxes.barh(width=measure_list,y=measure_list.index)\n\naxes.set_title('Sum of Feature Importance across Sensors')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#standard deviation in feature importance for each measure e.g. 'BH_pow' across all sensors\n\nmeasure_list = [x.lstrip('s9_') for x in SEL_FTS if 's9_' in x]\n\nmeasure_list = pd.Series(index=measure_list, data=0.0)\n\nfor sc in measure_list.index:\n    cols_ = [x for x in ft_importances.index if sc in x]\n    \n    measure_list[sc] = ft_importances[cols_].std()\n\nfig,axes=plt.subplots(figsize=(8,6))\naxes.barh(width=measure_list,y=measure_list.index)\n\naxes.set_title('Standard Deviation of Feature Importance across Sensors')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['time_to_eruption'] = test_predictions\nsample_submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}