{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# Importing core libraries\nimport os\nimport gc\nimport numpy as np\nimport pandas as pd\nfrom time import time\nimport pprint\nimport joblib\nimport logging\n\n\n# Suppressing warnings because of skopt verbosity\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# Boosting models\nimport catboost as cat\nimport xgboost as xgb\nimport lightgbm as lgb\n\n# Hyperparameters distributions\nfrom scipy.stats import randint\nfrom scipy.stats import uniform\n\n# Preprocesing\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import LabelEncoder\n\n# Utilities\nfrom sklearn.pipeline import Pipeline\nfrom tqdm import tqdm_notebook\nfrom itertools import chain\nfrom sklearn.model_selection import train_test_split\n\n# Visualizations\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n\n# Model selection\nfrom sklearn.model_selection import StratifiedKFold, KFold\nfrom sklearn.model_selection import GridSearchCV, RandomizedSearchCV\nfrom sklearn.model_selection import cross_val_score\n\n\n# Metrics\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import make_scorer\n\n# Skopt functions\nfrom skopt import BayesSearchCV\nfrom skopt import gp_minimize # Bayesian optimization using Gaussian Processes\nfrom skopt.space import Real, Categorical, Integer\nfrom skopt.utils import use_named_args # decorator to convert a list of parameters to named arguments\nfrom skopt.callbacks import DeadlineStopper # Stop the optimization before running out of a fixed budget of time.\nfrom skopt.callbacks import VerboseCallback # Callback to control the verbosity\nfrom skopt.callbacks import DeltaXStopper # Stop the optimization If the last two positions at which the objective has been evaluated are less than delta","execution_count":35,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IS_LOCAL = False\nif(IS_LOCAL):\n    PATH=\"../input/LANL/\"\nelse:\n    PATH=\"../input/\"\nos.listdir(PATH)","execution_count":36,"outputs":[{"output_type":"execute_result","execution_count":36,"data":{"text/plain":"['all-features', 'basic-features', 'LANL-Earthquake-Prediction']"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"scaled_train_X = pd.read_csv('../input/all-features/train_X.csv')\nscaled_test_X = pd.read_csv('../input/all-features/test_X.csv')\ntrain_y = pd.read_csv('../input/all-features/train_y.csv')","execution_count":37,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scaled_train_X.shape, scaled_test_X.shape, train_y.shape","execution_count":38,"outputs":[{"output_type":"execute_result","execution_count":38,"data":{"text/plain":"((4194, 1620), (2624, 1620), (4194, 1))"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/LANL-Earthquake-Prediction/sample_submission.csv', index_col='seg_id')\nsubmission.shape","execution_count":39,"outputs":[{"output_type":"execute_result","execution_count":39,"data":{"text/plain":"(2624, 1)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Converting average precision score into a scorer suitable for model selection\nmse_scoring = make_scorer(mean_squared_error, greater_is_better=False)\n# Setting a 5-fold stratified cross-validation (note: shuffle=True)\n# skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=0)S\nk_fold = KFold(n_splits=7, shuffle=True, random_state=13)","execution_count":40,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# The objective function to be minimized\ndef make_objective(model, X, y, space, cv, scoring):\n    # This decorator converts your objective function with named arguments into one that\n    # accepts a list as argument, while doing the conversion automatically.\n    @use_named_args(space) \n    def objective(**params):\n        model.set_params(**params)\n        return -np.mean(cross_val_score(model, \n                                        X, y, \n                                        cv=cv, \n                                        n_jobs=-1,\n                                        scoring=scoring))\n\n    return objective","execution_count":41,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#xgb_bayes_params = best_params\n\nxgb_bayes_params = {'colsample_bytree': 0.3706219857878677,\n 'learning_rate': 0.16624226726409647,\n 'max_bin': 93400,\n 'max_depth': 134,\n 'min_child_samples': 22,\n 'min_child_weight': 4,\n 'n_estimators': 8028,\n 'num_leaves': 27,\n 'reg_alpha': 1.081049236893711e-05,\n 'reg_lambda': 1.043686239159047,\n 'scale_pos_weight': 0.19222548462579486,\n 'subsample': 0.6941640075502717,\n 'subsample_for_bin': 375140,\n 'subsample_freq': 7}\nparams=xgb_bayes_params","execution_count":42,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"counter = 0\n\nxgb_model = xgb.XGBRegressor(\n    metric='mae',\n    n_jobs=1, \n    verbose=0\n)\n\ndimensions = [Real(0.01, 1.0, 'log-uniform', name='learning_rate'),\n              Integer(2, 500, name='num_leaves'),\n              Integer(0, 500, name='max_depth'),\n              Integer(0, 200, name='min_child_samples'),\n              Integer(100, 100000, name='max_bin'),\n              Real(0.01, 1.0, 'uniform', name='subsample'),\n              Integer(0, 10, name='subsample_freq'),\n              Real(0.01, 1.0, 'uniform', name='colsample_bytree'),\n              Integer(0, 10, name='min_child_weight'),\n              Integer(100000, 500000, name='subsample_for_bin'),\n              Real(1e-9, 1000, 'log-uniform', name='reg_lambda'),\n              Real(1e-9, 1.0, 'log-uniform', name='reg_alpha'),\n              Real(1e-6, 500, 'log-uniform', name='scale_pos_weight'),\n              Integer(10, 10000, name='n_estimators')]\n\nobjective = make_objective(xgb_model,\n                           scaled_train_X, train_y,\n                           space=dimensions,\n                           cv=k_fold,\n                           scoring=mse_scoring)","execution_count":43,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from typing import TypeVar, List, Dict, Tuple\nPandasDataFrame = TypeVar('pandas.core.frame.DataFrame')\n\ndef run_xgb(\n    X_tr: PandasDataFrame,\n    X_val: PandasDataFrame,\n    y_tr: PandasDataFrame,\n    y_val: PandasDataFrame,\n    test_data: PandasDataFrame,\n    params: Dict\n):\n    \"\"\"CV train lgb Booster.\n    \n    Args:\n        params: Params for Booster.\n        X_train: Training dataset.\n        X_test: Testing dataset.\n        \n    Returns:\n        model: Trained model.\n        oof_train_lgb:  Training CV predictions.\n        oof_test_lgb:  Testing CV predictions.\n    \"\"\"\n    \n    early_stop = 200\n    num_rounds = 10000\n    verbose_eval=1000\n\n    d_train = xgb.DMatrix(X_tr.values, label = y_tr)\n    d_valid = xgb.DMatrix(X_val.values, label = y_val )\n    d_test = xgb.DMatrix(test_data.values) #+ mg\n    watchlist = [(d_train, 'train'), (d_valid, 'valid')]\n    \n    #1st run to get importance\n    model = xgb.train(params, dtrain=d_train,num_boost_round=num_rounds,evals=watchlist, \n                      verbose_eval=verbose_eval,early_stopping_rounds=early_stop)    \n    #importance\n    imps=model.get_score(importance_type='gain')\n    f_nums = model.feature_names\n    f_names = list(scaled_train_X)\n    f_dict = dict(zip(f_nums,f_names))\n    keys=[]\n    for i in imps:\n        keys.append(f_dict.get(i))\n    X_tr_new = X_tr.loc[:,keys]\n    X_val_new=X_val.loc[:,keys]\n    test_data_new=test_data.loc[:,keys]\n    \n    d_train = xgb.DMatrix(X_tr_new.values, label = y_tr)\n    d_valid = xgb.DMatrix(X_val_new.values, label = y_val )\n    d_test = xgb.DMatrix(test_data_new.values) #+ mg\n    watchlist = [(d_train, 'train'), (d_valid, 'valid')]\n    \n    model = xgb.train(params, dtrain=d_train,num_boost_round=num_rounds,evals=watchlist, \n                      verbose_eval=verbose_eval,early_stopping_rounds=early_stop) \n\n    val_pred = model.predict(d_valid)\n    prediction = model.predict(d_test)\n    \n    return val_pred, prediction\n\ndef run_cv_model(\n    train_data,\n    train_target,\n    test_data,\n    model_fn, \n    params,\n    eval_fn=None,\n    label='model',\n    feature_imp=False,\n    n_folds = 5\n):\n    oof_val = np.zeros(len(train_data))\n    predictions = np.zeros(len(test_data))\n    oof_predict = np.zeros((n_folds, test_data.shape[0]))\n    scores = []\n\n   # feature_importance_df = pd.DataFrame()\n    \n    folds = KFold(n_splits=n_folds, shuffle=True, random_state=42)\n    train_columns = train_data.columns.values\n    \n\n    for fold_, (trn_idx, val_idx) in enumerate(folds.split(train_data, train_target.values)):\n        strLog = \"fold {} for {}\".format(fold_, label)\n        print(strLog)\n        X_tr, X_val = train_data.iloc[trn_idx], train_data.iloc[val_idx]\n        y_tr, y_val = train_target.iloc[trn_idx], train_target.iloc[val_idx]\n        \n        val_pred, prediction= model_fn( #, feature_importances \n            X_tr, X_val,\n            y_tr, y_val,\n            test_data,\n            params\n        )\n        score = mean_squared_error(y_val, val_pred)\n        scores.append(score)\n\n        \n        oof_val[val_idx] = val_pred\n        \n        #feature importance\n        if feature_imp == True:\n            fold_importance_df = pd.DataFrame()\n            fold_importance_df[\"feature\"] = train_columns\n            fold_importance_df[\"importance\"] = feature_importances[:len(train_columns)]\n            fold_importance_df[\"fold\"] = fold_ + 1\n            feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n        \n        #predictions\n        oof_predict[fold_] = prediction\n        predictions += prediction/folds.n_splits\n        \n        print('CV score: {0:.4f}, std: {1:.4f}.\\n'.format(np.mean(score), np.std(score)))  \n        print('CV mean score: {0:.4f}, std: {1:.4f}.\\n'.format(np.mean(scores), np.std(scores)))  \n\n\n            \n    return oof_val, oof_predict, predictions #, feature_importance_df","execution_count":65,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fix_xgb_params = {\n    \"boosting\": \"gblinear\",\n    \"metric\": 'mae',\n}\n\nxgb_params = {**fix_xgb_params, **xgb_bayes_params}\n\n\noof_val, oof_predict, predictions  = run_cv_model( #extra param: '', feature_importance_df'''\n    train_data=scaled_train_X,\n    train_target=train_y,\n    test_data=scaled_test_X,\n    model_fn=run_xgb,\n    params=xgb_params,\n    eval_fn=None,\n    label=\"xgb\",\n    feature_imp=False,\n    n_folds=10\n)\n","execution_count":66,"outputs":[{"output_type":"stream","text":"fold 0 for xgb\n[0]\ttrain-rmse:5.46787\tvalid-rmse:5.46655\nMultiple eval metrics have been passed: 'valid-rmse' will be used for early stopping.\n\nWill train until valid-rmse hasn't improved in 200 rounds.\n","name":"stdout"},{"output_type":"error","ename":"KeyboardInterrupt","evalue":"","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)","\u001b[0;32m<ipython-input-66-36ca2f35a7df>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     16\u001b[0m     \u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"xgb\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m     \u001b[0mfeature_imp\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m     \u001b[0mn_folds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m )\n","\u001b[0;32m<ipython-input-65-28f44b52fddd>\u001b[0m in \u001b[0;36mrun_cv_model\u001b[0;34m(train_data, train_target, test_data, model_fn, params, eval_fn, label, feature_imp, n_folds)\u001b[0m\n\u001b[1;32m     92\u001b[0m             \u001b[0my_tr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_val\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m             \u001b[0mtest_data\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 94\u001b[0;31m             \u001b[0mparams\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     95\u001b[0m         )\n\u001b[1;32m     96\u001b[0m         \u001b[0mscore\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmean_squared_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_pred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m<ipython-input-65-28f44b52fddd>\u001b[0m in \u001b[0;36mrun_xgb\u001b[0;34m(X_tr, X_val, y_tr, y_val, test_data, params)\u001b[0m\n\u001b[1;32m     34\u001b[0m     \u001b[0;31m#1st run to get importance\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m     model = xgb.train(params, dtrain=d_train,num_boost_round=num_rounds,evals=watchlist, \n\u001b[0;32m---> 36\u001b[0;31m                       verbose_eval=verbose_eval,early_stopping_rounds=early_stop)    \n\u001b[0m\u001b[1;32m     37\u001b[0m     \u001b[0;31m#importance\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m     \u001b[0mimps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimportance_type\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'gain'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/xgboost/training.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(params, dtrain, num_boost_round, evals, obj, feval, maximize, early_stopping_rounds, evals_result, verbose_eval, xgb_model, callbacks, learning_rates)\u001b[0m\n\u001b[1;32m    214\u001b[0m                            \u001b[0mevals\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mevals\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    215\u001b[0m                            \u001b[0mobj\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeval\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 216\u001b[0;31m                            xgb_model=xgb_model, callbacks=callbacks)\n\u001b[0m\u001b[1;32m    217\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    218\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/xgboost/training.py\u001b[0m in \u001b[0;36m_train_internal\u001b[0;34m(params, dtrain, num_boost_round, evals, obj, feval, xgb_model, callbacks)\u001b[0m\n\u001b[1;32m     72\u001b[0m         \u001b[0;31m# Skip the first update if it is a recovery step.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     73\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mversion\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 74\u001b[0;31m             \u001b[0mbst\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     75\u001b[0m             \u001b[0mbst\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_rabit_checkpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     76\u001b[0m             \u001b[0mversion\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/xgboost/core.py\u001b[0m in \u001b[0;36mupdate\u001b[0;34m(self, dtrain, iteration, fobj)\u001b[0m\n\u001b[1;32m   1108\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mfobj\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1109\u001b[0m             _check_call(_LIB.XGBoosterUpdateOneIter(self.handle, ctypes.c_int(iteration),\n\u001b[0;32m-> 1110\u001b[0;31m                                                     dtrain.handle))\n\u001b[0m\u001b[1;32m   1111\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1112\u001b[0m             \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mKeyboardInterrupt\u001b[0m: "]}]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.time_to_failure = predictions\nsubmission.to_csv('submission.csv',index=True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}