{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport time\nimport logging\nimport datetime\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom scipy import stats\nfrom scipy.signal import hann\nfrom tqdm import tqdm_notebook\nimport matplotlib.pyplot as plt\nfrom scipy.signal import hilbert\nfrom scipy.signal import convolve\nfrom sklearn.svm import NuSVR, SVR\nfrom catboost import CatBoostRegressor\nfrom sklearn.kernel_ridge import KernelRidge\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.model_selection import KFold,StratifiedKFold, RepeatedKFold\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import GridSearchCV,RandomizedSearchCV\nfrom catboost import CatBoostRegressor\nwarnings.filterwarnings(\"ignore\")\nfrom typing import TypeVar, List, Dict, Tuple\nPandasDataFrame = TypeVar('pandas.core.frame.DataFrame')\n\n# Refs: https://www.kaggle.com/byfone/basic-feature-feat-catboost\n# Refs: https://www.kaggle.com/kernels/scriptcontent/13873316/download","execution_count":1,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-input":false,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"IS_LOCAL = False\nif(IS_LOCAL):\n    PATH=\"../input/LANL/\"\nelse:\n    PATH=\"../input/\"\nos.listdir(PATH)\n\ntrain_X = pd.read_csv(os.path.join(PATH,'../input/all-features/train_X.csv'))\ntrain_y = pd.read_csv(os.path.join(PATH,'../input/all-features/train_y.csv'))\ntrain_X.shape, train_y.shape","execution_count":6,"outputs":[{"output_type":"execute_result","execution_count":6,"data":{"text/plain":"((4194, 1620), (4194, 1))"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv(os.path.join(PATH,'LANL-Earthquake-Prediction/sample_submission.csv'), index_col='seg_id')\ntest_X = pd.read_csv(os.path.join(PATH,'../input/all-features/test_X.csv'))\nsubmission.shape, test_X.shape","execution_count":7,"outputs":[{"output_type":"execute_result","execution_count":7,"data":{"text/plain":"((2624, 1), (2624, 1620))"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"Use cross-validation (e.g., KFold)\"\"\"\ndef fold_generator(x, y, groups=None, num_folds=10, shuffle=True, seed=2019):\n    folds = KFold(num_folds, shuffle=shuffle, random_state=seed)\n    for train_index, test_index in folds.split(x, y, groups):\n        yield train_index, test_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"Search the hyperparameters and return the estimator with the best parameters\"\"\"\ndef search_cv(x, y, model, grid, max_iter=None, num_folds=10, shuffle=True):\n    t0 = time.time()\n    \n    cv = fold_generator(x, y, num_folds=num_folds)\n    if max_iter is None:\n        # Exhaustive search over specified parameter values for an estimator (model)\n        search = GridSearchCV(model, grid, cv=cv,\n                              scoring='neg_mean_absolute_error')\n    else:\n        # Randomized search on hyper parameters with \n        # The number of parameter settings that are tried is given by n_iter (not all parameter values are tried out)\n        search = RandomizedSearchCV(model, grid, n_iter=max_iter, cv=cv,\n                                    scoring='neg_mean_absolute_error')\n    search.fit(x, y, silent=True)\n    \n    t0 = time.time() - t0\n    print(\"Best CV score: {:.4f}, time: {:.1f}s\".format(-search.best_score_, t0))\n    print(search.best_params_)\n    return search.best_estimator_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"Train, make predictions & plot results\"\"\"\ndef make_predictions(x, y, model, num_folds=10, shuffle=True, test=None, plot=True):\n    if test is not None:\n        sub_prediction = np.zeros(test.shape[0])\n        \n    oof_prediction = np.zeros(x.shape[0])\n    # use cross-validation (10-fold cross-validation)\n    for tr_idx, val_idx in fold_generator(x, y, num_folds=num_folds):\n        model.fit(x.iloc[tr_idx], y.iloc[tr_idx])\n        oof_prediction[val_idx] = model.predict(x.iloc[val_idx])\n\n        if test is not None:\n            sub_prediction += model.predict(test) / num_folds\n    \n    if plot:\n        plot_predictions(y, oof_prediction)\n    if test is None:\n        return oof_prediction\n    else:\n        return oof_prediction, sub_prediction","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_predictions(y, oof_predictions):\n    \"\"\"Plot out-of-fold predictions vs actual values.\"\"\"\n    fig, axis = plt.subplots(1, 2, figsize=(14, 6))\n    ax1, ax2 = axis\n    ax1.set_xlabel('actual')\n    ax1.set_ylabel('predicted')\n    ax1.set_ylim([-5, 20])\n    ax2.set_xlabel('train index')\n    ax2.set_ylabel('time to failure')\n    ax2.set_ylim([-2, 18])\n    ax1.scatter(y, oof_predictions, color='brown')\n    ax1.plot([(0, 0), (20, 20)], [(0, 0), (20, 20)], color='blue')\n    ax2.plot(y, color='blue', label='y_train')\n    ax2.plot(oof_predictions, color='orange')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scaler = StandardScaler()\nscaler.fit(train_X)\nscaled_train_X = pd.DataFrame(scaler.transform(train_X), columns=train_X.columns)\nscaled_test_X = pd.DataFrame(scaler.transform(test_X), columns=test_X.columns)\nscaled_test_X.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grid = {\n    'depth': [4, 6, 8],\n    'random_seed' : [400, 100, 200],\n    'learning_rate' : [0.01, 0.03, 0.1]\n}\ncat_model = CatBoostRegressor(loss_function='MAE', task_type = \"GPU\")\ncat_model = search_cv(scaled_train_X, train_y, cat_model, grid, max_iter=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Feature Importance\nfea_imp = pd.DataFrame({'imp': cat_model.feature_importances_, 'col': scaled_train_X.columns})\nfea_imp = fea_imp.sort_values(['imp', 'col'], ascending=[True, False]).iloc[-30:]\n_ = fea_imp.plot(kind='barh', x='col', y='imp', figsize=(20, 10))\nplt.savefig('catboost_feature_importance.png')  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof = make_predictions(scaled_train_X, train_y, cat_model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof, sub = make_predictions(scaled_train_X, train_y, cat_model,\n                                  test=scaled_test_X, plot=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.time_to_failure = sub\nsubmission.to_csv('submission.csv',index=True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.3"}},"nbformat":4,"nbformat_minor":1}