{"cells":[{"metadata":{},"cell_type":"markdown","source":"* Below you may notice the effect of using MAE/RMSE as objective functions of the resulting MAE for the competition\n* snippet from https://www.kaggle.com/tocha4/lanl-master-s-approach"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport scipy as sc\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport gc\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nfrom tqdm import tqdm_notebook\nimport datetime\nimport time\nimport random\nfrom joblib import Parallel, delayed\n\n\nimport lightgbm as lgb\nfrom tensorflow import keras\nfrom gplearn.genetic import SymbolicRegressor\nfrom catboost import Pool, CatBoostRegressor\n\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import GridSearchCV, KFold, RandomizedSearchCV\nfrom sklearn.feature_selection import RFECV, SelectFromModel\n\nfrom sklearn.linear_model import LinearRegression, Ridge\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.svm import NuSVR, SVR\nfrom sklearn.kernel_ridge import KernelRidge\nfrom sklearn.ensemble import AdaBoostRegressor\nfrom sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_X_0 = pd.read_csv(\"../input/lanl-master-s-features-creating-0/train_X_features_865.csv\")\ntrain_X_1 = pd.read_csv(\"../input/lanl-master-s-features-creating-1/train_X_features_865.csv\")\ny_0 = pd.read_csv(\"../input/lanl-master-s-features-creating-0/train_y.csv\", index_col=False,  header=None)\ny_1 = pd.read_csv(\"../input/lanl-master-s-features-creating-1/train_y.csv\", index_col=False,  header=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X = pd.concat([train_X_0, train_X_1], axis=0)\ntrain_X = train_X.reset_index(drop=True)\nprint(train_X.shape)\ntrain_X.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = pd.concat([y_0, y_1], axis=0)\ny = y.reset_index(drop=True)\ny[0].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_y = pd.Series(y[0].values)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"test_X = pd.read_csv(\"../input/lanl-master-s-features-creating-0/test_X_features_10.csv\")\n# del X[\"seg_id\"], test_X[\"seg_id\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scaler = StandardScaler()\ntrain_columns = train_X.columns\n\ntrain_X[train_columns] = scaler.fit_transform(train_X[train_columns])\ntest_X[train_columns] = scaler.transform(test_X[train_columns])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CatBoost"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_columns = train_X.columns\nn_fold = 5","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"MAE"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfolds = KFold(n_splits=n_fold, shuffle=True, random_state=42)\n\noof = np.zeros(len(train_X))\ntrain_score = []\nfold_idxs = []\n# if PREDICTION: \npredictions = np.zeros(len(test_X))\n\nfeature_importance_df = pd.DataFrame()\n#run model\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train_X,train_y.values)):\n    strLog = \"fold {}\".format(fold_)\n    print(strLog)\n    fold_idxs.append(val_idx)\n\n    X_tr, X_val = train_X[train_columns].iloc[trn_idx], train_X[train_columns].iloc[val_idx]\n    y_tr, y_val = train_y.iloc[trn_idx], train_y.iloc[val_idx]\n\n    model = CatBoostRegressor(n_estimators=30000, verbose=-1, objective=\"MAE\", loss_function=\"MAE\", boosting_type=\"Ordered\", task_type=\"GPU\")\n    model.fit(X_tr, \n              y_tr, \n              eval_set=[(X_val, y_val)], \n#               eval_metric='mae',\n              verbose=2500, \n              early_stopping_rounds=600)\n    oof[val_idx] = model.predict(X_val)\n\n    #feature importance\n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"Feature\"] = train_columns\n    fold_importance_df[\"importance\"] = model.feature_importances_[:len(train_columns)]\n    fold_importance_df[\"fold\"] = fold_ + 1\n    feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n    #predictions\n#     if PREDICTION:\n\n    predictions += model.predict(test_X[train_columns]) / folds.n_splits\n    train_score.append(model.best_score_['learn'][\"MAE\"])\n\ncv_score = mean_absolute_error(train_y, oof)\nprint(f\"After {n_fold} test_CV = {cv_score:.3f} | train_CV = {np.mean(train_score):.3f} | {cv_score-np.mean(train_score):.3f}\", end=\" \")\n\ntoday = str(datetime.date.today())\nsubmission = pd.read_csv('../input/LANL-Earthquake-Prediction/sample_submission.csv')\n\nsubmission[\"time_to_failure\"] = predictions\nsubmission.to_csv(f'CatBoost_MAE_{today}_test_{cv_score:.3f}_train_{np.mean(train_score):.3f}.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"RMSE"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfolds = KFold(n_splits=n_fold, shuffle=True, random_state=42)\n\noof = np.zeros(len(train_X))\ntrain_score = []\nfold_idxs = []\n# if PREDICTION: \npredictions = np.zeros(len(test_X))\n\nfeature_importance_df = pd.DataFrame()\n#run model\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train_X,train_y.values)):\n    strLog = \"fold {}\".format(fold_)\n    print(strLog)\n    fold_idxs.append(val_idx)\n\n    X_tr, X_val = train_X[train_columns].iloc[trn_idx], train_X[train_columns].iloc[val_idx]\n    y_tr, y_val = train_y.iloc[trn_idx], train_y.iloc[val_idx]\n\n    model = CatBoostRegressor(n_estimators=30000, verbose=-1, objective=\"RMSE\", loss_function=\"RMSE\", boosting_type=\"Ordered\", task_type=\"GPU\")\n    model.fit(X_tr, \n              y_tr, \n              eval_set=[(X_val, y_val)], \n#               eval_metric='mae',\n              verbose=2500, \n              early_stopping_rounds=600)\n    oof[val_idx] = model.predict(X_val)\n\n    #feature importance\n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"Feature\"] = train_columns\n    fold_importance_df[\"importance\"] = model.feature_importances_[:len(train_columns)]\n    fold_importance_df[\"fold\"] = fold_ + 1\n    feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n    #predictions\n#     if PREDICTION:\n\n    predictions += model.predict(test_X[train_columns]) / folds.n_splits\n    train_score.append(model.best_score_['learn'][\"RMSE\"])\n\ncv_score = mean_absolute_error(train_y, oof)\nprint(f\"After {n_fold} test_CV = {cv_score:.3f} | train_CV = {np.mean(train_score):.3f} | {cv_score-np.mean(train_score):.3f}\", end=\" \")\n\ntoday = str(datetime.date.today())\nsubmission = pd.read_csv('../input/LANL-Earthquake-Prediction/sample_submission.csv')\n\nsubmission[\"time_to_failure\"] = predictions\nsubmission.to_csv(f'CatBoost_RMSE_{today}_test_{cv_score:.3f}_train_{np.mean(train_score):.3f}.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}