{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# Imports\nimport numpy as np\nimport pandas as pd\nimport os\n\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.svm import NuSVR, SVR\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold, KFold, RepeatedKFold\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error\nfrom sklearn.linear_model import Ridge, RidgeCV\n\nimport lightgbm as lgb\nimport xgboost as xgb\nimport time\nimport datetime\npd.options.display.precision = 15\n\nimport psutil\nimport gc\nfrom catboost import CatBoostRegressor\nimport seaborn as sns\nfrom scipy import stats\n\nimport matplotlib.pyplot as plt\nget_ipython().run_line_magic('matplotlib', 'inline')\nfrom tqdm import tqdm_notebook\n\nfrom tsfresh.feature_extraction import feature_calculators\n\nimport warnings\nwarnings.filterwarnings('ignore')\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport scipy as sc\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport gc\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nfrom tqdm import tqdm_notebook\nimport datetime\nimport time\nimport random\nfrom joblib import Parallel, delayed\n\n\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom tensorflow import keras\nfrom gplearn.genetic import SymbolicRegressor\nfrom catboost import Pool, CatBoostRegressor\n\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import GridSearchCV, KFold, RandomizedSearchCV\nfrom sklearn.feature_selection import RFECV, SelectFromModel\n\nfrom sklearn.linear_model import LinearRegression, Ridge\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.svm import NuSVR, SVR\nfrom sklearn.kernel_ridge import KernelRidge\nfrom sklearn.ensemble import AdaBoostRegressor\nfrom sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir('../input'))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# # Read the csv file to take input into train dataframe using pandas, \n# # 2 attributes from the training data: acoustic data: records the seismic activity and\n# # time_to_failure: time left for the next laboratory earthquake\n# train_dataset = pd.read_csv('../input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(train_dataset.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b369f9462919c0c6e29f8a5b6183cb3c57f057a","scrolled":true},"cell_type":"code","source":"# # number of instances for each segment to be specified as 150000 because each test segment has 150000 observations.\n# instances = 150000\n# segments = int(np.floor(train_dataset.shape[0] / instances))\n\n# # X_trainset is the training data: acoustic data\n# # y_trainset is the value to be predicted, that is the time left for the next lab earthquake\n# X_trainset = pd.DataFrame(index=range(segments), dtype=np.float64)\n# y_trainset = pd.DataFrame(index=range(segments), dtype=np.float64,\n#                        columns=['time_to_failure'])\n\n# # for every segment id, do feature engineering\n# for segment in tqdm_notebook(range(segments)):\n\n# #   creating segments of size 150000 starting from the row of segment id and upto segment id + instances\n#     each_seg = train_dataset.iloc[segment * instances : segment * instances + instances]\n# #     create x and y having acoustic_data and time_to _failure respectively\n#     x_rawdata = each_seg['acoustic_data']\n#     x = x_rawdata.values\n#     x = pd.Series(x)\n    \n#     zc = np.fft.fft(x)\n#     realFFT = pd.Series(np.real(zc))\n#     imagFFT = pd.Series(np.imag(zc))\n    \n#     # most important features 30\n#     X_trainset.loc[segment, 'autocorrelation_5'] = feature_calculators.autocorrelation(x, 5)\n#     x_imagFFT_std = imagFFT.rolling(50).std().dropna().values\n#     X_trainset.loc[segment, 'quantile_50_percentile_imagFFT_standard_deviation_50'] = np.quantile(x_imagFFT_std,0.50)\n#     x_roll_std = x.rolling(1000).std().dropna().values\n#     X_trainset.loc[segment, 'quantile_5_percentile_rolling_standard_deviation_1000'] = np.quantile(x_roll_std,0.05)\n#     x_imagFFT_std = imagFFT.rolling(1000).std().dropna().values\n#     X_trainset.loc[segment, 'quantile_50_percentile_imagFFT_standard_deviation_1000'] = np.quantile(x_imagFFT_std,0.50)\n#     x_roll_std = x.rolling(50).std().dropna().values\n#     X_trainset.loc[segment, 'quantile_5_percentile_rolling_standard_deviation_50'] = np.quantile(x_roll_std,0.05)\n#     x_imagFFT_mean = imagFFT.rolling(100).mean().dropna().values\n#     X_trainset.loc[segment, 'quantile_50_percentile_imagFFT_mean_100'] = np.quantile(x_imagFFT_mean,0.50)\n#     x_roll_std = x.rolling(100).std().dropna().values\n#     X_trainset.loc[segment, 'minimum_rolling_standard_deviation_100'] = x_roll_std.min()\n#     x_roll_std = x.rolling(50).std().dropna().values\n#     X_trainset.loc[segment, 'minimum_rolling_standard_deviation_50'] = x_roll_std.min()\n#     x_roll_std = x.rolling(500).std().dropna().values\n#     X_trainset.loc[segment, 'minimum_rolling_standard_deviation_500'] = x_roll_std.min()\n#     x_imagFFT_std = imagFFT.rolling(100).std().dropna().values\n#     X_trainset.loc[segment, 'quantile_50_percentile_imagFFT_standard_deviation_100'] = np.quantile(x_imagFFT_std,0.50)\n#     x_imagFFT_std = imagFFT.rolling(10000).std().dropna().values\n#     X_trainset.loc[segment, 'average_change_absolute_imagFFT_standard_deviation_10000'] = np.mean(np.diff(x_imagFFT_std))\n#     x_imagFFT_std = imagFFT.rolling(50).std().dropna().values\n#     X_trainset.loc[segment, 'average_change_absolute_imagFFT_standard_deviation_50'] = np.mean(np.diff(x_imagFFT_std))\n#     x_realFFT_std = realFFT.rolling(500).std().dropna().values\n#     X_trainset.loc[segment, 'minimum_realFFT_standard_deviation_500'] = x_realFFT_std.min()\n#     x_realFFT_std = realFFT.rolling(10).std().dropna().values\n#     X_trainset.loc[segment, 'quantile_5_percentile_realFFT_standard_deviation_10'] = np.quantile(x_realFFT_std,0.05)\n#     x_imagFFT_mean = imagFFT.rolling(1000).mean().dropna().values\n#     X_trainset.loc[segment, 'quantile_50_percentile_imagFFT_mean_1000'] = np.quantile(x_imagFFT_mean,0.50)\n#     x_realFFT_std = realFFT.rolling(10).std().dropna().values\n#     X_trainset.loc[segment, 'quantile_1_percentile_realFFT_standard_deviation_10'] = np.quantile(x_realFFT_std,0.01)\n#     x_imagFFT_mean = imagFFT.rolling(50).mean().dropna().values\n#     X_trainset.loc[segment, 'minimum_imagFFT_standard_deviation_50'] = x_imagFFT_std.min()\n#     x_realFFT_mean = realFFT.rolling(1000).mean().dropna().values\n#     X_trainset.loc[segment, 'quantile_5_percentile_realFFT_mean_1000'] = np.quantile(x_realFFT_mean,0.05)\n#     x_realFFT_mean = realFFT.rolling(10).mean().dropna().values\n#     X_trainset.loc[segment, 'maximum_realFFT_mean_10'] = x_realFFT_mean.max()\n#     x_imagFFT_std = imagFFT.rolling(10).std().dropna().values\n#     X_trainset.loc[segment, 'average_change_absolute_imagFFT_standard_deviation_10'] = np.mean(np.diff(x_imagFFT_std))\n#     X_trainset.loc[segment, 'Isum'] = imagFFT.sum()\n#     X_trainset.loc[segment, 'Rmedian_absolute_last_5000'] = np.median(np.abs(realFFT[-5000:])) \n#     X_trainset.loc[segment, 'num_peaks_10'] = feature_calculators.number_peaks(x, 10)\n    \n#     y = each_seg['time_to_failure'].values[-1]\n    \n# #   y train data is the time_to_failure for that segment instance\n#     y_trainset.loc[segment, 'time_to_failure'] = y\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8a8fefba8af693620aa3598ac3de8f2ad521bbf4"},"cell_type":"code","source":"# submission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')\n# X_testset = pd.DataFrame(columns=X_trainset.columns, dtype=np.float64, index=submission.index)\n\n# for i, seg_id in enumerate(tqdm_notebook(X_testset.index)):\n#     each_seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n#     x_rawdata = each_seg['acoustic_data']\n#     x = x_rawdata.values\n#     x = pd.Series(x)\n    \n#     zc = np.fft.fft(x)\n#     realFFT = pd.Series(np.real(zc))\n#     imagFFT = pd.Series(np.imag(zc))\n    \n#     # most important features 30\n#     X_testset.loc[seg_id, 'autocorrelation_5'] = feature_calculators.autocorrelation(x, 5)\n#     x_imagFFT_std = imagFFT.rolling(50).std().dropna().values\n#     X_testset.loc[seg_id, 'quantile_50_percentile_imagFFT_standard_deviation_50'] = np.quantile(x_imagFFT_std,0.50)\n#     x_roll_std = x.rolling(1000).std().dropna().values\n#     X_testset.loc[seg_id, 'quantile_5_percentile_rolling_standard_deviation_1000'] = np.quantile(x_roll_std,0.05)\n#     x_imagFFT_std = imagFFT.rolling(1000).std().dropna().values\n#     X_testset.loc[seg_id, 'quantile_50_percentile_imagFFT_standard_deviation_1000'] = np.quantile(x_imagFFT_std,0.50)\n#     x_roll_std = x.rolling(50).std().dropna().values\n#     X_testset.loc[seg_id, 'quantile_5_percentile_rolling_standard_deviation_50'] = np.quantile(x_roll_std,0.05)\n#     x_imagFFT_mean = imagFFT.rolling(100).mean().dropna().values\n#     X_testset.loc[seg_id, 'quantile_50_percentile_imagFFT_mean_100'] = np.quantile(x_imagFFT_mean,0.50)\n#     x_roll_std = x.rolling(100).std().dropna().values\n#     X_testset.loc[seg_id, 'minimum_rolling_standard_deviation_100'] = x_roll_std.min()\n#     x_roll_std = x.rolling(50).std().dropna().values\n#     X_testset.loc[seg_id, 'minimum_rolling_standard_deviation_50'] = x_roll_std.min()\n#     x_roll_std = x.rolling(500).std().dropna().values\n#     X_testset.loc[seg_id, 'minimum_rolling_standard_deviation_500'] = x_roll_std.min()\n#     x_imagFFT_std = imagFFT.rolling(100).std().dropna().values\n#     X_testset.loc[seg_id, 'quantile_50_percentile_imagFFT_standard_deviation_100'] = np.quantile(x_imagFFT_std,0.50)\n#     x_imagFFT_std = imagFFT.rolling(10000).std().dropna().values\n#     X_testset.loc[seg_id, 'average_change_absolute_imagFFT_standard_deviation_10000'] = np.mean(np.diff(x_imagFFT_std))\n#     x_imagFFT_std = imagFFT.rolling(50).std().dropna().values\n#     X_testset.loc[seg_id, 'average_change_absolute_imagFFT_standard_deviation_50'] = np.mean(np.diff(x_imagFFT_std))\n#     x_realFFT_std = realFFT.rolling(500).std().dropna().values\n#     X_testset.loc[seg_id, 'minimum_realFFT_standard_deviation_500'] = x_realFFT_std.min()\n#     x_realFFT_std = realFFT.rolling(10).std().dropna().values\n#     X_testset.loc[seg_id, 'quantile_5_percentile_realFFT_standard_deviation_10'] = np.quantile(x_realFFT_std,0.05)\n#     x_imagFFT_mean = imagFFT.rolling(1000).mean().dropna().values\n#     X_testset.loc[seg_id, 'quantile_50_percentile_imagFFT_mean_1000'] = np.quantile(x_imagFFT_mean,0.50)\n#     x_realFFT_std = realFFT.rolling(10).std().dropna().values\n#     X_testset.loc[seg_id, 'quantile_1_percentile_realFFT_standard_deviation_10'] = np.quantile(x_realFFT_std,0.01)\n#     x_imagFFT_mean = imagFFT.rolling(50).mean().dropna().values\n#     X_testset.loc[seg_id, 'minimum_imagFFT_standard_deviation_50'] = x_imagFFT_std.min()\n#     x_realFFT_mean = realFFT.rolling(1000).mean().dropna().values\n#     X_testset.loc[seg_id, 'quantile_5_percentile_realFFT_mean_1000'] = np.quantile(x_realFFT_mean,0.05)\n#     x_realFFT_mean = realFFT.rolling(10).mean().dropna().values\n#     X_testset.loc[seg_id, 'maximum_realFFT_mean_10'] = x_realFFT_mean.max()\n#     x_imagFFT_std = imagFFT.rolling(10).std().dropna().values\n#     X_testset.loc[seg_id, 'average_change_absolute_imagFFT_standard_deviation_10'] = np.mean(np.diff(x_imagFFT_std))\n#     X_testset.loc[seg_id, 'Isum'] = imagFFT.sum()\n#     X_testset.loc[seg_id, 'Rmedian_absolute_last_5000'] = np.median(np.abs(realFFT[-5000:])) \n#     X_testset.loc[seg_id, 'num_peaks_10'] = feature_calculators.number_peaks(x, 10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# X_trainset.to_csv('X_trainset.csv')\n# X_testset.to_csv('X_testset.csv')\n# y_trainset.to_csv('y_trainset.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_trainset = pd.read_csv('../input/earthquakefe/X_trainset.csv').reset_index(drop=True)\nX_testset = pd.read_csv('../input/earthquakefe/X_testset.csv').reset_index(drop=True)\ny_trainset = pd.read_csv('../input/earthquakefe/y_trainset.csv')['time_to_failure']\nsubmission = pd.read_csv('../input/LANL-Earthquake-Prediction/sample_submission.csv', index_col='seg_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(y_trainset.shape)\ny_trainset.head()\nX_trainset.head()\ndel X_trainset['Unnamed: 0']\ndel X_testset[\"seg_id\"]\nprint(X_trainset.shape)\nprint(X_testset.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LGBM"},{"metadata":{"trusted":true,"_uuid":"9172a3903b178e151d8aefbc68e01426a39e9e2e"},"cell_type":"code","source":"num_folds = 5\nk_folds = KFold(n_splits=num_folds, shuffle=True, random_state=630)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model_lgb(X=X_trainset, X_testset=X_testset, y=y_trainset, params=None, k_folds=k_folds, model=None):\n    columns = X.columns\n    \n    x_values = np.zeros(len(X))\n    prediction = np.zeros(len(X_testset))\n    scores = []\n    feature_importance = pd.DataFrame()\n    for fold_n, (trainset_index, valid_set_index) in enumerate(k_folds.split(X)):\n        print('Fold', fold_n, 'started at', time.ctime())\n        X_train_per_fold, X_valid_per_fold = X.iloc[trainset_index], X.iloc[valid_set_index]\n        y_train_per_fold, y_valid_per_fold = y.iloc[trainset_index], y.iloc[valid_set_index]\n\n        model = lgb.LGBMRegressor(**params, n_estimators = 50000, nthread = 4, n_jobs = -1)\n        model.fit(X_train_per_fold, y_train_per_fold, \n               eval_set=[(X_train_per_fold, y_train_per_fold), (X_valid_per_fold, y_valid_per_fold)], eval_metric='mae',\n               verbose=10000, early_stopping_rounds=200)\n\n        y_pred_valid = model.predict(X_valid_per_fold)\n        y_pred = model.predict(X_testset, num_iteration=model.best_iteration_)\n\n        x_values[valid_set_index] = y_pred_valid.reshape(-1,)\n        scores.append(mean_absolute_error(y_valid_per_fold, y_pred_valid))\n        \n        fold_importance = pd.DataFrame()\n        fold_importance[\"feature\"] = columns\n        fold_importance[\"importance\"] = model.feature_importances_\n        fold_importance[\"fold\"] = fold_n + 1\n        feature_importance = pd.concat([feature_importance, fold_importance], axis=0)\n        \n        prediction += y_pred\n        \n    prediction /= num_folds\n#     feature_importance[\"importance\"] /= k_folds.n_splits\n    cols = feature_importance[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(by=\"importance\", ascending=False)[:30].index\n    best_features = feature_importance.loc[feature_importance.feature.isin(cols)]\n\n    plt.figure(figsize=(16, 12));\n    sns.barplot(x=\"importance\", y=\"feature\", data=best_features.sort_values(by=\"importance\", ascending=False));\n    plt.title('LGB Features (avg over folds)');\n\n    print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))\n    return x_values, prediction","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c532c79cb6f3417c3a9eec579979aaea91a0aa10"},"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline\n\n# lgb_params = {'num_leaves': 32,\n#               'min_data_in_leaf': 16,\n#               'objective': 'regression',\n#               'max_depth': -1,\n#               'learning_rate': 0.001,\n#               \"boosting\": \"gbdt\",\n#               \"feature_fraction\": 0.8,\n#               \"bagging_freq\": 2,\n#               \"bagging_fraction\": 0.8,\n#               \"bagging_seed\": 630,\n#               \"metric\": 'mae',\n#               \"verbosity\": -1,\n#               'reg_alpha': 0.15,\n#               'reg_lambda': 0.36,\n#          }\nlgb_params = {'num_leaves': 21,\n              'min_data_in_leaf': 20,\n              'objective': 'regression',\n              'max_depth': 108,\n              'learning_rate': 0.001,\n              'boosting': 'gbdt',\n              'feature_fraction': 0.8,\n              'bagging_freq': 2,\n              'bagging_fraction': 0.8,\n              'bagging_seed': 630,\n              'metric': 'mae',\n              'verbosity': -1,\n              'reg_alpha': 0.15,\n              'reg_lambda': 0.36,\n              'random_state': 630,\n         }\nx_value_lgb, prediction_lgb = train_model_lgb(params = lgb_params)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"acf417f3540ca69267b1088e199e4ae358b36439"},"cell_type":"code","source":"submission = pd.read_csv('../input/LANL-Earthquake-Prediction/sample_submission.csv')\nsubmission['time_to_failure'] = prediction_lgb\nprint(submission.head())\nsubmission.to_csv('30_lgbm.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline\n\nlgb_params = {'num_leaves': 32,\n              'min_data_in_leaf': 16,\n              'objective': 'regression',\n              'max_depth': -1,\n              'learning_rate': 0.001,\n              \"boosting\": \"gbdt\",\n              \"feature_fraction\": 0.8,\n              \"bagging_freq\": 2,\n              \"bagging_fraction\": 0.8,\n              \"bagging_seed\": 630,\n              \"metric\": 'mae',\n              \"verbosity\": -1,\n              'reg_alpha': 0.15,\n              'reg_lambda': 0.36,\n         }\nx_value_lgb, prediction_lgb_2 = train_model_lgb(params = lgb_params)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/LANL-Earthquake-Prediction/sample_submission.csv')\nsubmission['time_to_failure'] = prediction_lgb_2\nprint(submission.head())\nsubmission.to_csv('30_lgbm_2.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# NN"},{"metadata":{"trusted":true},"cell_type":"code","source":"scaler = StandardScaler()\ntrain_columns = X_trainset.columns\n\nX_trainset[train_columns] = scaler.fit_transform(X_trainset[train_columns])\nX_testset[train_columns] = scaler.transform(X_testset[train_columns])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(input_dim=10):\n\n    model = keras.Sequential()\n    model.add(keras.layers.Dense(256, activation=\"relu\", use_bias=True, kernel_initializer='he_normal', input_dim=input_dim))\n    model.add(keras.layers.Dropout(0.5))\n    model.add(keras.layers.Dense(128, activation=\"relu\", use_bias=True, kernel_initializer='he_normal'))\n    model.add(keras.layers.Dropout(0.5))\n    model.add(keras.layers.Dense(96, activation=\"relu\", use_bias=True, kernel_initializer='he_normal'))\n    model.add(keras.layers.Dropout(0.5))\n    model.add(keras.layers.Dense(1, activation=\"linear\"))\n\n    optimizer = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False) #'rmsprop'\n    model.compile(optimizer=optimizer,loss='mae', metrics=['accuracy'])\n    return model\n\npatience = 50\ncall_ES = keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=patience, verbose=1, mode='auto', baseline=None, restore_best_weights=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nn_fold = 5\nfolds = KFold(n_splits=n_fold, shuffle=True, random_state=630)\n\nNN_oof = np.zeros(len(X_trainset))\ntrain_score = []\nfold_idxs = []\n\nNN_predictions = np.zeros(len(X_testset))\n\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(X_trainset,y_trainset.values)):\n    strLog = \"fold {}\".format(fold_)\n    print(strLog)\n    fold_idxs.append(val_idx)\n    \n    X_tr, X_val = X_trainset[train_columns].iloc[trn_idx], X_trainset[train_columns].iloc[val_idx]\n    y_tr, y_val = y_trainset.iloc[trn_idx], y_trainset.iloc[val_idx]\n    model = create_model(X_trainset.shape[-1])\n    model.fit(X_tr, y_tr, epochs=500, batch_size=32, verbose=0, callbacks=[call_ES,], validation_data=[X_val, y_val]) #\n    \n    NN_oof[val_idx] = model.predict(X_val)[:,0]\n    \n    NN_predictions += model.predict(X_testset[train_columns])[:,0] / folds.n_splits\n    history = model.history.history\n    tr_loss = history[\"loss\"]\n    val_loss = history[\"val_loss\"]\n    print(f\"loss: {tr_loss[-patience]:.3f} | val_loss: {val_loss[-patience]:.3f} | diff: {val_loss[-patience]-tr_loss[-patience]:.3f}\")\n    train_score.append(tr_loss[-patience])\n#     break\n    \ncv_score = mean_absolute_error(y_trainset, NN_oof)\nprint(f\"After {n_fold} test_CV = {cv_score:.3f} | train_CV = {np.mean(train_score):.3f} | {cv_score-np.mean(train_score):.3f}\", end=\" \")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/LANL-Earthquake-Prediction/sample_submission.csv')\nsubmission[\"time_to_failure\"] = NN_predictions\nsubmission.to_csv('30_nn.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CAT"},{"metadata":{"trusted":true},"cell_type":"code","source":"num_folds = 5\nk_folds = KFold(n_splits=num_folds, shuffle=True, random_state=630)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model_cat(X=X_trainset, X_testset=X_testset, y=y_trainset, params=None, k_folds=k_folds, model=None):\n    train_columns = X.columns\n    \n    x_values = np.zeros(len(X))\n    prediction = np.zeros(len(X_testset))\n    scores = []\n    feature_importance = pd.DataFrame()\n\n    #run model\n    for fold_n, (trn_idx, val_idx) in enumerate(k_folds.split(X,y.values)):\n        print('Fold', fold_n, 'started at', time.ctime())\n\n        X_tr, X_val = X_trainset[train_columns].iloc[trn_idx], X_trainset[train_columns].iloc[val_idx]\n        y_tr, y_val = y_trainset.iloc[trn_idx], y_trainset.iloc[val_idx]\n\n        model = CatBoostRegressor(n_estimators=25000, eval_metric='MAE',\n                                  random_seed=630, l2_leaf_reg=3,\n                                  verbose=-1, objective=\"MAE\", loss_function=\"MAE\", \n                                  boosting_type=\"Ordered\")\n        model.fit(X_tr, \n                  y_tr, \n                  eval_set=[(X_val, y_val)], \n                  verbose=2500, \n                  early_stopping_rounds=250)\n        x_values[val_idx] = model.predict(X_val)\n        \n        y_pred_valid = model.predict(X_val)\n        y_pred = model.predict(X_testset)\n\n        x_values[val_idx] = y_pred_valid.reshape(-1,)\n        scores.append(mean_absolute_error(y_val, y_pred_valid))\n        \n        fold_importance = pd.DataFrame()\n        fold_importance[\"feature\"] = train_columns\n        fold_importance[\"importance\"] = model.feature_importances_\n        fold_importance[\"fold\"] = fold_n + 1\n        feature_importance = pd.concat([feature_importance, fold_importance], axis=0)\n        \n        prediction += y_pred\n        \n    prediction /= num_folds\n    cols = feature_importance[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(by=\"importance\", ascending=False)[:30].index\n    best_features = feature_importance.loc[feature_importance.feature.isin(cols)]\n\n    plt.figure(figsize=(16, 12));\n    sns.barplot(x=\"importance\", y=\"feature\", data=best_features.sort_values(by=\"importance\", ascending=False));\n    plt.title('CAT Features (avg over folds)');\n\n    print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))\n    return x_values, prediction","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nx_value_cat, prediction_cat = train_model_cat()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/LANL-Earthquake-Prediction/sample_submission.csv')\n\nsubmission[\"time_to_failure\"] = prediction_cat\nsubmission.to_csv('30_cat.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# XGB"},{"metadata":{"trusted":true},"cell_type":"code","source":"num_folds = 5\nk_folds = KFold(n_splits=num_folds, shuffle=True, random_state=630)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model_xgb(X=X_trainset, X_testset=X_testset, y=y_trainset, params=None, k_folds=k_folds, model=None):\n    columns = X.columns\n    \n    x_values = np.zeros(len(X))\n    prediction = np.zeros(len(X_testset))\n    scores = []\n    feature_importance = pd.DataFrame()\n    \n    for fold_n, (trainset_index, valid_set_index) in enumerate(k_folds.split(X)):\n        print('Fold', fold_n, 'started at', time.ctime())\n        X_train_per_fold, X_valid_per_fold = X[columns].iloc[trainset_index], X[columns].iloc[valid_set_index]\n        y_train_per_fold, y_valid_per_fold = y.iloc[trainset_index], y.iloc[valid_set_index]\n\n        train_data = xgb.DMatrix(data=X_train_per_fold, label=y_train_per_fold, feature_names=columns)\n        valid_data = xgb.DMatrix(data=X_valid_per_fold, label=y_valid_per_fold, feature_names=columns)\n\n        watchlist = [(train_data, 'train'), (valid_data, 'valid_data')]\n        model = xgb.train(dtrain=train_data, num_boost_round=25000, evals=watchlist, early_stopping_rounds=250, verbose_eval=2500, params=params)\n        y_pred_valid = model.predict(xgb.DMatrix(X_valid_per_fold, feature_names=X_trainset.columns), ntree_limit=model.best_ntree_limit)\n        y_pred = model.predict(xgb.DMatrix(X_testset, feature_names=X_trainset.columns), ntree_limit=model.best_ntree_limit)\n\n        x_values[valid_set_index] = y_pred_valid.reshape(-1,)\n        scores.append(mean_absolute_error(y_valid_per_fold, y_pred_valid))\n\n        prediction += y_pred\n       \n    prediction /= num_folds\n    print('CV mean score: {0:.4f}.'.format(mean_absolute_error(y, x_values)))\n    return x_values, prediction","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_params = {'eta': 0.01,\n              'max_depth': 6,\n              'subsample': 0.8,\n              'colsample_bytree': 0.8,\n              'colsample_bylevel': 0.8,\n              'colsample_bynode': 0.8,\n              'lambda': 0.1,\n              'alpha' : 0.1,\n              'objective': 'reg:linear',\n              'eval_metric': 'mae',\n              'silent': True,\n              'nthread': 4}\nx_value_xgb, prediction_xgb = train_model_xgb(params=xgb_params)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/LANL-Earthquake-Prediction/sample_submission.csv')\n\nsubmission[\"time_to_failure\"] = prediction_xgb\nsubmission.to_csv('30_xgb.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}