{"cells":[{"metadata":{},"cell_type":"markdown","source":"<pre>Inspired by Anton Enns's Kernel\nhttps://www.kaggle.com/tocha4/lanl-master-s-approach</pre>"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport scipy as sc\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport gc\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nimport os\nfrom tqdm import tqdm_notebook\nimport datetime\nimport time\nimport random\nfrom joblib import Parallel, delayed\n\nfrom catboost import Pool, CatBoostRegressor\n\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import KFold\nfrom sklearn.feature_selection import RFECV, SelectFromModel","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This is NN Model creation\nfrom keras.models import *\nfrom keras.optimizers import *\nfrom keras.regularizers import *\nfrom keras.layers import * # Keras is the most friendly Neural Network library, this Kernel use a lot of layers classes\nfrom keras import backend as K # The backend give us access to tensorflow operations and allow us to create the Attention class\nfrom keras.callbacks import *","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"print(os.listdir('../input/'))\ntrain_X_0 = pd.read_csv(\"../input/lanl-master-s-features-creating-0/train_X_features_865.csv\")\ntrain_X_1 = pd.read_csv(\"../input/lanl-master-s-features-creating-1/train_X_features_865.csv\")\ny_0 = pd.read_csv(\"../input/lanl-master-s-features-creating-0/train_y.csv\", index_col=False,  header=None)\ny_1 = pd.read_csv(\"../input/lanl-master-s-features-creating-1/train_y.csv\", index_col=False,  header=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X = pd.concat([train_X_0, train_X_1], axis=0)\ntrain_X = train_X.reset_index(drop=True)\nprint(train_X.shape)\ntrain_X.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = pd.concat([y_0, y_1], axis=0)\ny = y.reset_index(drop=True)\ny[0].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_y = pd.Series(y[0].values)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"test_X = pd.read_csv(\"../input/lanl-master-s-features-creating-0/test_X_features_10.csv\")\n# del X[\"seg_id\"], test_X[\"seg_id\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scaler = MinMaxScaler()\ntrain_columns = train_X.columns\n\ntrain_X[train_columns] = scaler.fit_transform(train_X[train_columns])\ntest_X[train_columns] = scaler.transform(test_X[train_columns])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<pre>CAT BOOST Algorithm</pre>"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_columns = train_X.columns\nn_fold = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfolds = KFold(n_splits=n_fold, shuffle = True, random_state=42)\n\noof = np.zeros(len(train_X))\ntrain_score = []\nfold_idxs = []\n# if PREDICTION: \npredictions = np.zeros(len(test_X))\n\nfeature_importance_df = pd.DataFrame()\n#run model\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train_X,train_y.values)):\n    strLog = \"fold {}\".format(fold_)\n    print(strLog)\n    fold_idxs.append(val_idx)\n\n    X_tr, X_val = train_X[train_columns].iloc[trn_idx], train_X[train_columns].iloc[val_idx]\n    y_tr, y_val = train_y.iloc[trn_idx], train_y.iloc[val_idx]\n\n    model = CatBoostRegressor(n_estimators=25000, verbose=-1, objective=\"MAE\", loss_function=\"MAE\", boosting_type=\"Ordered\", task_type=\"GPU\")\n    model.fit(X_tr, \n              y_tr, \n              eval_set=[(X_val, y_val)], \n#               eval_metric='mae',\n              verbose=2500, \n              early_stopping_rounds=500)\n    oof[val_idx] = model.predict(X_val)\n\n    #feature importance\n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"Feature\"] = train_columns\n    fold_importance_df[\"importance\"] = model.feature_importances_[:len(train_columns)]\n    fold_importance_df[\"fold\"] = fold_ + 1\n    feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n    #predictions\n#     if PREDICTION:\n\n    predictions += model.predict(test_X[train_columns]) / folds.n_splits\n    train_score.append(model.best_score_['learn'][\"MAE\"])\n\ncv_score = mean_absolute_error(train_y, oof)\nprint(f\"After {n_fold} test_CV = {cv_score:.3f} | train_CV = {np.mean(train_score):.3f} | {cv_score-np.mean(train_score):.3f}\", end=\" \")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"today = str(datetime.date.today())\nsubmission = pd.read_csv('../input/LANL-Earthquake-Prediction/sample_submission.csv')\n\nsubmission[\"time_to_failure\"] = predictions\nsubmission.to_csv(f'CatBoost_{today}_test_{cv_score:.3f}_train_{np.mean(train_score):.3f}.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<pre>Keras Neural Network</pre>"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(input_dim=10):\n    model = Sequential()\n    model.add(Dense(256, activation=\"relu\",input_dim=input_dim))\n    model.add(Dropout(0.5))\n    model.add(Dense(128, activation=\"relu\"))\n    model.add(Dropout(0.5))\n    model.add(Dense(64, activation=\"relu\"))\n    model.add(Dropout(0.5))\n    model.add(Dense(1, activation=\"linear\"))\n \n    opt = adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)\n    # Compile model\n    model.compile(\n        loss='mae',\n        optimizer=opt,\n    )\n    return model\n\npatience = 50\ncall_ES = EarlyStopping(monitor='val_loss', min_delta=0, patience=patience, verbose=1, mode='auto', baseline=None, restore_best_weights=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# n_fold = 5\nfolds = KFold(n_splits=n_fold, shuffle = True, random_state=42)\n\nNN_oof = np.zeros(len(train_X))\ntrain_score = []\nfold_idxs = []\n\nNN_predictions = np.zeros(len(test_X))\n\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train_X,train_y.values)):\n    strLog = \"fold {}\".format(fold_)\n    print(strLog)\n    fold_idxs.append(val_idx)\n    \n    X_tr, X_val = train_X[train_columns].iloc[trn_idx], train_X[train_columns].iloc[val_idx]\n    y_tr, y_val = train_y.iloc[trn_idx], train_y.iloc[val_idx]\n    model = create_model(train_X.shape[-1])\n    model.fit(X_tr, y_tr, epochs=500, batch_size=32, verbose=0, callbacks=[call_ES,], validation_data=[X_val, y_val]) #\n    \n    NN_oof[val_idx] = model.predict(X_val)[:,0]\n    \n    NN_predictions += model.predict(test_X[train_columns])[:,0] / folds.n_splits\n    history = model.history.history\n    tr_loss = history[\"loss\"]\n    val_loss = history[\"val_loss\"]\n    print(f\"loss: {tr_loss[-patience]:.3f} | val_loss: {val_loss[-patience]:.3f} | diff: {val_loss[-patience]-tr_loss[-patience]:.3f}\")\n    train_score.append(tr_loss[-patience])\n#     break\n    \ncv_score = mean_absolute_error(train_y, NN_oof)\nprint(f\"After {n_fold} test_CV = {cv_score:.3f} | train_CV = {np.mean(train_score):.3f} | {cv_score-np.mean(train_score):.3f}\", end=\" \")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"today = str(datetime.date.today())\nsubmission = pd.read_csv('../input/LANL-Earthquake-Prediction/sample_submission.csv')\n\nsubmission[\"time_to_failure\"] = NN_predictions\nsubmission.to_csv(f'NN_{today}_test_{cv_score:.3f}_train_{np.mean(train_score):.3f}.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Final Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"Scirpus_prediction = pd.read_csv(\"../input/andrews-new-script-plus-a-genetic-program-model/gpI.csv\")\nScirpus_prediction.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"today = str(datetime.date.today())\nsubmission = pd.read_csv('../input/LANL-Earthquake-Prediction/sample_submission.csv')\n\nsubmission[\"time_to_failure\"] = (predictions+NN_predictions+Scirpus_prediction.time_to_failure.values)/3\nsubmission.to_csv(f'FINAL_{today}_submission.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}