{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport time\nimport sys\nimport logging\nimport datetime\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom scipy import stats\nfrom scipy.signal import hann\nfrom tqdm import tqdm_notebook\nimport matplotlib.pyplot as plt\nfrom scipy.signal import hilbert\nfrom scipy.signal import convolve\nfrom sklearn.svm import NuSVR, SVR\nfrom catboost import CatBoostRegressor\nfrom sklearn.kernel_ridge import KernelRidge\nfrom sklearn.linear_model import ElasticNet, Lasso,  BayesianRidge, LassoLarsIC\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.ensemble import RandomForestRegressor,  GradientBoostingRegressor\nfrom sklearn.model_selection import KFold,StratifiedKFold, RepeatedKFold\nfrom sklearn.model_selection import cross_val_score, train_test_split, cross_val_predict\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.preprocessing import RobustScaler\nfrom sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone\n\nwarnings.filterwarnings(\"ignore\")\nimport os\nprint(os.listdir(\"../input\"))","execution_count":1,"outputs":[{"output_type":"stream","text":"['earthquake-data-overlapping4', 'LANL-Earthquake-Prediction']\n","name":"stdout"}]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_X = pd.read_csv('../input/earthquake-data-overlapping4/train_X.csv')\ntrain_y = pd.read_csv('../input/earthquake-data-overlapping4/train_y.csv')\ntest_X = pd.read_csv('../input/earthquake-data-overlapping4/test_X.csv')\ntrain_X.head()","execution_count":2,"outputs":[{"output_type":"execute_result","execution_count":2,"data":{"text/plain":"       mean           ...             abs_max_roll_mean_10000\n0  1.700578           ...                            1.893822\n1  1.687215           ...                            1.831696\n2  1.689548           ...                            1.831696\n3  1.681095           ...                            1.831696\n4  1.698655           ...                            1.831696\n\n[5 rows x 253 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>mean</th>\n      <th>std</th>\n      <th>max</th>\n      <th>Rmean</th>\n      <th>Rstd</th>\n      <th>Rmax</th>\n      <th>Rmin</th>\n      <th>Imean</th>\n      <th>Istd</th>\n      <th>Imax</th>\n      <th>Imin</th>\n      <th>Rmean_last_5000</th>\n      <th>Rstd__last_5000</th>\n      <th>Rmax_last_5000</th>\n      <th>Rmin_last_5000</th>\n      <th>Rmean_last_15000</th>\n      <th>Rstd_last_15000</th>\n      <th>Rmax_last_15000</th>\n      <th>Rmin_last_15000</th>\n      <th>mean_change_abs</th>\n      <th>mean_change_rate</th>\n      <th>abs_max</th>\n      <th>abs_min</th>\n      <th>std_first_50000</th>\n      <th>std_last_50000</th>\n      <th>std_first_10000</th>\n      <th>std_last_10000</th>\n      <th>avg_first_50000</th>\n      <th>avg_last_50000</th>\n      <th>avg_first_10000</th>\n      <th>avg_last_10000</th>\n      <th>min_first_50000</th>\n      <th>min_last_50000</th>\n      <th>min_first_10000</th>\n      <th>min_last_10000</th>\n      <th>max_first_50000</th>\n      <th>max_last_50000</th>\n      <th>max_first_10000</th>\n      <th>max_last_10000</th>\n      <th>max_to_min_diff</th>\n      <th>...</th>\n      <th>q01_roll_std_5000</th>\n      <th>q05_roll_std_5000</th>\n      <th>q95_roll_std_5000</th>\n      <th>q99_roll_std_5000</th>\n      <th>av_change_abs_roll_std_5000</th>\n      <th>av_change_rate_roll_std_5000</th>\n      <th>abs_max_roll_std_5000</th>\n      <th>ave_roll_mean_5000</th>\n      <th>std_roll_mean_5000</th>\n      <th>max_roll_mean_5000</th>\n      <th>min_roll_mean_5000</th>\n      <th>q01_roll_mean_5000</th>\n      <th>q05_roll_mean_5000</th>\n      <th>q95_roll_mean_5000</th>\n      <th>q99_roll_mean_5000</th>\n      <th>av_change_abs_roll_mean_5000</th>\n      <th>av_change_rate_roll_mean_5000</th>\n      <th>abs_max_roll_mean_5000</th>\n      <th>ave_roll_std_10000</th>\n      <th>std_roll_std_10000</th>\n      <th>max_roll_std_10000</th>\n      <th>min_roll_std_10000</th>\n      <th>q01_roll_std_10000</th>\n      <th>q05_roll_std_10000</th>\n      <th>q95_roll_std_10000</th>\n      <th>q99_roll_std_10000</th>\n      <th>av_change_abs_roll_std_10000</th>\n      <th>av_change_rate_roll_std_10000</th>\n      <th>abs_max_roll_std_10000</th>\n      <th>ave_roll_mean_10000</th>\n      <th>std_roll_mean_10000</th>\n      <th>max_roll_mean_10000</th>\n      <th>min_roll_mean_10000</th>\n      <th>q01_roll_mean_10000</th>\n      <th>q05_roll_mean_10000</th>\n      <th>q95_roll_mean_10000</th>\n      <th>q99_roll_mean_10000</th>\n      <th>av_change_abs_roll_mean_10000</th>\n      <th>av_change_rate_roll_mean_10000</th>\n      <th>abs_max_roll_mean_10000</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.700578</td>\n      <td>0.633686</td>\n      <td>4.653960</td>\n      <td>2.564949</td>\n      <td>680.894175</td>\n      <td>255058.322797</td>\n      <td>-2408.516087</td>\n      <td>1.455192e-16</td>\n      <td>174.151534</td>\n      <td>3239.416833</td>\n      <td>-3239.416833</td>\n      <td>-0.654161</td>\n      <td>267.514872</td>\n      <td>3454.906692</td>\n      <td>-2408.516087</td>\n      <td>-0.830288</td>\n      <td>281.033541</td>\n      <td>3454.906692</td>\n      <td>-2408.516087</td>\n      <td>-1.709978e-05</td>\n      <td>74843.221402</td>\n      <td>4.653960</td>\n      <td>0.0</td>\n      <td>0.672039</td>\n      <td>0.604280</td>\n      <td>0.782109</td>\n      <td>0.626411</td>\n      <td>1.739996</td>\n      <td>1.632751</td>\n      <td>1.893847</td>\n      <td>1.670074</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>4.653960</td>\n      <td>3.433987</td>\n      <td>4.653960</td>\n      <td>3.295837</td>\n      <td>4.653960</td>\n      <td>...</td>\n      <td>0.529758</td>\n      <td>0.549506</td>\n      <td>0.693419</td>\n      <td>0.878553</td>\n      <td>-1.962850e-06</td>\n      <td>72228.057274</td>\n      <td>0.891800</td>\n      <td>1.696024</td>\n      <td>0.074840</td>\n      <td>2.068372</td>\n      <td>1.567778</td>\n      <td>1.574001</td>\n      <td>1.586487</td>\n      <td>1.794616</td>\n      <td>2.041310</td>\n      <td>-2.761095e-06</td>\n      <td>72228.057274</td>\n      <td>2.068372</td>\n      <td>0.620969</td>\n      <td>0.036920</td>\n      <td>0.783651</td>\n      <td>0.548856</td>\n      <td>0.553488</td>\n      <td>0.567385</td>\n      <td>0.668806</td>\n      <td>0.777202</td>\n      <td>-1.112140e-06</td>\n      <td>69766.232090</td>\n      <td>0.783651</td>\n      <td>1.692042</td>\n      <td>0.056575</td>\n      <td>1.893822</td>\n      <td>1.573180</td>\n      <td>1.578145</td>\n      <td>1.587725</td>\n      <td>1.763602</td>\n      <td>1.879227</td>\n      <td>-1.598609e-06</td>\n      <td>69766.232090</td>\n      <td>1.893822</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.687215</td>\n      <td>0.631170</td>\n      <td>5.204007</td>\n      <td>1.945910</td>\n      <td>675.975105</td>\n      <td>253066.608763</td>\n      <td>-2328.675199</td>\n      <td>1.576457e-16</td>\n      <td>172.455209</td>\n      <td>2535.094444</td>\n      <td>-2535.094444</td>\n      <td>-2.370291</td>\n      <td>269.970140</td>\n      <td>2423.243075</td>\n      <td>-2328.675199</td>\n      <td>-1.217297</td>\n      <td>280.304439</td>\n      <td>2423.243075</td>\n      <td>-2328.675199</td>\n      <td>-2.243170e-06</td>\n      <td>75068.853673</td>\n      <td>5.204007</td>\n      <td>0.0</td>\n      <td>0.620690</td>\n      <td>0.665901</td>\n      <td>0.646171</td>\n      <td>0.675836</td>\n      <td>1.716995</td>\n      <td>1.690943</td>\n      <td>1.693608</td>\n      <td>1.702647</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>4.043051</td>\n      <td>5.204007</td>\n      <td>3.713572</td>\n      <td>3.784190</td>\n      <td>5.204007</td>\n      <td>...</td>\n      <td>0.529758</td>\n      <td>0.547089</td>\n      <td>0.706186</td>\n      <td>0.903979</td>\n      <td>5.646436e-07</td>\n      <td>72504.404671</td>\n      <td>0.918148</td>\n      <td>1.686305</td>\n      <td>0.077480</td>\n      <td>1.991082</td>\n      <td>1.567778</td>\n      <td>1.574001</td>\n      <td>1.586487</td>\n      <td>1.816882</td>\n      <td>1.984399</td>\n      <td>4.394670e-07</td>\n      <td>72504.404671</td>\n      <td>1.991082</td>\n      <td>0.623624</td>\n      <td>0.051871</td>\n      <td>0.797961</td>\n      <td>0.548856</td>\n      <td>0.553488</td>\n      <td>0.567268</td>\n      <td>0.785174</td>\n      <td>0.794373</td>\n      <td>2.119372e-07</td>\n      <td>69994.498756</td>\n      <td>0.797961</td>\n      <td>1.685211</td>\n      <td>0.061058</td>\n      <td>1.831696</td>\n      <td>1.573180</td>\n      <td>1.578145</td>\n      <td>1.587725</td>\n      <td>1.808517</td>\n      <td>1.826149</td>\n      <td>6.460775e-08</td>\n      <td>69994.498756</td>\n      <td>1.831696</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1.689548</td>\n      <td>0.645775</td>\n      <td>5.204007</td>\n      <td>2.079442</td>\n      <td>677.862304</td>\n      <td>253428.096553</td>\n      <td>-2562.230574</td>\n      <td>-1.576457e-16</td>\n      <td>176.639855</td>\n      <td>3228.149028</td>\n      <td>-3228.149028</td>\n      <td>1.211027</td>\n      <td>289.350010</td>\n      <td>3407.107638</td>\n      <td>-2562.230574</td>\n      <td>1.729464</td>\n      <td>289.262755</td>\n      <td>3407.107638</td>\n      <td>-2562.230574</td>\n      <td>-4.132616e-11</td>\n      <td>75056.126424</td>\n      <td>5.204007</td>\n      <td>0.0</td>\n      <td>0.603759</td>\n      <td>0.679278</td>\n      <td>0.549523</td>\n      <td>0.600103</td>\n      <td>1.691658</td>\n      <td>1.708584</td>\n      <td>1.726333</td>\n      <td>1.667097</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>3.970292</td>\n      <td>5.030438</td>\n      <td>3.258096</td>\n      <td>3.663562</td>\n      <td>5.204007</td>\n      <td>...</td>\n      <td>0.529758</td>\n      <td>0.554342</td>\n      <td>0.856148</td>\n      <td>0.903979</td>\n      <td>-8.374987e-08</td>\n      <td>72659.749225</td>\n      <td>0.918148</td>\n      <td>1.689388</td>\n      <td>0.081948</td>\n      <td>1.991082</td>\n      <td>1.567778</td>\n      <td>1.574001</td>\n      <td>1.586487</td>\n      <td>1.851968</td>\n      <td>1.984399</td>\n      <td>-5.674733e-07</td>\n      <td>72659.749225</td>\n      <td>1.991082</td>\n      <td>0.644311</td>\n      <td>0.063337</td>\n      <td>0.797961</td>\n      <td>0.549518</td>\n      <td>0.558957</td>\n      <td>0.583459</td>\n      <td>0.788207</td>\n      <td>0.794373</td>\n      <td>3.612787e-07</td>\n      <td>70241.279250</td>\n      <td>0.797961</td>\n      <td>1.688871</td>\n      <td>0.063925</td>\n      <td>1.831696</td>\n      <td>1.573180</td>\n      <td>1.578145</td>\n      <td>1.587725</td>\n      <td>1.808517</td>\n      <td>1.826149</td>\n      <td>-4.231743e-07</td>\n      <td>70241.279250</td>\n      <td>1.831696</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1.681095</td>\n      <td>0.647735</td>\n      <td>5.204007</td>\n      <td>1.609438</td>\n      <td>675.114788</td>\n      <td>252142.605786</td>\n      <td>-2923.794506</td>\n      <td>-1.637090e-16</td>\n      <td>176.069659</td>\n      <td>3361.815243</td>\n      <td>-3361.815243</td>\n      <td>-2.515107</td>\n      <td>309.301760</td>\n      <td>2927.914965</td>\n      <td>-2923.794506</td>\n      <td>-1.065583</td>\n      <td>293.424315</td>\n      <td>2927.914965</td>\n      <td>-2923.794506</td>\n      <td>-6.108616e-06</td>\n      <td>75014.658586</td>\n      <td>5.204007</td>\n      <td>0.0</td>\n      <td>0.641665</td>\n      <td>0.616641</td>\n      <td>0.592504</td>\n      <td>0.613068</td>\n      <td>1.663627</td>\n      <td>1.673473</td>\n      <td>1.654633</td>\n      <td>1.702923</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>5.204007</td>\n      <td>4.077538</td>\n      <td>3.433987</td>\n      <td>4.077538</td>\n      <td>5.204007</td>\n      <td>...</td>\n      <td>0.540541</td>\n      <td>0.555960</td>\n      <td>0.856148</td>\n      <td>0.903979</td>\n      <td>4.540788e-07</td>\n      <td>72517.200235</td>\n      <td>0.918148</td>\n      <td>1.680921</td>\n      <td>0.084756</td>\n      <td>1.991082</td>\n      <td>1.567778</td>\n      <td>1.574001</td>\n      <td>1.585027</td>\n      <td>1.851968</td>\n      <td>1.984399</td>\n      <td>4.925283e-07</td>\n      <td>72517.200235</td>\n      <td>1.991082</td>\n      <td>0.644449</td>\n      <td>0.063201</td>\n      <td>0.797961</td>\n      <td>0.565768</td>\n      <td>0.568237</td>\n      <td>0.580539</td>\n      <td>0.788207</td>\n      <td>0.794373</td>\n      <td>1.468546e-07</td>\n      <td>70089.806079</td>\n      <td>0.797961</td>\n      <td>1.680148</td>\n      <td>0.063915</td>\n      <td>1.831696</td>\n      <td>1.573180</td>\n      <td>1.578145</td>\n      <td>1.587725</td>\n      <td>1.808517</td>\n      <td>1.826149</td>\n      <td>3.450187e-07</td>\n      <td>70089.806079</td>\n      <td>1.831696</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1.698655</td>\n      <td>0.653879</td>\n      <td>5.204007</td>\n      <td>1.791759</td>\n      <td>681.772035</td>\n      <td>254770.876186</td>\n      <td>-2776.657547</td>\n      <td>8.488617e-17</td>\n      <td>179.008795</td>\n      <td>3512.836458</td>\n      <td>-3512.836458</td>\n      <td>0.425912</td>\n      <td>313.191541</td>\n      <td>3939.342086</td>\n      <td>-2776.657547</td>\n      <td>1.843800</td>\n      <td>294.725990</td>\n      <td>3939.342086</td>\n      <td>-2776.657547</td>\n      <td>-1.430521e-11</td>\n      <td>74929.080393</td>\n      <td>5.204007</td>\n      <td>0.0</td>\n      <td>0.676743</td>\n      <td>0.629306</td>\n      <td>0.599032</td>\n      <td>0.584076</td>\n      <td>1.716170</td>\n      <td>1.697456</td>\n      <td>1.674099</td>\n      <td>1.687141</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>5.204007</td>\n      <td>4.753590</td>\n      <td>3.401197</td>\n      <td>3.401197</td>\n      <td>5.204007</td>\n      <td>...</td>\n      <td>0.542711</td>\n      <td>0.555965</td>\n      <td>0.856148</td>\n      <td>0.903979</td>\n      <td>-1.722217e-07</td>\n      <td>72398.857988</td>\n      <td>0.918148</td>\n      <td>1.699388</td>\n      <td>0.082046</td>\n      <td>1.991082</td>\n      <td>1.577189</td>\n      <td>1.587279</td>\n      <td>1.600291</td>\n      <td>1.851968</td>\n      <td>1.984399</td>\n      <td>-2.605253e-07</td>\n      <td>72398.857988</td>\n      <td>1.991082</td>\n      <td>0.654010</td>\n      <td>0.061541</td>\n      <td>0.797961</td>\n      <td>0.563540</td>\n      <td>0.567927</td>\n      <td>0.580354</td>\n      <td>0.788207</td>\n      <td>0.794373</td>\n      <td>-1.068176e-07</td>\n      <td>69869.352990</td>\n      <td>0.797961</td>\n      <td>1.699477</td>\n      <td>0.055868</td>\n      <td>1.831696</td>\n      <td>1.590703</td>\n      <td>1.593973</td>\n      <td>1.606799</td>\n      <td>1.808517</td>\n      <td>1.826149</td>\n      <td>9.318692e-08</td>\n      <td>69869.352990</td>\n      <td>1.831696</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"metadata":{},"cell_type":"markdown","source":"對訓練資料進行尺度縮放的處理"},{"metadata":{"trusted":true},"cell_type":"code","source":"scaler = StandardScaler()\nscaler.fit(train_X)\nscaled_train_X = pd.DataFrame(scaler.transform(train_X), columns=train_X.columns)\n#scaled_train_X = train_X","execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scaled_test_X = pd.DataFrame(scaler.transform(test_X), columns=test_X.columns)\n#scaled_test_X = test_X","execution_count":4,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scaled_train_X.head(), scaled_test_X.head()","execution_count":5,"outputs":[{"output_type":"execute_result","execution_count":5,"data":{"text/plain":"(       mean           ...             abs_max_roll_mean_10000\n 0  0.653634           ...                            0.226685\n 1  0.440375           ...                           -0.045400\n 2  0.477612           ...                           -0.045400\n 3  0.342719           ...                           -0.045400\n 4  0.622946           ...                           -0.045400\n \n [5 rows x 253 columns],\n        mean           ...             abs_max_roll_mean_10000\n 0 -0.344491           ...                           -0.313423\n 1 -0.835153           ...                           -0.138089\n 2  0.200758           ...                            0.483292\n 3 -0.526375           ...                           -0.417852\n 4 -1.059244           ...                           -0.044151\n \n [5 rows x 253 columns])"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"scaled_train_X.shape, scaled_test_X.shape","execution_count":6,"outputs":[{"output_type":"execute_result","execution_count":6,"data":{"text/plain":"((16761, 253), (2624, 253))"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X_mean = np.mean(scaled_train_X, axis=0)\ntrain_X_mean.head()","execution_count":7,"outputs":[{"output_type":"execute_result","execution_count":7,"data":{"text/plain":"mean     1.466831e-15\nstd      7.358564e-16\nmax     -2.618009e-16\nRmean    4.673621e-17\nRstd    -9.704731e-16\ndtype: float64"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_X_mean = np.mean(scaled_test_X, axis=0)\ntest_X_mean.head()","execution_count":8,"outputs":[{"output_type":"execute_result","execution_count":8,"data":{"text/plain":"mean    -0.842562\nstd      0.284385\nmax      0.039534\nRmean   -0.080621\nRstd    -0.688147\ndtype: float64"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"c = 0\nfor _, i in (train_X_mean - test_X_mean).abs().sort_values().iteritems():\n    print(i, _)\n    if _ == 'std_roll_std_50':\n        print(c - 9)\n    c += 1","execution_count":10,"outputs":[{"output_type":"stream","text":"0.0 q01\n0.0 q001\n0.0 abs_q01\n0.0 count_big\n0.0 min_last_50000\n0.0 min_first_10000\n0.0 min_last_10000\n0.0 min_first_50000\n0.0 abs_min\n9.375776612797745e-05 av_change_abs_roll_std_10000\n0.0008578532076590063 av_change_abs_roll_mean_10000\n0.0012692648292819766 q95_roll_mean_50\n0.0019303365173806303 av_change_abs_roll_mean_100\n0.0024279128238885177 q99_roll_mean_1000\n0.002532143874549431 q95_roll_mean_100\n0.0033452644099660283 max_last_10000\n0.0035651638731541905 av_change_abs_roll_std_10\n0.004243453021396381 av_change_abs_roll_std_5\n0.004369214626621779 mean_change_abs\n0.0056125779592634175 max_first_10000\n0.006762595248573897 av_change_abs_roll_std_1000\n0.006863464433817255 av_change_abs_roll_mean_50\n0.007403280795349504 q99_roll_std_10\n0.0077433109837537 mean_change_rate\n0.008295769251100605 av_change_abs_roll_std_500\n0.008634430470347962 av_change_abs_roll_mean_5\n0.008849343691339592 av_change_rate_roll_std_50\n0.008901853165883033 av_change_rate_roll_mean_5\n0.009079429053427148 av_change_rate_roll_mean_10000\n0.0090951425222213 av_change_rate_roll_mean_50\n0.009112817439754563 av_change_rate_roll_std_10000\n0.009189138717078072 av_change_rate_roll_std_5\n0.00959078353876873 av_change_rate_roll_std_100\n0.009649057606376342 av_change_rate_roll_mean_100\n0.010687407330870652 av_change_rate_roll_mean_10\n0.010991946593080698 av_change_rate_roll_mean_500\n0.0109994538623052 av_change_rate_roll_std_500\n0.011083937093292279 av_change_rate_roll_std_10\n0.01117732977835759 av_change_abs_roll_mean_5000\n0.011247462580155111 av_change_rate_roll_std_1000\n0.011251769887490822 av_change_rate_roll_mean_1000\n0.0117624637629543 av_change_rate_roll_std_5000\n0.01176605271310174 av_change_rate_roll_mean_5000\n0.012820507253685392 Imean\n0.01393575766739337 av_change_abs_roll_std_5000\n0.014156127505205581 av_change_abs_roll_mean_500\n0.0143295956063366 mean_change_rate_first_10000\n0.014817018439677445 min_roll_std_5\n0.014852860252948892 av_change_abs_roll_mean_1000\n0.015131251780447882 mean_change_rate_last_50000\n0.016066908507425766 Rmean_last_15000\n0.01784541363865257 av_change_abs_roll_mean_10\n0.02025674359435788 mean_change_rate_first_50000\n0.022713523359470546 abs_max_roll_std_10\n0.022713523359470546 max_roll_std_10\n0.025559804953091503 q99_roll_mean_500\n0.02578478152668598 std_roll_std_1000\n0.026771462981520746 max_first_50000\n0.02950113410046658 max_last_50000\n0.03002100462249221 av_change_abs_roll_std_50\n0.03378546074154727 trend\n0.03378546074154727 abs_trend\n0.03546090791439615 av_change_abs_roll_std_100\n0.035589688382461765 q99_roll_mean_5\n0.038099436633158755 q99_roll_std_50\n0.038230490753267184 max_roll_mean_1000\n0.038230490753267184 abs_max_roll_mean_1000\n0.038567356241263895 abs_max_roll_mean_100\n0.038567356241263895 max_roll_mean_100\n0.03953432997296425 max\n0.03953432997296425 abs_max\n0.03953432997296425 max_to_min_diff\n0.03997049823411585 q95_roll_std_50\n0.04022759681428748 max_roll_mean_10\n0.04022759681428748 abs_max_roll_mean_10\n0.04023787261715593 q95_roll_mean_500\n0.04104187708571164 abs_max_roll_mean_5\n0.04104187708571164 max_roll_mean_5\n0.041297892310712216 abs_max_roll_std_5\n0.041297892310712216 max_roll_std_5\n0.04261677138076923 max_roll_mean_50\n0.04261677138076923 abs_max_roll_mean_50\n0.04291383154638998 abs_q99\n0.04291383154638998 q99\n0.04344004745887479 mean_change_rate_last_10000\n0.043584665894527926 q99_roll_mean_10\n0.04430335071321909 Rmean_last_5000\n0.04463796598465848 q999\n0.04583558564825784 abs_max_roll_mean_500\n0.04583558564825784 max_roll_mean_500\n0.04592998738577915 q99_roll_mean_100\n0.048558281553201695 std_roll_std_10000\n0.05074899269206343 q99_roll_mean_50\n0.05306819766920754 std_roll_std_5000\n0.053599997904941674 abs_q95\n0.053599997904941674 q95\n0.05416555762398919 q95_roll_mean_10\n0.05648031845245055 q95_roll_std_100\n0.05711050641459597 q99_roll_std_5\n0.05922018245458933 q95_roll_std_10\n0.0618330464375494 abs_max_roll_mean_5000\n0.0618330464375494 max_roll_mean_5000\n0.06578929016125165 q95_roll_mean_5\n0.07390370326448858 q99_roll_std_100\n0.07887796420312676 max_roll_std_50\n0.07887796420312676 abs_max_roll_std_50\n0.08016702320685384 q99_roll_mean_5000\n0.08062101547808273 Rmean\n0.0847672509299746 q95_roll_mean_1000\n0.11010868644808636 std_roll_std_5\n0.11698424978282602 std_roll_std_500\n0.12393735552900298 std_roll_mean_10000\n0.12939115943228904 Imin\n0.12939115943228907 Imax\n0.1297900285659843 q95_roll_std_500\n0.1300091970863681 max_roll_std_100\n0.1300091970863681 abs_max_roll_std_100\n0.13028138932912756 abs_max_roll_std_5000\n0.13028138932912756 max_roll_std_5000\n0.13883655749651447 min_roll_std_10\n0.1398057231966404 q99_roll_std_5000\n0.14034440402706141 q95_roll_std_5\n0.14194189942484323 max_roll_std_10000\n0.14194189942484323 abs_max_roll_std_10000\n0.14550177438371775 q99_roll_std_10000\n0.15027480486597125 abs_max_roll_mean_10000\n0.15027480486597125 max_roll_mean_10000\n0.15072009333136366 std_roll_mean_5000\n0.15103856975485577 Rmax_last_15000\n0.15126781495534 Rmax_last_5000\n0.15619430463386114 Rmin_last_15000\n0.15619430463386116 Rmin\n0.15640943161221707 Rmin_last_5000\n0.1599768545413991 q99_roll_std_500\n0.16145580073398044 q99_roll_mean_10000\n0.16316139225907586 abs_max_roll_std_1000\n0.16316139225907586 max_roll_std_1000\n0.16401077563742472 Rstd_last_15000\n0.16409176426047803 q99_roll_std_1000\n0.16454130271221754 q95_roll_std_1000\n0.16764255353582025 abs_max_roll_std_500\n0.16764255353582025 max_roll_std_500\n0.17758306749913075 q95_roll_std_10000\n0.1784852660367125 std_roll_mean_5\n0.1838598029462956 std_first_10000\n0.18605486229008722 std_last_10000\n0.19745983463159525 std_roll_mean_10\n0.20183182255341436 std_roll_mean_1000\n0.2029996163981847 q95_roll_mean_5000\n0.2055609798898456 q95_roll_std_5000\n0.2111243261691561 MA_700MA_BB_high_mean\n0.21361790193576616 std_roll_mean_500\n0.22549908678191263 std_roll_mean_100\n0.22704498266627365 q95_roll_mean_10000\n0.22720429479208334 MA_400MA_BB_high_mean\n0.2277252172060087 std_roll_mean_50\n0.22827876838343922 min_roll_mean_5\n0.23108023386808377 Rstd__last_5000\n0.24709141117481997 std_last_50000\n0.27000213332280787 std_first_50000\n0.27677303734952885 Istd\n0.28438524475259636 std\n0.28438524475259636 abs_std\n0.34237650480689863 ave_roll_std_10000\n0.35309751343132445 ave_roll_std_5000\n0.3680619571175911 std_roll_std_100\n0.3754981387379842 MA_1000MA_std_mean\n0.37549813873798443 ave_roll_std_1000\n0.3789881760566835 mad\n0.37991944630311836 MA_700MA_std_mean\n0.38128666730706184 min_roll_mean_10\n0.3834321535688642 ave_roll_std_500\n0.3854670246608462 MA_400MA_std_mean\n0.3975221679181727 ave_roll_std_100\n0.407461900450977 ave_roll_std_50\n0.47922839120820676 std_roll_std_50\n166\n0.54764128128751 avg_last_10000\n0.5533534762173437 avg_first_10000\n0.5752748603467279 q05_roll_std_10000\n0.5881826846154409 std_roll_std_10\n0.5964971555883632 ave_roll_std_10\n0.6449476390106669 q01_roll_std_10000\n0.6786055779060366 min_roll_std_10000\n0.6810306212313717 avg_first_50000\n0.6881472646122985 Rstd\n0.6925766136063256 q05_roll_std_5000\n0.7245650811789461 avg_last_50000\n0.784230324420598 q01_roll_std_5000\n0.8252836321459884 sum\n0.8252840689068993 Rmax\n0.8285409741414174 ave_roll_mean_10000\n0.8307532443407578 min_roll_std_5000\n0.8322643240033981 Moving_average_6000_mean\n0.8332774349701242 ave_roll_mean_5000\n0.8357795904753716 Moving_average_3000_mean\n0.8379488589906899 Moving_average_1500_mean\n0.8387194923391819 ave_roll_mean_1000\n0.8391817860981752 Moving_average_700_mean\n0.8395139675099427 ave_roll_mean_500\n0.8400696571157371 exp_Moving_average_3000_mean\n0.8400800700564626 exp_Moving_average_30000_mean\n0.8401490938794745 ave_roll_mean_100\n0.8402218280197483 ave_roll_mean_50\n0.8402700764838392 exp_Moving_average_300_mean\n0.8402827435906769 ave_roll_mean_10\n0.8402923174658032 ave_roll_mean_5\n0.8411526614139948 min_roll_std_50\n0.8425622710312342 mean\n0.8425622710312342 abs_mean\n0.9181136217871926 q01_roll_mean_5\n0.9213189822416354 min_roll_std_100\n0.9213640167722567 q05_roll_std_50\n0.9235934190436899 q05_roll_std_1000\n0.9276514739498546 q05_roll_std_100\n0.9497903332390125 iqr\n0.9516002658214106 q05_roll_std_500\n0.9995954320751559 min_roll_std_500\n1.0078639982717803 min_roll_std_1000\n1.0156555017457989 q01_roll_std_1000\n1.0413376717800324 q01_roll_std_100\n1.0447792586977922 q01_roll_std_500\n1.0513639186994967 q01_roll_std_50\n1.0561066635646077 ave10\n1.0647717557824836 ave_roll_std_5\n1.0756470281006312 q01_roll_std_5\n1.0928784512811391 q05_roll_mean_5\n1.1043037604461143 min_roll_mean_50\n1.11291862267596 q01_roll_mean_10\n1.1281178400979042 q05_roll_std_5\n1.1786540160480488 MA_700MA_BB_low_mean\n1.1945782708780044 MA_400MA_BB_low_mean\n1.2462844290361097 min_roll_mean_100\n1.2612109125775708 q05_roll_mean_10000\n1.2641051886195402 q05_roll_mean_10\n1.2742882273928617 q01_roll_mean_10000\n1.2871265838288306 min_roll_mean_10000\n1.2893656842657168 min_roll_mean_500\n1.2915922620507196 q05_roll_std_10\n1.29612204013701 q01_roll_mean_5000\n1.3024681297336893 min_roll_mean_1000\n1.3029340309019297 min_roll_mean_5000\n1.3249621835542713 q05_roll_mean_5000\n1.3563992198693693 q01_roll_mean_1000\n1.3721553092386727 q01_roll_mean_500\n1.3946106231055204 q05_roll_mean_1000\n1.4082081867727783 q05_roll_mean_500\n1.4133530474909426 q01_roll_mean_100\n1.414303951276835 q01_roll_std_10\n1.4253587191447614 q05_roll_mean_100\n1.4286037531827798 q01_roll_mean_50\n1.4312582662425308 q05_roll_mean_50\n1.5750240502463575 abs_q05\n1.5750240502463575 q05\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = (train_X_mean - test_X_mean).abs().sort_values()\nfeatures = features[features > 0]\ndrop_features = features.index[167:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X_less = scaled_train_X.drop(drop_features, axis=1)\ntest_X_less = scaled_test_X.drop(drop_features, axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Cross Validation Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"n_fold = 5\ndef mae_cv (model):\n    folds = KFold(n_splits=n_fold, shuffle=True, random_state=42).get_n_splits(train_X_less.values)\n    mae = -cross_val_score (model, train_X_less.values, train_y, scoring=\"neg_mean_absolute_error\",\n                           verbose=0,\n                           cv=folds)\n    return mae","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Random Forest"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nrf_model = RandomForestRegressor(n_estimators=120, n_jobs=-1, min_samples_leaf=1, \n                           max_features = \"auto\",max_depth=15, )\n#score = mae_cv(rf_model)\n#print(\"Random Forest score: {:.4f} ({:.4f})\\n\" .format(score.mean(), score.std()))\nrf_model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Cat Boost"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nparams = {'loss_function':'MAE',}\ncat_model = CatBoostRegressor(iterations=1000,  eval_metric='MAE', verbose=False, **params)\n\n#score = mae_cv(cat_model)\n#print(\"Cat Boost score: {:.4f} ({:.4f})\\n\" .format(score.mean(), score.std()))\ncat_model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Elastic Net"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#ENet_model = make_pipeline(RobustScaler(), ElasticNet(alpha=0.0005, l1_ratio=0.9, random_state=3,max_iter=5000))\nENet_model = ElasticNet(alpha=0.0005, l1_ratio=0.9, random_state=3,max_iter=5000)\n#score = mae_cv(ENet_model)\n#print(\"Elastic Net score: {:.4f} ({:.4f})\\n\" .format(score.mean(), score.std()))\nENet_model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Lasso"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nlasso_model = Lasso(alpha =0.0005, random_state=1)\n#score = mae_cv(lasso_model)\n#print(\"Lasso score: {:.4f} ({:.4f})\\n\" .format(score.mean(), score.std()))\nlasso_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):\n    def __init__(self, models):\n        self.models = models\n        \n    # we define clones of the original models to fit the data in\n    def fit(self, X, y):\n        self.models_ = [clone(x) for x in self.models]\n        \n        # Train cloned base models\n        for model in self.models_:\n            model.fit(X, y)\n\n        return self\n    \n    #Now we do the predictions for cloned models and average them\n    def predict(self, X):\n        \n        predictions = np.column_stack([\n            model.predict(X) for model in self.models_\n        ])\n        return np.mean(predictions, axis=1)   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\naveraged_models = AveragingModels(models = (rf_model,cat_model))\n#score_rf_cat = mae_cv(averaged_models_rf_cat)\n#print(\" Averaged base models score_rf_cat: {:.4f} ({:.4f})\\n\".format(score_rf_cat.mean(), score_rf_cat.std()))\n\n#averaged_models_cat_enet = AveragingModels(models = (cat_model, ENet_model))\n#score_cat_enet = mae_cv(averaged_models_cat_enet)\n#print(\" Averaged base models score_cat_enet: {:.4f} ({:.4f})\\n\".format(score_cat_enet.mean(), score_cat_enet.std()))\n\n#averaged_models_cat_lasso = AveragingModels(models = (cat_model,lasso_model))\n#score_cat_lasso = mae_cv(averaged_models_cat_lasso)\n#print(\" Averaged base models score_cat_lasso: {:.4f} ({:.4f})\\n\".format(score_cat_lasso.mean(), score_cat_lasso.std()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"averaged_models.fit (train_X_less.values, train_y)\naveraged_train_predict = averaged_models.predict(train_X_less.values)\nprint(mean_absolute_error(train_y, averaged_train_predict))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"averaged_prediction = np.zeros(len(test_X_less))\naveraged_prediction += averaged_models.predict(test_X_less.values)\naveraged_prediction","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/LANL-Earthquake-Prediction/sample_submission.csv', index_col='seg_id')\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.time_to_failure = averaged_prediction\nsubmission.to_csv('submission_average_167_features.csv',index=True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}