{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Setup"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nfrom gplearn.genetic import SymbolicRegressor,SymbolicTransformer\nfrom gplearn.functions import make_function\n\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.preprocessing import StandardScaler\n\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.feature_selection import RFE\nfrom sklearn.feature_selection import RFECV\nfrom sklearn.model_selection import RandomizedSearchCV\n\nimport os\n\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"../input/LANL-Earthquake-Prediction\"))\nprint(os.listdir(\"../input/lanl-features\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Read Data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"X = pd.read_csv('../input/lanl-features/train_features_denoised.csv')\nX_test = pd.read_csv('../input/lanl-features/test_features_denoised.csv')\ny = pd.read_csv('../input/lanl-features/y.csv')\nsubmission = pd.read_csv('../input/LANL-Earthquake-Prediction/sample_submission.csv',index_col='seg_id')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Scaling"},{"metadata":{"trusted":true},"cell_type":"code","source":"X.drop('seg_id',axis=1,inplace=True)\nX_test.drop('seg_id',axis=1,inplace=True)\nX.drop('target',axis=1,inplace=True)\nX_test.drop('target',axis=1,inplace=True)\n\nalldata = pd.concat([X, X_test])\n\nscaler = StandardScaler()\n\nalldata = pd.DataFrame(scaler.fit_transform(alldata), columns=alldata.columns)\n\nX = alldata[:X.shape[0]]\nX_test = alldata[X.shape[0]:]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Feature Selection"},{"metadata":{},"cell_type":"markdown","source":"## Drop highly correlated features"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ncorr_matrix = X.corr()\ncorr_matrix = corr_matrix.abs()\nupper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))\nto_drop = [column for column in upper.columns if any(upper[column] > 0.95)]\n\nX = X.drop(to_drop, axis=1)\nX_test = X_test.drop(to_drop, axis=1)\nprint(X.shape)\nprint(X_test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Recursive feature elimination with cross validation and random forest regression"},{"metadata":{},"cell_type":"markdown","source":"Give training data some insight of the time range of this experiment. Little bit cheating"},{"metadata":{"trusted":true},"cell_type":"code","source":"X[\"mean_y\"] = np.full(len(y), y.values.mean())\nX[\"max_y\"] = np.full(len(y), y.values.max())\nX[\"min_y\"] = np.full(len(y), y.values.min())\nX[\"std_y\"] = np.full(len(y), y.values.std())\n\nX_test[\"mean_y\"] = np.full(len(X_test), y.values.mean())\nX_test[\"max_y\"] = np.full(len(X_test), y.values.max())\nX_test[\"min_y\"] = np.full(len(X_test), y.values.min())\nX_test[\"std_y\"] = np.full(len(X_test), y.values.std())\n\nprint(X.shape)\nprint(X_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nrf = RandomForestRegressor(n_estimators = 10)\nrfecv = RFECV(estimator=rf, step=1, cv=3, scoring='neg_mean_absolute_error', verbose=0, n_jobs=-1) #3-fold cross-validation with mae\nrfecv = rfecv.fit(X, y.values)\nprint('Optimal number of features :', rfecv.n_features_)\nprint('Best features :', X.columns[rfecv.support_])\n\nX = X[X.columns[rfecv.support_].values]\nX_test = X_test[X_test.columns[rfecv.support_].values]\nprint(X.shape)\nprint(X_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Features:\", list(X.columns))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Random Forest Hyperparameter Tuning"},{"metadata":{},"cell_type":"markdown","source":"To use RandomizedSearchCV, we first need to create a parameter grid to sample from during fitting:"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of trees in random forest\nn_estimators = [int(x) for x in np.linspace(start = 100, stop = 1000, num = 10)]\n# Number of features to consider at every split\nmax_features = ['auto', 'sqrt']\n# Maximum number of levels in tree\nmax_depth = [int(x) for x in np.linspace(10, 110, num = 11)]\nmax_depth.append(None)\n# Minimum number of samples required to split a node\nmin_samples_split = [2, 5, 10]\n# Minimum number of samples required at each leaf node\nmin_samples_leaf = [1, 2, 4]\n# Method of selecting samples for training each tree\nbootstrap = [True, False]\n# Create the random grid\nrandom_grid = {'n_estimators': n_estimators,\n               'max_features': max_features,\n               'max_depth': max_depth,\n               'min_samples_split': min_samples_split,\n               'min_samples_leaf': min_samples_leaf,\n               'bootstrap': bootstrap}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Grid Search Parameters\", random_grid)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<p>On each iteration, the algorithm will choose a difference combination of the features. Altogether, there are 2 \\* 12 \\* 2 \\* 3 \\* 3 \\* 10 = 4320 settings! However, the benefit of a random search is that we are not trying every combination, but selecting at random to sample a wide range of values.</p>"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Use the random grid to search for best hyperparameters\n# First create the base model to tune\nrf = RandomForestRegressor()\n# Random search of parameters, using 3 fold cross validation, \n# search across 100 different combinations, and use all available cores\nrf_random = RandomizedSearchCV(estimator = rf, param_distributions = random_grid, n_iter = 100, cv = 3, verbose=2, random_state=42, n_jobs = -1)\n# Fit the random search model\nrf_random.fit(X, y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The most important arguments in RandomizedSearchCV are n_iter, which controls the number of different combinations to try, and cv which is the number of folds to use for cross validation (we use 100 and 3 respectively). More iterations will cover a wider search space and more cv folds reduces the chances of overfitting, but raising each will increase the run time. Machine learning is a field of trade-offs, and performance vs time is one of the most fundamental.\n\nWe can view the best parameters from fitting the random search:"},{"metadata":{"trusted":true},"cell_type":"code","source":"rf_random.best_params_","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Evaluate Random Search"},{"metadata":{"trusted":true},"cell_type":"code","source":"def evaluate(model, features=X, labels=y):\n    predictions = model.predict(features)    \n    mae=mean_absolute_error(labels, predictions)\n    print('Model Performance')\n    print('Mean Absolute Error: {:0.4f}.'.format(mae))\n    return mae","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train a base model"},{"metadata":{"trusted":true},"cell_type":"code","source":"base_model = RandomForestRegressor(n_estimators = 10, random_state = 42)\nbase_model.fit(X, y)\nbase_mae = evaluate(base_model, X, y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train Best Random Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"best_random = rf_random.best_estimator_\nrandom_mae = evaluate(best_random, X, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Improvement of {:0.2f}%.'.format(100 * (base_mae - random_mae) / base_mae))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.time_to_failure = best_random.predict(X_test)\nsubmission.to_csv('submission.csv', index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head(10)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}